@procedures/mongo 0.47.1 → 0.48.0

package/dist/storage/materializer.d.ts

@@ -1,5 +1,5 @@
 /**
- * Materialized views driven by change streams (Graph's split plan, part P4). A view is a document
+ * Materialized views driven by change streams. A view is a document
  * of a views collection: an aggregation pipeline over source collections whose result goes into a
  * target collection, for one partition (for example one tenant). The materializer runs each view
  * when its sources change, and keeps the target equal to the result of the pipeline.
@@ -31,8 +31,11 @@
  *   of another view. The materializer writes the rows.
  * - A pass keeps in memory the key of each row of the result (at most `maxRows`) and one group of
  *   1000 rows. It keeps nothing between passes.
+ * - Each source has one change stream, for all views of all partitions (roadmap CLI-92). A stream at
+ *   rest holds one connection of the pool for its `getMore`, so the count of streams must not grow
+ *   with the partitions. The handler of the stream gives a change to the views of its partition.
  * - A change stream that ends (for example after `dropDatabase`) opens again after `rewatch`,
- *   and a pass catches up on the changes that the stream missed.
+ *   and a pass of each partition of that source catches up on the changes that the stream missed.
  * - Each stream is open on the server before the materializer reads what it watches: the views
  *   before the read of the views, the sources before a run. A change between the two reaches both.
  * - The status of each view (`ok`, `errored`, `cycle`) goes into the field `statusField` of its
@@ -299,9 +302,15 @@ export declare class ViewMaterializer {
     private db;
     private viewsStream;
     private readonly running;
+    /** The one change stream of each source that an active view reads (roadmap CLI-92). */
+    private readonly sources;
     /** The collections that changed since the last pass, for each partition. */
     private readonly dirty;
     private readonly timers;
+    /** The time of the last change of each partition with a timer: the end of the wait of the pass. */
+    private readonly lastChange;
+    /** The passes that run at one time. A burst that reaches each partition does not fill the pool. */
+    private passes;
     /** The timers of the streams that open again, and of the ends of the grace times. */
     private readonly retries;
     private readonly clock;
@@ -341,7 +350,7 @@ export declare class ViewMaterializer {
      * deleted item) is not a row of a source (roadmap CLI-79).
      */
     private scoped;
-    /** The change streams that are open: the views stream and the source streams of the active views. */
+    /** The change streams that are open: the views stream and one stream for each source of the active views. */
     openStreamCount(): number;
     /** The passes, rebalances and status writes in flight. */
     inFlight(): number;
@@ -389,19 +398,18 @@ export declare class ViewMaterializer {
     private activate;
     private tearDown;
     /**
-     * The filter of the change stream of a source for a partition. It only removes the events that
-     * cannot change a view of the partition: an event whose partition is unknown passes, and the
-     * pass of the partition reads its own documents only.
+     * The partitions that a change of a source can reach (roadmap CLI-92). This list only removes the
+     * partitions that the change cannot touch: the pass of a partition reads its own documents only.
      *
      * - An insert, a replace or an update has the document after the change (`fullDocument`).
      * - With a pre-image (a collection with `changeStreamPreAndPostImages`), each change also has the
      *   document before it: a delete, and an update that moves a document out of the partition.
      * - Without a pre-image, a delete, a replace and an update of the partition field (a top-level
-     *   field) can leave any partition: they pass. In a source of `storageSources`, the handler then
-     *   reads the partition from the id (`isOtherPartition`).
-     * - The handler checks the fields of an update against `sourceFields` (`touchesFields`).
+     *   field) can leave any partition: the change reaches each partition of the source. In a source
+     *   of `storageSources`, the handler then reads the partition from the id (`isOtherPartition`).
+     * - The handler checks the fields of an update against `sourceFields` (`touchesNoField`).
      */
-    private sourceMatch;
+    private partitionsOfChange;
     /**
      * True for an update that changes no field of `fields` and not the partition field. The keys of
      * an update are paths with dots (`tags.3` after a `$push`, `a.b` after a `$set` of `a.b`), so a key
@@ -421,12 +429,36 @@ export declare class ViewMaterializer {
      */
     private isOtherPartition;
     private isCurrent;
-    /** This method opens the stream of a source for a view. The promise resolves when the stream is open on the server. */
-    private watchSource;
+    /**
+     * This method adds a view to the stream of each of its sources, and opens a stream that does not
+     * exist. The promise resolves when each stream is open on the server, so a change after that
+     * reaches the view.
+     */
+    private watchSources;
+    /** This method removes a view from its streams. The stream of a source with no view left closes. */
+    private unwatch;
+    /** This method closes the stream of a source and forgets it. */
+    private dropWatch;
+    /** This method opens the stream of a source. The promise resolves when the stream is open on the server. */
+    private openSource;
+    /**
+     * This method opens the stream of a source again after `rewatch`. The changes while no stream was
+     * open are lost, so a pass of each partition of the source catches up, after the new stream is
+     * open. A view that joins before then is in that pass.
+     */
+    private afterStreamEnd;
+    /** This method gives a change of a source to the views of its partitions. */
+    private route;
     /** This method runs a function after `rewatch`, unless the materializer stops. The timer does not keep the process alive. */
     private later;
     /** This method marks a collection as changed, and starts the pass of the partition after the burst. */
     private schedule;
+    /**
+     * This method starts the timer of the pass of a partition. A change during the wait moves the end
+     * of the wait and makes no new timer: 1,000 deletes with no pre-image at 1,000 partitions are a
+     * million changes. A pass waits for its place in `passes` after the work of its partition before it.
+     */
+    private waitForPass;
     /** The views of a partition, in the order of their dependencies, and the views in a cycle. */
     private order;
     /** This method moves the views of a partition between active and cyclic, and runs each view that becomes active. */

package/dist/storage/materializer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"materializer.d.ts","sourceRoot":"","sources":["../../src/storage/materializer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqDG;AAIH,OAAO,EAAQ,QAAQ,EAAsF,KAAK,EAAE,EAAE,KAAK,QAAQ,EAAE,MAAM,SAAS,CAAC;AACrJ,OAAO,EAA2E,KAAK,KAAK,EAA8B,KAAK,QAAQ,EAAkB,KAAK,MAAM,EAAE,MAAM,oBAAoB,CAAC;AACjM,OAAO,EAAgD,oBAAoB,EAAgE,KAAK,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAE1L,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAK/D,uEAAuE;AACvE,MAAM,WAAW,gBAAgB;IAC/B,GAAG,EAAE,MAAM,CAAC;IACZ,uHAAuH;IACvH,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,kDAAkD;IAClD,QAAQ,EAAE,QAAQ,EAAE,CAAC;IACrB,2CAA2C;IAC3C,gBAAgB,EAAE,MAAM,CAAC;IACzB;;;;;OAKG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;IACxC,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC;CAC1B;AAED,wEAAwE;AACxE,MAAM,WAAW,sBAAsB;IACrC,KAAK,EAAE,IAAI,GAAG,SAAS,GAAG,OAAO,CAAC;IAClC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,+DAA+D;IAC/D,EAAE,EAAE,MAAM,CAAC;IACX,sDAAsD;IACtD,UAAU,CAAC,EAAE,QAAQ,CAAC;IACtB,gHAAgH;IAChH,QAAQ,CAAC,EAAE;QAAE,UAAU,EAAE,QAAQ,GAAG,IAAI,CAAC;QAAC,SAAS,EAAE,IAAI,CAAA;KAAE,CAAC;IAC5D,0GAA0G;IAC1G,KAAK,CAAC,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,QAAQ,CAAC;QAAC,KAAK,EAAE,IAAI,CAAA;KAAE,CAAC;CAC9D;AAED;;;GAGG;AACH,MAAM,MAAM,iBAAiB,GAAG,CAAC,QAAQ,EAAE,QAAQ,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,gBAAgB,EAAE,KAAK,EAAE,QAAQ,KAAK,QAAQ,EAAE,CAAC;AAEjI,2CAA2C;AAC3C,MAAM,WAAW,uBAAuB;IACtC,8DAA8D;IAC9D,EAAE,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,EAAE,CAAC,CAAC;IAC3B,sEAAsE;IACtE,KAAK,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3B,sHAAsH;IACtH,cAAc,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACpC,yFAAyF;IACzF,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACjC;;;OAGG;IACH,KAAK,CAAC,EAAE,iBAAiB,GAAG,SAAS,CAAC;IACtC,qEAAqE;IACrE,QAAQ,CAAC,EAAE,QAAQ,GAAG,SAAS,CAAC;IAChC,4EAA4E;IAC5E,OAAO,CAAC,EAAE,QAAQ,GAAG,SAAS,CAAC;IAC/B,qDAAqD;IACrD,OAAO,CAAC,EAAE,QAAQ,GAAG,SAAS,CAAC;IAC/B,uGAAuG;IACvG,OAAO,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B;;;;OAIG;IACH,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B;;;OAGG;IACH,GAAG,CAAC,EAAE,CAAC,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,EAAE,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC,GAAG,SAAS,CAAC;IAChF,sIAAsI;IACtI,KAAK,CAAC,EAAE,KAAK,GAAG,SAAS,CAAC;IAC1B;;;;;;OAMG;IACH,QAAQ,CAAC,EAAE,YAAY,GAAG,SAAS,CAAC;IACpC,iHAAiH;IACjH,kBAAkB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACxC;;;OAGG;IACH,eAAe,CAAC,EAAE,QAAQ,GAAG,SAAS,CAAC;IACvC;;;OAGG;IACH,SAAS,CAAC,EAAE,QAAQ,GAAG,SAAS,CAAC;IACjC;;;;;;;;OAQG;IACH,cAAc,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,eAAe,GAAG,iBAAiB,CAAC,CAAC,GAAG,SAAS,CAAC;IAC3F;;;;;;;;;;;;OAYG;IACH,cAAc,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,sBAAsB,CAAC,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,MAAM,KAAK,sBAAsB,GAAG,SAAS,CAAC,GAAG,SAAS,CAAC;IAC7I;;;;OAIG;IACH,cAAc,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,OAAO,KAAK,MAAM,CAAC,GAAG,SAAS,CAAC;IACvD;;;;;;;OAOG;IACH,OAAO,CAAC,EAAE,oBAAoB,GAAG,SAAS,CAAC;IAC3C,8GAA8G;IAC9G,SAAS,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,gBAAgB,EAAE,SAAS,EAAE,MAAM,KAAK,MAAM,GAAG,SAAS,CAAC,GAAG,SAAS,CAAC;IAC5F;;;OAGG;IACH,OAAO,CAAC,EAAE,CAAC,CAAC,QAAQ,EAAE,OAAO,KAAK,OAAO,CAAC,GAAG,SAAS,CAAC;CACxD;AAED;;;GAGG;AACH,KAAK,kBAAkB,GAAG,oBAAoB,CAAC,GAAG,CAAC,CAAC;AAEpD,wGAAwG;AACxG,MAAM,MAAM,sBAAsB,GAAG,kBAAkB,GAAG,UAAU,GAAG,iBAAiB,CAAC;AAEzF;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,MAAM,WAAW,iBAAiB;IAChC,2KAA2K;IAC3K,MAAM,EAAE,kBAAkB,GAAG,UAAU,CAAC;IACxC,4IAA4I;IAC5I,kBAAkB,CAAC,EAAE,CAAC,CAAC,aAAa,EAAE,MAAM,KAAK,MAAM,GAAG,SAAS,CAAC,GAAG,SAAS,CAAC;CAClF;AAED,mEAAmE;AACnE,MAAM,MAAM,YAAY,GAAG,CAAC,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,UAAU,EAAE,QAAQ,KAAK,OAAO,CAAC;AAE7F,iHAAiH;AACjH,eAAO,MAAM,iBAAiB,EAAE,YAA6E,CAAC;AAE9G,8DAA8D;AAC9D,eAAO,MAAM,qBAAqB,aAAa,CAAC;AAChD,uEAAuE;AACvE,eAAO,MAAM,eAAe,cAAc,CAAC;AAE3C;;;;GAIG;AACH,wBAAgB,UAAU,CAAC,IAAI,EAAE,gBAAgB,EAAE,OAAO,GAAE;IAAE,cAAc,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;CAAO,GAAG,QAAQ,CAIpJ;AAyCD,sFAAsF;AACtF,wBAAgB,YAAY,CAAC,QAAQ,EAAE,OAAO,GAAG,OAAO,CAEvD;AAED;;;;;GAKG;AACH,wBAAgB,mBAAmB,CAAC,EAAE,EAAE,OAAO,GAAG,MAAM,CAIvD;AA6CD,gIAAgI;AAChI,MAAM,MAAM,eAAe,GAAG,CAAC,gBAAgB,EAAE,MAAM,KAAK,MAAM,GAAG,SAAS,CAAC;AAE/E,kEAAkE;AAClE,eAAO,MAAM,iBAAiB,EAAE,eAAwD,CAAC;AAEzF,0EAA0E;AAC1E,MAAM,WAAW,iBAAiB;IAChC,+HAA+H;IAC/H,IAAI,EAAE,IAAI,CAAC,UAAU,EAAE,aAAa,CAAC,CAAC;IACtC,oEAAoE;IACpE,eAAe,EAAE,eAAe,CAAC;CAClC;AAoDD;;;;;;;;;;GAUG;AACH,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAA0B;IAClD,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAS;IACxC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAS;IACpC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,EAAE,CAAiB;IAC3B,OAAO,CAAC,WAAW,CAA2B;IAC9C,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAkC;IAC1D,4EAA4E;IAC5E,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAkC;IACxD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA6B;IACpD,qFAAqF;IACrF,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAqB;IAC7C,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAQ;IAC9B,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAe;IACxC,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAS;IAC5C,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAyC;IACxE,+HAA+H;IAC/H,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAuF;IAC1H,mIAAmI;IACnI,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAyE;IACvG,OAAO,CAAC,QAAQ,CAAC,cAAc,CAA0B;IACzD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAiC;IACzD,2DAA2D;IAC3D,OAAO,CAAC,MAAM,CAAS;IACvB,gFAAgF;IAChF,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAoC;IAC3D,OAAO,CAAC,QAAQ,CAAC,OAAO,CAA+B;IACvD,gIAAgI;IAChI,OAAO,CAAC,SAAS,CAAuC;IACxD,OAAO,CAAC,OAAO,CAAQ;IAEvB,YAAY,OAAO,EAAE,uBAAuB,EA8B3C;IAED;;;;OAIG;IACH,OAAO,CAAC,eAAe;IAqCvB,kIAAkI;IAClI,OAAO,CAAC,gBAAgB;IAOxB,wIAAwI;IACxI,OAAO,CAAC,GAAG;IAMX;;;;OAIG;IACH,OAAO,CAAC,MAAM;IAWd,qGAAqG;IACrG,eAAe,IAAI,MAAM,CAIxB;IAED,0DAA0D;IAC1D,QAAQ,IAAI,MAAM,CAEjB;IAED,4FAA4F;IAC5F,KAAK,IAAI,GAAG,CAAC,MAAM,EAAE;QAAE,KAAK,EAAE,QAAQ,GAAG,QAAQ,CAAC;QAAC,OAAO,EAAE,MAAM,EAAE,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC,CAEtF;IAED,iHAAiH;IAC3G,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAQ3B;IAED,+GAA+G;IAC/G,UAAU,IAAI,IAAI,CAKjB;IAED;;;OAGG;IACG,KAAK,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAWjE;IAED,iGAAiG;IAC3F,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAS3B;IAED,qGAAqG;IAC/F,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAK1B;IAED,sFAAsF;IACtF,OAAO,CAAC,KAAK;IASb,uFAAuF;IACvF,OAAO,CAAC,MAAM;IAcd,OAAO,CAAC,KAAK;IAIb,OAAO,CAAC,WAAW;IAKnB;;;OAGG;IACH,OAAO,CAAC,WAAW;IAMnB;;;OAGG;YACW,MAAM;IAUpB,6GAA6G;IAC7G,OAAO,CAAC,UAAU;YA0CJ,YAAY;IA4B1B,mHAAmH;IACnH,OAAO,CAAC,QAAQ;IAchB,qHAAqH;YACvG,QAAQ;IAuBtB,OAAO,CAAC,QAAQ;IAQhB;;;;;;;;;;;;OAYG;IACH,OAAO,CAAC,WAAW;IAenB;;;;;;OAMG;IACH,OAAO,CAAC,cAAc;IAQtB;;;;;;;;OAQG;IACH,OAAO,CAAC,gBAAgB;IAyBxB,OAAO,CAAC,SAAS;IAIjB,uHAAuH;IACvH,OAAO,CAAC,WAAW;IA0CnB,6HAA6H;IAC7H,OAAO,CAAC,KAAK;IASb,uGAAuG;IACvG,OAAO,CAAC,QAAQ;IAiBhB,8FAA8F;IAC9F,OAAO,CAAC,KAAK;IA2Bb,oHAAoH;IACpH,OAAO,CAAC,SAAS;IAyBjB,uGAAuG;YACzF,IAAI;IAclB,oGAAoG;YACtF,iBAAiB;IAmB/B;;;OAGG;YACW,GAAG;IA0CjB,qGAAqG;IACrG,OAAO,CAAC,KAAK;IAOb;;;;OAIG;IACH,OAAO,CAAC,eAAe;IAqCvB;;;OAGG;YACW,QAAQ;IAUtB,0GAA0G;YAC5F,kBAAkB;IAKhC;;;;OAIG;YACW,kBAAkB;IAehC,wIAAwI;YACzH,YAAY;IAwD3B;;;;;;;OAOG;IACH,OAAO,CAAC,YAAY;IA6BpB,OAAO,CAAC,SAAS;IAIjB;;;;;;;;;OASG;YACW,gBAAgB;IAmD9B;;;;;OAKG;IACH,OAAO,CAAC,WAAW;IAmBnB;;;;OAIG;YACW,WAAW;IA+CzB;;;;;OAKG;YACW,aAAa;IAmB3B;;;;;;OAMG;YACY,eAAe;IAgC9B;;;;OAIG;IACH,OAAO,CAAC,iBAAiB;IAoBzB;;;;;;OAMG;YACW,uBAAuB;IA+CrC;;;;;;OAMG;YACW,kBAAkB;IA8ChC,8EAA8E;IAC9E,OAAO,CAAC,aAAa;IAIrB;;;;;OAKG;YACW,aAAa;IAoB3B,6HAA6H;YAC/G,SAAS;IASvB,wHAAwH;YAC1G,UAAU;IAUxB,sEAAsE;YACxD,YAAY;IAI1B,sFAAsF;YACxE,WAAW;CAY1B"}
+{"version":3,"file":"materializer.d.ts","sourceRoot":"","sources":["../../src/storage/materializer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAwDG;AAIH,OAAO,EAAQ,QAAQ,EAAsF,KAAK,EAAE,EAAE,KAAK,QAAQ,EAAE,MAAM,SAAS,CAAC;AACrJ,OAAO,EAAsF,KAAK,KAAK,EAA8B,KAAK,QAAQ,EAAkB,KAAK,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAC5M,OAAO,EAAgD,oBAAoB,EAAgE,KAAK,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAE1L,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAK/D,uEAAuE;AACvE,MAAM,WAAW,gBAAgB;IAC/B,GAAG,EAAE,MAAM,CAAC;IACZ,uHAAuH;IACvH,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,kDAAkD;IAClD,QAAQ,EAAE,QAAQ,EAAE,CAAC;IACrB,2CAA2C;IAC3C,gBAAgB,EAAE,MAAM,CAAC;IACzB;;;;;OAKG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;IACxC,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC;CAC1B;AAED,wEAAwE;AACxE,MAAM,WAAW,sBAAsB;IACrC,KAAK,EAAE,IAAI,GAAG,SAAS,GAAG,OAAO,CAAC;IAClC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,+DAA+D;IAC/D,EAAE,EAAE,MAAM,CAAC;IACX,sDAAsD;IACtD,UAAU,CAAC,EAAE,QAAQ,CAAC;IACtB,gHAAgH;IAChH,QAAQ,CAAC,EAAE;QAAE,UAAU,EAAE,QAAQ,GAAG,IAAI,CAAC;QAAC,SAAS,EAAE,IAAI,CAAA;KAAE,CAAC;IAC5D,0GAA0G;IAC1G,KAAK,CAAC,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,QAAQ,CAAC;QAAC,KAAK,EAAE,IAAI,CAAA;KAAE,CAAC;CAC9D;AAED;;;GAGG;AACH,MAAM,MAAM,iBAAiB,GAAG,CAAC,QAAQ,EAAE,QAAQ,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,gBAAgB,EAAE,KAAK,EAAE,QAAQ,KAAK,QAAQ,EAAE,CAAC;AAEjI,2CAA2C;AAC3C,MAAM,WAAW,uBAAuB;IACtC,8DAA8D;IAC9D,EAAE,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,EAAE,CAAC,CAAC;IAC3B,sEAAsE;IACtE,KAAK,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3B,sHAAsH;IACtH,cAAc,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACpC,yFAAyF;IACzF,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACjC;;;OAGG;IACH,KAAK,CAAC,EAAE,iBAAiB,GAAG,SAAS,CAAC;IACtC,qEAAqE;IACrE,QAAQ,CAAC,EAAE,QAAQ,GAAG,SAAS,CAAC;IAChC,4EAA4E;IAC5E,OAAO,CAAC,EAAE,QAAQ,GAAG,SAAS,CAAC;IAC/B,qDAAqD;IACrD,OAAO,CAAC,EAAE,QAAQ,GAAG,SAAS,CAAC;IAC/B,uGAAuG;IACvG,OAAO,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B;;;;OAIG;IACH,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B;;;OAGG;IACH,GAAG,CAAC,EAAE,CAAC,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,EAAE,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC,GAAG,SAAS,CAAC;IAChF,sIAAsI;IACtI,KAAK,CAAC,EAAE,KAAK,GAAG,SAAS,CAAC;IAC1B;;;;;;OAMG;IACH,QAAQ,CAAC,EAAE,YAAY,GAAG,SAAS,CAAC;IACpC,iHAAiH;IACjH,kBAAkB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACxC;;;OAGG;IACH,eAAe,CAAC,EAAE,QAAQ,GAAG,SAAS,CAAC;IACvC;;;OAGG;IACH,SAAS,CAAC,EAAE,QAAQ,GAAG,SAAS,CAAC;IACjC;;;;;;;;OAQG;IACH,cAAc,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,eAAe,GAAG,iBAAiB,CAAC,CAAC,GAAG,SAAS,CAAC;IAC3F;;;;;;;;;;;;OAYG;IACH,cAAc,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,sBAAsB,CAAC,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,MAAM,KAAK,sBAAsB,GAAG,SAAS,CAAC,GAAG,SAAS,CAAC;IAC7I;;;;OAIG;IACH,cAAc,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,OAAO,KAAK,MAAM,CAAC,GAAG,SAAS,CAAC;IACvD;;;;;;;OAOG;IACH,OAAO,CAAC,EAAE,oBAAoB,GAAG,SAAS,CAAC;IAC3C,8GAA8G;IAC9G,SAAS,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,gBAAgB,EAAE,SAAS,EAAE,MAAM,KAAK,MAAM,GAAG,SAAS,CAAC,GAAG,SAAS,CAAC;IAC5F;;;OAGG;IACH,OAAO,CAAC,EAAE,CAAC,CAAC,QAAQ,EAAE,OAAO,KAAK,OAAO,CAAC,GAAG,SAAS,CAAC;CACxD;AAED;;;GAGG;AACH,KAAK,kBAAkB,GAAG,oBAAoB,CAAC,GAAG,CAAC,CAAC;AAEpD,wGAAwG;AACxG,MAAM,MAAM,sBAAsB,GAAG,kBAAkB,GAAG,UAAU,GAAG,iBAAiB,CAAC;AAEzF;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,MAAM,WAAW,iBAAiB;IAChC,2KAA2K;IAC3K,MAAM,EAAE,kBAAkB,GAAG,UAAU,CAAC;IACxC,4IAA4I;IAC5I,kBAAkB,CAAC,EAAE,CAAC,CAAC,aAAa,EAAE,MAAM,KAAK,MAAM,GAAG,SAAS,CAAC,GAAG,SAAS,CAAC;CAClF;AAED,mEAAmE;AACnE,MAAM,MAAM,YAAY,GAAG,CAAC,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,UAAU,EAAE,QAAQ,KAAK,OAAO,CAAC;AAE7F,iHAAiH;AACjH,eAAO,MAAM,iBAAiB,EAAE,YAA6E,CAAC;AAE9G,8DAA8D;AAC9D,eAAO,MAAM,qBAAqB,aAAa,CAAC;AAChD,uEAAuE;AACvE,eAAO,MAAM,eAAe,cAAc,CAAC;AAE3C;;;;GAIG;AACH,wBAAgB,UAAU,CAAC,IAAI,EAAE,gBAAgB,EAAE,OAAO,GAAE;IAAE,cAAc,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;CAAO,GAAG,QAAQ,CAIpJ;AAsDD,sFAAsF;AACtF,wBAAgB,YAAY,CAAC,QAAQ,EAAE,OAAO,GAAG,OAAO,CAEvD;AAED;;;;;GAKG;AACH,wBAAgB,mBAAmB,CAAC,EAAE,EAAE,OAAO,GAAG,MAAM,CAIvD;AA6CD,gIAAgI;AAChI,MAAM,MAAM,eAAe,GAAG,CAAC,gBAAgB,EAAE,MAAM,KAAK,MAAM,GAAG,SAAS,CAAC;AAE/E,kEAAkE;AAClE,eAAO,MAAM,iBAAiB,EAAE,eAAwD,CAAC;AAEzF,0EAA0E;AAC1E,MAAM,WAAW,iBAAiB;IAChC,+HAA+H;IAC/H,IAAI,EAAE,IAAI,CAAC,UAAU,EAAE,aAAa,CAAC,CAAC;IACtC,oEAAoE;IACpE,eAAe,EAAE,eAAe,CAAC;CAClC;AAgED;;;;;;;;;;GAUG;AACH,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAA0B;IAClD,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAS;IACxC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAS;IACpC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,EAAE,CAAiB;IAC3B,OAAO,CAAC,WAAW,CAA2B;IAC9C,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAkC;IAC1D,uFAAuF;IACvF,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAkC;IAC1D,4EAA4E;IAC5E,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAkC;IACxD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA6B;IACpD,mGAAmG;IACnG,OAAO,CAAC,QAAQ,CAAC,UAAU,CAA6B;IACxD,mGAAmG;IACnG,OAAO,CAAC,MAAM,CAA8B;IAC5C,qFAAqF;IACrF,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAqB;IAC7C,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAQ;IAC9B,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAe;IACxC,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAS;IAC5C,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAyC;IACxE,+HAA+H;IAC/H,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAuF;IAC1H,mIAAmI;IACnI,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAyE;IACvG,OAAO,CAAC,QAAQ,CAAC,cAAc,CAA0B;IACzD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAiC;IACzD,2DAA2D;IAC3D,OAAO,CAAC,MAAM,CAAS;IACvB,gFAAgF;IAChF,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAoC;IAC3D,OAAO,CAAC,QAAQ,CAAC,OAAO,CAA+B;IACvD,gIAAgI;IAChI,OAAO,CAAC,SAAS,CAAuC;IACxD,OAAO,CAAC,OAAO,CAAQ;IAEvB,YAAY,OAAO,EAAE,uBAAuB,EA8B3C;IAED;;;;OAIG;IACH,OAAO,CAAC,eAAe;IAqCvB,kIAAkI;IAClI,OAAO,CAAC,gBAAgB;IAOxB,wIAAwI;IACxI,OAAO,CAAC,GAAG;IAMX;;;;OAIG;IACH,OAAO,CAAC,MAAM;IAWd,6GAA6G;IAC7G,eAAe,IAAI,MAAM,CAIxB;IAED,0DAA0D;IAC1D,QAAQ,IAAI,MAAM,CAEjB;IAED,4FAA4F;IAC5F,KAAK,IAAI,GAAG,CAAC,MAAM,EAAE;QAAE,KAAK,EAAE,QAAQ,GAAG,QAAQ,CAAC;QAAC,OAAO,EAAE,MAAM,EAAE,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC,CAEtF;IAED,iHAAiH;IAC3G,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAS3B;IAED,+GAA+G;IAC/G,UAAU,IAAI,IAAI,CAMjB;IAED;;;OAGG;IACG,KAAK,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAWjE;IAED,iGAAiG;IAC3F,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAS3B;IAED,qGAAqG;IAC/F,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAK1B;IAED,sFAAsF;IACtF,OAAO,CAAC,KAAK;IASb,uFAAuF;IACvF,OAAO,CAAC,MAAM;IAcd,OAAO,CAAC,KAAK;IAIb,OAAO,CAAC,WAAW;IAKnB;;;OAGG;IACH,OAAO,CAAC,WAAW;IAMnB;;;OAGG;YACW,MAAM;IAUpB,6GAA6G;IAC7G,OAAO,CAAC,UAAU;YA0CJ,YAAY;IA4B1B,mHAAmH;IACnH,OAAO,CAAC,QAAQ;IAchB,qHAAqH;YACvG,QAAQ;IAuBtB,OAAO,CAAC,QAAQ;IAQhB;;;;;;;;;;;OAWG;IACH,OAAO,CAAC,kBAAkB;IAe1B;;;;;;OAMG;IACH,OAAO,CAAC,cAAc;IAQtB;;;;;;;;OAQG;IACH,OAAO,CAAC,gBAAgB;IAyBxB,OAAO,CAAC,SAAS;IAIjB;;;;OAIG;IACH,OAAO,CAAC,YAAY;IAkBpB,oGAAoG;IACpG,OAAO,CAAC,OAAO;IAYf,gEAAgE;IAChE,OAAO,CAAC,SAAS;IAOjB,4GAA4G;IAC5G,OAAO,CAAC,UAAU;IAgClB;;;;OAIG;IACH,OAAO,CAAC,cAAc;IAUtB,6EAA6E;IAC7E,OAAO,CAAC,KAAK;IAab,6HAA6H;IAC7H,OAAO,CAAC,KAAK;IASb,uGAAuG;IACvG,OAAO,CAAC,QAAQ;IAShB;;;;OAIG;IACH,OAAO,CAAC,WAAW;IAkBnB,8FAA8F;IAC9F,OAAO,CAAC,KAAK;IA2Bb,oHAAoH;IACpH,OAAO,CAAC,SAAS;IAwBjB,uGAAuG;YACzF,IAAI;IAclB,oGAAoG;YACtF,iBAAiB;IAmB/B;;;OAGG;YACW,GAAG;IA0CjB,qGAAqG;IACrG,OAAO,CAAC,KAAK;IAOb;;;;OAIG;IACH,OAAO,CAAC,eAAe;IAqCvB;;;OAGG;YACW,QAAQ;IAUtB,0GAA0G;YAC5F,kBAAkB;IAKhC;;;;OAIG;YACW,kBAAkB;IAehC,wIAAwI;YACzH,YAAY;IAwD3B;;;;;;;OAOG;IACH,OAAO,CAAC,YAAY;IA6BpB,OAAO,CAAC,SAAS;IAIjB;;;;;;;;;OASG;YACW,gBAAgB;IAmD9B;;;;;OAKG;IACH,OAAO,CAAC,WAAW;IAmBnB;;;;OAIG;YACW,WAAW;IA+CzB;;;;;OAKG;YACW,aAAa;IAmB3B;;;;;;OAMG;YACY,eAAe;IAgC9B;;;;OAIG;IACH,OAAO,CAAC,iBAAiB;IAoBzB;;;;;;OAMG;YACW,uBAAuB;IA+CrC;;;;;;OAMG;YACW,kBAAkB;IA8ChC,8EAA8E;IAC9E,OAAO,CAAC,aAAa;IAIrB;;;;;OAKG;YACW,aAAa;IAoB3B,6HAA6H;YAC/G,SAAS;IASvB,wHAAwH;YAC1G,UAAU;IAUxB,sEAAsE;YACxD,YAAY;IAI1B,sFAAsF;YACxE,WAAW;CAY1B"}

package/dist/storage/materializer.js

@@ -1,5 +1,5 @@
 /**
- * Materialized views driven by change streams (Graph's split plan, part P4). A view is a document
+ * Materialized views driven by change streams. A view is a document
  * of a views collection: an aggregation pipeline over source collections whose result goes into a
  * target collection, for one partition (for example one tenant). The materializer runs each view
  * when its sources change, and keeps the target equal to the result of the pipeline.
@@ -31,8 +31,11 @@
  *   of another view. The materializer writes the rows.
  * - A pass keeps in memory the key of each row of the result (at most `maxRows`) and one group of
  *   1000 rows. It keeps nothing between passes.
+ * - Each source has one change stream, for all views of all partitions (roadmap CLI-92). A stream at
+ *   rest holds one connection of the pool for its `getMore`, so the count of streams must not grow
+ *   with the partitions. The handler of the stream gives a change to the views of its partition.
  * - A change stream that ends (for example after `dropDatabase`) opens again after `rewatch`,
- *   and a pass catches up on the changes that the stream missed.
+ *   and a pass of each partition of that source catches up on the changes that the stream missed.
  * - Each stream is open on the server before the materializer reads what it watches: the views
  *   before the read of the views, the sources before a run. A change between the two reaches both.
  * - The status of each view (`ok`, `errored`, `cycle`) goes into the field `statusField` of its
@@ -55,7 +58,7 @@
 import { createHash } from "node:crypto";
 import { isDeepStrictEqual } from "node:util";
 import { BSON, ObjectId } from "mongodb";
-import { collectionChangeTopic, ms, optionalMs, realClock, writeLog } from "@procedures/client";
+import { collectionChangeTopic, ms, optionalMs, realClock, Semaphore, writeLog } from "@procedures/client";
 import { DEFAULT_STORED_FORM, DOCUMENT_STORAGE_FIELDS, MongoDocumentStorage, TOMBSTONE_FIELD, storageWriteTarget } from "./document-storage.js";
 import { scopePipeline } from "./pipeline-scope.js";
 /** The fields of a storage that a row never keeps: its sequence number and its tombstone flag (roadmap CLI-77). */
@@ -102,6 +105,17 @@ function groups(items) {
 const CONFLICT_RETRIES = 3;
 /** The most ids of a hint of a target not in a stored form. More ids give a reset hint. */
 const MAX_HINT_IDS = 10_000;
+/** The most passes that run at one time: about one third of the default pool of the driver (100). */
+const MOST_PASSES = 32;
+/**
+ * This function gives the most passes that run at one time on a pool. A pass uses one connection at
+ * a time, so two thirds of the pool stay free for the other operations of the process.
+ */
+function passLimit(db) {
+    // A fake database of a test has no client
+    const pool = db.client?.options.maxPoolSize;
+    return pool ? Math.max(1, Math.min(MOST_PASSES, Math.floor(pool / 3))) : MOST_PASSES;
+}
 /** The `_id` of a row in the result, from the stored `_id` of `partitionTargetId`. */
 export function defaultRowId(storedId) {
     return typeof storedId === "object" && storedId !== null && "id" in storedId ? storedId.id : storedId;
@@ -192,9 +206,15 @@ export class ViewMaterializer {
     db;
     viewsStream;
     running = new Map();
+    /** The one change stream of each source that an active view reads (roadmap CLI-92). */
+    sources = new Map();
     /** The collections that changed since the last pass, for each partition. */
     dirty = new Map();
     timers = new Map();
+    /** The time of the last change of each partition with a timer: the end of the wait of the pass. */
+    lastChange = new Map();
+    /** The passes that run at one time. A burst that reaches each partition does not fill the pool. */
+    passes = new Semaphore(MOST_PASSES);
     /** The timers of the streams that open again, and of the ends of the grace times. */
     retries = new Set();
     clock;
@@ -325,11 +345,12 @@ export class ViewMaterializer {
         // No stamp: the materializer writes the rows, so the guard refuses a $merge of the pipeline
         return scopePipeline(pipeline, { match, collections: view.collections });
     }
-    /** The change streams that are open: the views stream and the source streams of the active views. */
+    /** The change streams that are open: the views stream and one stream for each source of the active views. */
     openStreamCount() {
         let count = this.viewsStream ? 1 : 0;
-        for (const view of this.running.values())
-            count += view.streams.length;
+        for (const watch of this.sources.values())
+            if (watch.stream)
+                count++;
         return count;
     }
     /** The passes, rebalances and status writes in flight. */
@@ -345,6 +366,7 @@ export class ViewMaterializer {
         this.stopped = false;
         this.intake = true;
         this.db = await this.options.db();
+        this.passes = new Semaphore(passLimit(this.db));
         // The stream of the views opens before the read of the views: a change between the two reaches both (roadmap O63)
         await this.watchViews();
         const count = await this.resync();
@@ -356,6 +378,7 @@ export class ViewMaterializer {
         for (const cancel of this.timers.values())
             cancel();
         this.timers.clear();
+        this.lastChange.clear();
         this.dirty.clear();
     }
     /**
@@ -533,7 +556,7 @@ export class ViewMaterializer {
             this.log("warn", "The view needs collections, a pipeline and a targetCollection: it does not run", { view: view._id, partition }, `the view ${view._id} needs collections, a pipeline and a targetCollection: it does not run`);
             return false;
         }
-        this.running.set(this.keyOf(partition, view._id), { view, partition, state: "active", streams: [] });
+        this.running.set(this.keyOf(partition, view._id), { view, partition, state: "active", watched: [] });
         return true;
     }
     /** This method makes the index of the target of a registered view, and rebalances its partition: a new view runs. */
@@ -564,37 +587,38 @@ export class ViewMaterializer {
         const view = this.running.get(key);
         if (!view)
             return;
-        for (const stream of view.streams)
-            stream.close().catch(() => undefined);
+        this.unwatch(view);
         this.running.delete(key);
         this.storageClaims.delete(key);
     }
     /**
-     * The filter of the change stream of a source for a partition. It only removes the events that
-     * cannot change a view of the partition: an event whose partition is unknown passes, and the
-     * pass of the partition reads its own documents only.
+     * The partitions that a change of a source can reach (roadmap CLI-92). This list only removes the
+     * partitions that the change cannot touch: the pass of a partition reads its own documents only.
      *
      * - An insert, a replace or an update has the document after the change (`fullDocument`).
      * - With a pre-image (a collection with `changeStreamPreAndPostImages`), each change also has the
      *   document before it: a delete, and an update that moves a document out of the partition.
      * - Without a pre-image, a delete, a replace and an update of the partition field (a top-level
-     *   field) can leave any partition: they pass. In a source of `storageSources`, the handler then
-     *   reads the partition from the id (`isOtherPartition`).
-     * - The handler checks the fields of an update against `sourceFields` (`touchesFields`).
+     *   field) can leave any partition: the change reaches each partition of the source. In a source
+     *   of `storageSources`, the handler then reads the partition from the id (`isOtherPartition`).
+     * - The handler checks the fields of an update against `sourceFields` (`touchesNoField`).
      */
-    sourceMatch(partition) {
+    partitionsOfChange(change, watch) {
         const field = this.partitionField;
-        const noPreImage = { fullDocumentBeforeChange: null };
-        return {
-            $match: {
-                $or: [
-                    { [`fullDocument.${field}`]: partition },
-                    { [`fullDocumentBeforeChange.${field}`]: partition },
-                    { operationType: { $in: ["delete", "replace"] }, ...noPreImage },
-                    { operationType: "update", ...noPreImage, $or: [{ [`updateDescription.updatedFields.${field}`]: { $exists: true } }, { "updateDescription.removedFields": field }] },
-                ],
-            },
-        };
+        const type = change["operationType"];
+        const before = change["fullDocumentBeforeChange"];
+        if (before === null || before === undefined) {
+            const description = (change["updateDescription"] ?? {});
+            const movable = type === "delete" || type === "replace" || (type === "update" && (Object.hasOwn(description.updatedFields ?? {}, field) || (description.removedFields ?? []).includes(field)));
+            if (movable)
+                return watch.partitions.keys();
+        }
+        const after = change["fullDocument"];
+        const named = new Set();
+        for (const value of [after?.[field], before?.[field]].flat())
+            if (typeof value === "string")
+                named.add(value);
+        return named;
     }
     /**
      * True for an update that changes no field of `fields` and not the partition field. The keys of
@@ -654,57 +678,122 @@ export class ViewMaterializer {
     isCurrent(running) {
         return this.running.get(this.keyOf(running.partition, running.view._id)) === running;
     }
-    /** This method opens the stream of a source for a view. The promise resolves when the stream is open on the server. */
-    watchSource(running, source) {
+    /**
+     * This method adds a view to the stream of each of its sources, and opens a stream that does not
+     * exist. The promise resolves when each stream is open on the server, so a change after that
+     * reaches the view.
+     */
+    watchSources(running) {
+        const opened = [];
+        for (const source of new Set(running.view.collections)) {
+            let watch = this.sources.get(source);
+            if (!watch) {
+                watch = { source, stream: undefined, partitions: new Map(), opened: Promise.resolve() };
+                this.sources.set(source, watch);
+                watch.opened = this.openSource(watch);
+            }
+            let views = watch.partitions.get(running.partition);
+            if (!views)
+                watch.partitions.set(running.partition, (views = new Set()));
+            views.add(running);
+            running.watched.push(source);
+            opened.push(watch.opened);
+        }
+        return Promise.all(opened).then(() => undefined);
+    }
+    /** This method removes a view from its streams. The stream of a source with no view left closes. */
+    unwatch(running) {
+        for (const source of running.watched) {
+            const watch = this.sources.get(source);
+            const views = watch?.partitions.get(running.partition);
+            if (!watch || !views)
+                continue;
+            views.delete(running);
+            if (views.size === 0)
+                watch.partitions.delete(running.partition);
+            if (watch.partitions.size === 0)
+                this.dropWatch(watch);
+        }
+        running.watched = [];
+    }
+    /** This method closes the stream of a source and forgets it. */
+    dropWatch(watch) {
+        if (this.sources.get(watch.source) === watch)
+            this.sources.delete(watch.source);
+        const stream = watch.stream;
+        watch.stream = undefined;
+        stream?.close().catch(() => undefined);
+    }
+    /** This method opens the stream of a source. The promise resolves when the stream is open on the server. */
+    openSource(watch) {
         if (this.stopped || !this.db)
             return Promise.resolve();
+        const { source } = watch;
         let stream;
         try {
-            stream = this.db
-                .collection(source)
-                .watch([this.sourceMatch(running.partition)], { fullDocument: "updateLookup", fullDocumentBeforeChange: "whenAvailable" });
+            stream = this.db.collection(source).watch([], { fullDocument: "updateLookup", fullDocumentBeforeChange: "whenAvailable" });
         }
         catch (error) {
-            this.log("error", "Cannot watch a source of the view", { view: running.view._id, partition: running.partition, collection: source, error }, `cannot watch ${source} for the view ${running.view._id}: ${String(error?.message ?? error)}`);
-            this.later(() => {
-                if (this.isCurrent(running) && running.state === "active")
-                    void this.watchSource(running, source);
-            });
+            this.log("error", "Cannot watch a source", { collection: source, error }, `cannot watch ${source}: ${String(error?.message ?? error)}`);
+            this.afterStreamEnd(watch);
             return Promise.resolve();
         }
-        running.streams.push(stream);
+        watch.stream = stream;
         let again = false;
         const rewatch = (reason) => {
             if (again)
                 return;
             again = true;
             stream.close().catch(() => undefined);
-            const index = running.streams.indexOf(stream);
-            // A close of a teardown also ends the stream: open again only a stream of a current active view
-            if (index === -1 || !this.isCurrent(running) || running.state !== "active")
+            // A close of a teardown also ends the stream: open again only the stream of a source with views
+            if (watch.stream !== stream || this.sources.get(source) !== watch)
                 return;
-            running.streams.splice(index, 1);
-            this.log("warn", "The stream of a source ended: it opens again", { view: running.view._id, partition: running.partition, collection: source, reason, delayMs: this.rewatchMs }, `the stream of ${source} for the view ${running.view._id} ${reason}: it opens again in ${this.rewatchMs} ms`);
-            this.later(() => {
-                if (!this.isCurrent(running) || running.state !== "active")
-                    return;
-                // The changes while the stream was closed are lost: a pass catches up, after the new stream is open
-                void this.watchSource(running, source).then(() => this.schedule(running.partition, source));
-            });
+            watch.stream = undefined;
+            this.log("warn", "The stream of a source ended: it opens again", { collection: source, reason, delayMs: this.rewatchMs }, `the stream of ${source} ${reason}: it opens again in ${this.rewatchMs} ms`);
+            this.afterStreamEnd(watch);
         };
         stream.on("change", (change) => {
             if (change["operationType"] === "invalidate")
                 return rewatch("ended (invalidate)");
-            if (this.isOtherPartition(change, source, running.partition))
-                return;
-            if (this.touchesNoField(change, running.view.sourceFields?.[source]))
-                return;
-            this.schedule(running.partition, source);
+            if (watch.stream === stream)
+                this.route(change, watch);
         });
         stream.on("error", (error) => rewatch(`failed: ${error.message}`));
         stream.on("close", () => rewatch("closed"));
         return openOnServer(stream);
     }
+    /**
+     * This method opens the stream of a source again after `rewatch`. The changes while no stream was
+     * open are lost, so a pass of each partition of the source catches up, after the new stream is
+     * open. A view that joins before then is in that pass.
+     */
+    afterStreamEnd(watch) {
+        this.later(() => {
+            if (this.sources.get(watch.source) !== watch)
+                return;
+            void this.openSource(watch).then(() => {
+                if (this.sources.get(watch.source) !== watch)
+                    return;
+                for (const partition of watch.partitions.keys())
+                    this.schedule(partition, watch.source);
+            });
+        });
+    }
+    /** This method gives a change of a source to the views of its partitions. */
+    route(change, watch) {
+        const { source } = watch;
+        for (const partition of this.partitionsOfChange(change, watch)) {
+            const views = watch.partitions.get(partition);
+            if (!views || this.isOtherPartition(change, source, partition))
+                continue;
+            for (const running of views) {
+                if (this.touchesNoField(change, running.view.sourceFields?.[source]))
+                    continue;
+                this.schedule(partition, source);
+                break;
+            }
+        }
+    }
     /** This method runs a function after `rewatch`, unless the materializer stops. The timer does not keep the process alive. */
     later(work) {
         const cancel = this.clock.after(this.rewatchMs, () => {
@@ -723,12 +812,27 @@ export class ViewMaterializer {
         if (!dirty)
             this.dirty.set(partition, (dirty = new Set()));
         dirty.add(collection);
-        this.timers.get(partition)?.();
-        const cancel = this.clock.after(this.debounceMs, () => {
+        this.lastChange.set(partition, this.clock.now());
+        if (!this.timers.has(partition))
+            this.waitForPass(partition, this.debounceMs);
+    }
+    /**
+     * This method starts the timer of the pass of a partition. A change during the wait moves the end
+     * of the wait and makes no new timer: 1,000 deletes with no pre-image at 1,000 partitions are a
+     * million changes. A pass waits for its place in `passes` after the work of its partition before it.
+     */
+    waitForPass(partition, delay) {
+        const cancel = this.clock.after(delay, () => {
+            const left = this.lastChange.get(partition) + this.debounceMs - this.clock.now();
+            if (left > 0) {
+                this.waitForPass(partition, left);
+                return;
+            }
             this.timers.delete(partition);
+            this.lastChange.delete(partition);
             if (!this.intake)
                 return;
-            void this.serial(partition, () => this.pass(partition)).catch((error) => this.log("error", "The pass failed", { partition, error }, `the pass of ${partition} failed: ${String(error?.message ?? error)}`));
+            void this.serial(partition, () => this.passes.withPermit(() => this.pass(partition))).catch((error) => this.log("error", "The pass failed", { partition, error }, `the pass of ${partition} failed: ${String(error?.message ?? error)}`));
         });
         cancel.unref?.();
         this.timers.set(partition, cancel);
@@ -771,17 +875,15 @@ export class ViewMaterializer {
                     continue;
                 if (cyclic.has(view._id)) {
                     if (running.state === "active") {
-                        for (const stream of running.streams)
-                            stream.close().catch(() => undefined);
-                        running.streams = [];
+                        this.unwatch(running);
                         running.state = "cyclic";
                         await this.writeStatus(view, partition, { state: "cycle", at: Date.now() });
                     }
                 }
-                else if (running.state === "cyclic" || running.streams.length === 0) {
+                else if (running.state === "cyclic" || running.watched.length === 0) {
                     running.state = "active";
                     // The streams open before the run reads the sources: a change between the two reaches both (roadmap O63)
-                    await Promise.all(view.collections.map((source) => this.watchSource(running, source)));
+                    await this.watchSources(running);
                     // A change of the view document during the wait replaced this view: the rebalance of the new one runs it
                     if (!this.isCurrent(running))
                         continue;