@effectstream/sync 0.100.26 → 0.100.27

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@effectstream/sync",
-  "version": "0.100.26",
+  "version": "0.100.27",
   "license": "MIT",
   "homepage": "https://effectstream.github.io/docs/",
   "repository": {
@@ -13,10 +13,10 @@
   },
   "exports": "./src/mod.ts",
   "dependencies": {
-    "@effectstream/config": "0.100.26",
-    "@effectstream/db": "0.100.26",
-    "@effectstream/log": "0.100.26",
-    "@effectstream/utils": "0.100.26",
+    "@effectstream/config": "0.100.27",
+    "@effectstream/db": "0.100.27",
+    "@effectstream/log": "0.100.27",
+    "@effectstream/utils": "0.100.27",
     "@midnight-ntwrk/ledger-v8": "8.0.3",
     "@midnight-ntwrk/midnight-js-indexer-public-data-provider": "4.0.4",
     "@midnight-ntwrk/onchain-runtime": "npm:@midnight-ntwrk/onchain-runtime-v3@3.0.0",

package/src/sync-protocols/common/root.ts

@@ -34,6 +34,7 @@ export type ChainBlock = {
     >
     & { source: string }
   )[];
+  coalescedCount?: number;
 };
 
 export const chainPageRelation: PageRelation<ChainPage> = {

package/src/sync-protocols/orchestration/merge.ts

@@ -11,11 +11,98 @@ import type Deque from "denque";
 import type { ChainBlock } from "../common/root.ts";
 import { chainPageRelation } from "../common/root.ts";
 import type { CacheCleanup, OutputAndCleanup } from "../base/state.ts";
+import { ENV } from "@effectstream/utils/node-env";
+
+export const mergeCoalescingBoundaries = {
+  minScheduledBlockHeight: null as number | null,
+  minScheduledTimestampMs: null as number | null,
+  migrationBlockHeights: [] as number[],
+};
+
+function isCoalescableEmpty(block: ChainBlock): boolean {
+  if (block.primitives.length > 0) return false;
+
+  const bh = block.blockNumber;
+  if (mergeCoalescingBoundaries.migrationBlockHeights.includes(bh)) {
+    return false;
+  }
+
+  if (
+    mergeCoalescingBoundaries.minScheduledBlockHeight != null &&
+    bh >= mergeCoalescingBoundaries.minScheduledBlockHeight
+  ) {
+    return false;
+  }
+
+  if (
+    mergeCoalescingBoundaries.minScheduledTimestampMs != null &&
+    block.timestamp >= mergeCoalescingBoundaries.minScheduledTimestampMs
+  ) {
+    return false;
+  }
+
+  return true;
+}
+
+function mergeBlockInfo(
+  target: ChainBlock["blockInfo"],
+  source: ChainBlock["blockInfo"],
+): ChainBlock["blockInfo"] {
+  const map = new Map<string, ChainBlock["blockInfo"][number]>();
+  for (const item of target) {
+    map.set(item.protocol_name, item);
+  }
+  for (const item of source) {
+    map.set(item.protocol_name, item);
+  }
+  return Array.from(map.values());
+}
+
+function mergeResumePages(
+  target: ChainBlock["resumePages"],
+  source: ChainBlock["resumePages"],
+): ChainBlock["resumePages"] {
+  const map = new Map<string, ChainBlock["resumePages"][number]>();
+  for (const item of target) {
+    map.set(item.protocol_name, item);
+  }
+  for (const item of source) {
+    map.set(item.protocol_name, item);
+  }
+  return Array.from(map.values());
+}
 
 export function* startMerge(
   syncProtocols: AllSyncProtocols[],
   finalizedBlockStream: Channel<ChainBlock, void>,
 ): Operation<ChainBlock> {
+  let pendingEmptyRun: {
+    endpoint: ChainBlock;
+    count: number;
+  } | null = null;
+
+  const flushPendingRun = function* (): Operation<void> {
+    if (pendingEmptyRun == null) return;
+    const { endpoint } = pendingEmptyRun;
+    endpoint.coalescedCount = pendingEmptyRun.count;
+
+    log.remote(
+      ComponentNames.EFFECTSTREAM_SYNC,
+      "block-merge",
+      SeverityNumber.INFO,
+      (log) =>
+        log(
+          `producing coalesced empty block run of length ${pendingEmptyRun!.count} ending at block ${endpoint.blockNumber}`,
+        ),
+    );
+
+    yield* finalizedBlockStream.send(endpoint);
+    pendingEmptyRun = null;
+  };
+
+  const enabled = ENV.EFFECTSTREAM_COALESCE_EMPTY_BLOCKS;
+  const MAX_PRODUCER_COALESCE = 1000;
+
   while (true) {
     let newBlock: ChainBlock | undefined = undefined;
 
@@ -44,17 +131,66 @@ export function* startMerge(
       }
     }
     if (newBlock == null) continue;
-    log.remote(
-      ComponentNames.EFFECTSTREAM_SYNC,
-      "block-merge",
-      SeverityNumber.INFO,
-      (log) => log(`producing block ${newBlock?.blockNumber}`),
-    );
 
-    yield* finalizedBlockStream.send(newBlock);
+    if (enabled && isCoalescableEmpty(newBlock)) {
+      if (pendingEmptyRun == null) {
+        pendingEmptyRun = {
+          endpoint: newBlock,
+          count: 1,
+        };
+      } else {
+        pendingEmptyRun.endpoint.blockNumber = newBlock.blockNumber;
+        pendingEmptyRun.endpoint.timestamp = newBlock.timestamp;
+        pendingEmptyRun.endpoint.blockInfo = mergeBlockInfo(
+          pendingEmptyRun.endpoint.blockInfo,
+          newBlock.blockInfo,
+        );
+        pendingEmptyRun.endpoint.resumePages = mergeResumePages(
+          pendingEmptyRun.endpoint.resumePages,
+          newBlock.resumePages,
+        );
+        pendingEmptyRun.count++;
+      }
+
+      // Run cleanups eagerly so the consumed datum leaves the Deque now, instead
+      // of deferring to flush. This (a) keeps the main chain's Deque from growing
+      // by one entry per coalesced block across a long run, and (b) makes the
+      // tip check below meaningful: with the consumed datum removed,
+      // bufferedData.size() === 0 means the buffer is drained and we're at the
+      // tip. Safe because, during an empty run, every cleanup is a no-op — the
+      // main (NTP) datum's cleanup is `() => {}` and parallel chains merge
+      // nothing (cleanups.length === 0). Resume durability is unaffected: the
+      // runtime is the sole writer of the persisted resume marker, which only
+      // lands when the coalesced block commits downstream (CLAUDE.md #5).
+      for (const fn of cleanup) {
+        fn();
+      }
 
-    for (const fn of cleanup) {
-      fn();
+      // At the tip (steady state) the main chain produces one slot per
+      // blockTimeMS, so the Deque drains to empty after each one — flush
+      // immediately so trickle-in blocks aren't held back. While catching up the
+      // Deque stays non-empty and the run keeps accumulating until the cap.
+      const ntpState = syncProtocols[0];
+      const atTip = ntpState.bufferedData.size() === 0;
+      if (pendingEmptyRun.count >= MAX_PRODUCER_COALESCE || atTip) {
+        yield* flushPendingRun();
+      }
+    } else {
+      if (pendingEmptyRun != null) {
+        yield* flushPendingRun();
+      }
+      log.remote(
+        ComponentNames.EFFECTSTREAM_SYNC,
+        "block-merge",
+        SeverityNumber.INFO,
+        (log) => log(`producing block ${newBlock?.blockNumber}`),
+      );
+
+      yield* finalizedBlockStream.send(newBlock);
+
+      for (const fn of cleanup) {
+        fn();
+      }
     }
   }
 }