@effectstream/sync 0.100.18 → 0.100.19

package/src/sync-protocols/common/page-helpers.ts

@@ -5,6 +5,43 @@ import { ComponentNames, log, SeverityNumber } from "@effectstream/log";
 import type { PaginatedSyncProtocols } from "../types.ts";
 import type { PageTypeOf } from "../base/state.ts";
 
+// Fetch backpressure. See README "Backpressure (`maxBufferedPages`)".
+const MAX_BUFFER_MULTIPLE = 4;
+// Fallback chunk size for chains that fetch without an explicit stepSize
+// (e.g. utxorpc streams one block per pass).
+const DEFAULT_STEP_SIZE = 1000;
+
+/** Resolved fetch-backpressure cap for a chain. See README "Backpressure". */
+export function bufferCapFor(
+  syncProtocol: { maxBufferedPages?: number; stepSize?: number },
+): number {
+  const step = syncProtocol.stepSize ?? DEFAULT_STEP_SIZE;
+  return Math.max(
+    syncProtocol.maxBufferedPages ?? MAX_BUFFER_MULTIPLE * step,
+    step + 1,
+  );
+}
+
+/**
+ * Whether a chain should pause fetching because its in-memory buffer is at/over
+ * the cap. Each chain's `stateToInput` calls this first and returns `undefined`
+ * when true, so the fetch loop sleeps + retries while the merge drains. Also
+ * records backpressure metrics on `state` (for `/debug/metrics`).
+ * See README "Backpressure (`maxBufferedPages`)".
+ */
+export function bufferAtCap(
+  state: {
+    bufferedData: { size(): number };
+    recordBackpressure(atCap: boolean, cap: number): void;
+  },
+  syncProtocol: { maxBufferedPages?: number; stepSize?: number },
+): boolean {
+  const cap = bufferCapFor(syncProtocol);
+  const atCap = state.bufferedData.size() >= cap;
+  state.recordBackpressure(atCap, cap);
+  return atCap;
+}
+
 type ConfigSubset<Page> = {
   name: string;
   /**

package/src/sync-protocols/common/root.ts

@@ -5,6 +5,7 @@ import type {
   TimestampMs,
 } from "@effectstream/utils";
 import type { PageRelation } from "../base/page.ts";
+import type { LastPage } from "../base/state.ts";
 import type {
   ConfigSyncProtocolType,
   FlattenSyncProtocolIOFor,
@@ -19,6 +20,14 @@ export type ChainBlock = {
     block_number: BlockNumber;
     blockHash: BlockHash;
   }[];
+  /**
+   * Per-protocol block-accurate resume markers the runtime persists on commit.
+   * See `@effectstream/sync` CLAUDE.md, design idea #5 (restart/resume).
+   */
+  resumePages: {
+    protocol_name: string;
+    lastPage: LastPage<unknown, unknown>;
+  }[];
   primitives: (
     & FlattenSyncProtocolIOFor<
       ConfigSyncProtocolType

package/src/sync-protocols/evm/state.ts

@@ -7,7 +7,7 @@ import { bound, type EvmBlockNumber } from "@effectstream/utils";
 import { type LastPage, SyncState } from "../base/state.ts";
 import type { RootOutput, RootPage } from "../types.ts";
 import type { EvmFetcher } from "./fetcher.ts";
-import { genInputRange } from "../common/page-helpers.ts";
+import { bufferAtCap, genInputRange } from "../common/page-helpers.ts";
 import type { Input, Output, Page } from "./types.ts";
 import { toMsTimestamp } from "./types.ts";
 import type { ConfigNetworkType, SyncProtocolWithNetwork } from "@effectstream/config";
@@ -53,6 +53,7 @@ export class EvmSyncState extends SyncState<
 
   @bound
   override *stateToInput(): Operation<Input | undefined> {
+    if (bufferAtCap(this, this.config.syncProtocol)) return undefined;
     return yield* genInputRange(
       this as EvmSyncState,
       this.config.syncProtocol.startBlockHeight as EvmBlockNumber,
@@ -90,6 +91,16 @@ export class EvmSyncState extends SyncState<
     rootOutput.blockInfo.push(...blockInfo);
   }
 
+  @bound
+  override outputToLastPage(data: Output): LastPage<Page, RootPage> {
+    const block = Number(data.raw.number);
+    return {
+      own: block,
+      ownBlockNumber: block,
+      root: this.toRootPage(data),
+    };
+  }
+
   @bound
   override getNamespace(): string[] {
     return [this.config.network.name, this.config.syncProtocol.name];

package/src/sync-protocols/midnight/state.ts

@@ -10,6 +10,7 @@ import type { ConfigNetworkType, SyncProtocolWithNetwork } from "@effectstream/c
 import { getPage } from "@effectstream/db";
 import { MidnightClient } from "./MidnightClient.ts";
 import { applyDelay } from "../common/utils.ts";
+import { bufferAtCap } from "../common/page-helpers.ts";
 
 type LatestBlock = {
   block: {
@@ -72,6 +73,7 @@ export class MidnightSyncState extends SyncState<
 
   @bound
   override *stateToInput(): Operation<Input | undefined> {
+    if (bufferAtCap(this, this.config.syncProtocol)) return undefined;
     const latestBlockResult = yield* call(() => this.client.fetchLatestBlock());
     const latestHeight = latestBlockResult.block.height;
 
@@ -110,6 +112,15 @@ export class MidnightSyncState extends SyncState<
     rootOutput.primitives.push(...primitives);
   }
 
+  @bound
+  override outputToLastPage(data: Output): LastPage<Page, RootPage> {
+    return {
+      own: { height: data.raw.height, hash: data.raw.hash },
+      ownBlockNumber: data.raw.height,
+      root: this.toRootPage(data),
+    };
+  }
+
   @bound
   override getNamespace(): string[] {
     return [this.config.network.name, this.config.syncProtocol.name];

package/src/sync-protocols/mod.ts

@@ -19,6 +19,10 @@ export * from "./utxorpc/state.ts";
 export * from "./celestia/fetcher.ts";
 export * from "./celestia/state.ts";
 
+export * from "./test/fetcher.ts";
+export * from "./test/state.ts";
+export * from "./test/control.ts";
+
 export type * from "./base/fetcher.ts";
 export type * from "./base/state.ts";
 export * from "./orchestration/merge.ts";

package/src/sync-protocols/near/state.ts

@@ -10,6 +10,7 @@ import type { ConfigNetworkType, SyncProtocolWithNetwork } from "@effectstream/c
 import { getPage } from "@effectstream/db";
 import { NearClient } from "./NearClient.ts";
 import { applyDelay } from "../common/utils.ts";
+import { bufferAtCap } from "../common/page-helpers.ts";
 
 export class NearSyncState extends SyncState<
   Input,
@@ -60,6 +61,7 @@ export class NearSyncState extends SyncState<
 
   @bound
   override *stateToInput(): Operation<Input | undefined> {
+    if (bufferAtCap(this, this.config.syncProtocol)) return undefined;
     const latestHeight = yield* call(() =>
       this.client.getLatestBlockHeight()
     );
@@ -98,6 +100,15 @@ export class NearSyncState extends SyncState<
     rootOutput.primitives.push(...primitives);
   }
 
+  @bound
+  override outputToLastPage(data: Output): LastPage<Page, RootPage> {
+    return {
+      own: { height: data.height, hash: data.hash },
+      ownBlockNumber: data.height,
+      root: this.toRootPage(data),
+    };
+  }
+
   @bound
   override getNamespace(): string[] {
     return [this.config.network.name, this.config.syncProtocol.name];

package/src/sync-protocols/ntp/state.ts

@@ -7,7 +7,7 @@ import { bound, type NtpBlockNumber } from "@effectstream/utils";
 import { type LastPage, SyncState } from "../base/state.ts";
 import type { RootOutput, RootPage } from "../types.ts";
 import type { NtpFetcher } from "./fetcher.ts";
-import { genInputRange } from "../common/page-helpers.ts";
+import { bufferAtCap, genInputRange } from "../common/page-helpers.ts";
 import type { Input, Output, Page } from "./types.ts";
 import { toMsTimestamp } from "./types.ts";
 import type { ConfigNetworkType, SyncProtocolWithNetwork } from "@effectstream/config";
@@ -48,6 +48,7 @@ export class NtpSyncState extends SyncState<
       })),
       blockNumber: Number(data.raw.blockNumber),
       timestamp: this.toRootPage(data),
+      resumePages: [],
       primitives: [],
     };
   }
@@ -59,6 +60,7 @@ export class NtpSyncState extends SyncState<
 
   @bound
   override *stateToInput(): Operation<Input | undefined> {
+    if (bufferAtCap(this, this.config.syncProtocol)) return undefined;
     return yield* genInputRange(
       this as NtpSyncState,
       1, // NTP Pages start from 1, and the fetcher offsets using the start page (date)
@@ -86,6 +88,16 @@ export class NtpSyncState extends SyncState<
     throw new Error("Only parallel chains merge into root");
   }
 
+  @bound
+  override outputToLastPage(data: Output): LastPage<Page, RootPage> {
+    const block = Number(data.raw.blockNumber);
+    return {
+      own: block,
+      ownBlockNumber: block,
+      root: this.toRootPage(data),
+    };
+  }
+
   @bound
   override getNamespace(): string[] {
     return [this.config.network.name, this.config.syncProtocol.name];

package/src/sync-protocols/orchestration/merge.ts

@@ -124,6 +124,11 @@ export function* mergeIntoRoot<SyncProtocol extends AllSyncProtocols>(
     }
     const output = iState.bufferedData.peekAt(0)!;
     const newRoot = iState.toRootOutput(output.output);
+    // Resume marker: the single datum this root block is built from (CLAUDE.md #5).
+    newRoot.resumePages.push({
+      protocol_name: state.name,
+      lastPage: iState.outputToLastPage(output.output),
+    });
     return {
       updateCache: () => {
         output.cleanup();
@@ -168,6 +173,17 @@ export function* mergeIntoRoot<SyncProtocol extends AllSyncProtocols>(
     iState.mergeDatum,
   );
 
+  // Resume marker: the last (highest) datum merged this iteration, still in the
+  // Deque at `cleanups.length - 1` (cleanup runs after send). Nothing consumed →
+  // no marker. See CLAUDE.md #5.
+  if (cleanups.length > 0) {
+    const lastConsumed = iState.bufferedData.peekAt(cleanups.length - 1)!;
+    rootInfo.value.resumePages.push({
+      protocol_name: state.name,
+      lastPage: iState.outputToLastPage(lastConsumed.output),
+    });
+  }
+
   return {
     updateCache: () => {
       state.bufferedData.remove(0, cleanups.length);

package/src/sync-protocols/test/control.ts

@@ -0,0 +1,27 @@
+/**
+ * Test-only control plane for the synthetic `test` sync protocol.
+ *
+ * The chain tip ("latest block") is held in a process-wide registry keyed by
+ * sync-protocol name, rather than in config, so a single test can advance it
+ * deterministically between simulated restarts. (Immutable config fields like
+ * `startTime` cannot change across a restart, so they can't drive this.)
+ *
+ * When no tip is registered for a protocol, the fetcher falls back to a
+ * wall-clock tip derived from `startTime`/`blockTimeMS` (see fetcher.ts).
+ */
+const tips = new Map<string, number>();
+
+export const TestChainControl = {
+  /** Set the current chain tip (latest available block) for a protocol. */
+  setTip(protocolName: string, tip: number): void {
+    tips.set(protocolName, tip);
+  },
+  /** Get the manually-controlled tip, or undefined to fall back to wall-clock. */
+  getTip(protocolName: string): number | undefined {
+    return tips.get(protocolName);
+  },
+  /** Reset all tips (call between tests). */
+  clear(): void {
+    tips.clear();
+  },
+};

package/src/sync-protocols/test/fetcher.ts

@@ -0,0 +1,175 @@
+import {
+  BaseDataFetcher,
+  type DataFetched,
+  type PaginatedFetcher,
+} from "../base/fetcher.ts";
+import type { RootOutput, RootPage } from "../types.ts";
+import type { Operation } from "effection";
+import { bound } from "@effectstream/utils";
+import { blockNumberRelation } from "../common/utils.ts";
+import type { Input, Output, Page, PrimitiveType } from "./types.ts";
+import {
+  fetchNewestPage,
+  type PageRange,
+  type PageRequest,
+} from "../base/page.ts";
+import type { PrimitiveFetcher } from "../base/primitive.ts";
+import type { OutputAndCleanup, RootConversion } from "../base/state.ts";
+import {
+  ConfigSyncProtocolType,
+  type ConfigNetworkType,
+  type PrimitiveEntry,
+  type SyncProtocolWithNetwork,
+} from "@effectstream/config";
+import { TestChainControl } from "./control.ts";
+
+type TestConfig = Extract<
+  SyncProtocolWithNetwork,
+  { networkType: ConfigNetworkType.TEST }
+>;
+
+type TestEvent = { atBlock: number; payload: Record<string, unknown> };
+
+/**
+ * Synthetic chain fetcher. Blocks are computed arithmetically (no RPC):
+ *   timestamp(page) = startTime + blockTimeMS * page
+ * The chain tip comes from {@link TestChainControl} when set (deterministic
+ * tests), otherwise from the wall clock. Configured `events` are emitted as
+ * primitives when their `atBlock` is reached.
+ */
+export class TestFetcher
+  extends BaseDataFetcher<Input, Output, RootOutput, Page, RootPage>
+  implements
+    PrimitiveFetcher<
+      Input,
+      Page,
+      { blockNumber: number; timestamp: bigint; hash: string },
+      PrimitiveType,
+      ConfigSyncProtocolType.TEST_PARALLEL
+    >,
+    PaginatedFetcher<Page> {
+  constructor(
+    readonly config: TestConfig,
+  ) {
+    super(config.syncProtocol.name);
+  }
+
+  private blockOf(page: Page): Output["raw"] {
+    const timestamp = BigInt(this.config.network.startTime) +
+      BigInt(this.config.network.blockTimeMS) * BigInt(page);
+    return {
+      blockNumber: page,
+      timestamp,
+      hash: `0x${timestamp.toString(16)}`,
+    };
+  }
+
+  @bound
+  override *readData(
+    data: Input,
+    rootConversion: RootConversion<Output, RootOutput, RootPage>,
+  ): Operation<DataFetched<Output, Page, RootPage>> {
+    // Events are declared on the (parallel) sync protocol config. The primitive
+    // instance name to tag them with is the first declared primitive's id.
+    const events: TestEvent[] =
+      (this.config.syncProtocol as { events?: TestEvent[] }).events ?? [];
+    const primitiveId = this.config.primitives[0]?.id;
+
+    const output: OutputAndCleanup<Output>[] = [];
+    for (let page = data.from; page <= data.to; page++) {
+      const raw = this.blockOf(page);
+      const primitives: PrimitiveType[] = [];
+      if (primitiveId != null) {
+        for (const ev of events) {
+          if (ev.atBlock === page) {
+            primitives.push({
+              syncProtocol: {
+                name: ConfigSyncProtocolType.TEST_PARALLEL,
+                blockNumber: page,
+                transactionHash: `0x${page.toString(16)}`,
+                contractAddress: "0x0",
+              },
+              primitive: primitiveId,
+              output: { payloadType: "test-event", payload: ev.payload },
+            });
+          }
+        }
+      }
+      output.push({
+        output: { raw, primitives, blockHashes: [raw.hash] },
+        cleanup: () => {},
+      });
+    }
+
+    return {
+      output,
+      lastPage: {
+        own: Number(data.to),
+        ownBlockNumber: Number(data.to),
+        root: rootConversion.toRootPage({
+          raw: this.blockOf(Number(data.to)),
+          primitives: [],
+          blockHashes: [],
+        }),
+      },
+    };
+  }
+
+  @bound
+  groupByPage(
+    _primitives: PrimitiveType[],
+  ): Record<string, PrimitiveType[]> {
+    // Primitives are attached inline in readData; not used.
+    return {};
+  }
+
+  @bound
+  *readPrimitives(
+    _data: Input,
+    _pageRequest: PageRequest<Page, { timestamp: bigint; hash: string }>,
+    _primitiveEntries: Extract<
+      PrimitiveEntry,
+      { syncProtocol: ConfigSyncProtocolType.TEST_PARALLEL }
+    >[],
+  ): Operation<PrimitiveType[]> {
+    return [];
+  }
+
+  @bound
+  *getLatestPage(
+    knownLastPage: undefined | Page,
+  ): Operation<Page> {
+    const self = this;
+    return yield* fetchNewestPage<Page>(
+      blockNumberRelation,
+      function* (): Operation<Page> {
+        const manual = TestChainControl.getTip(self.config.syncProtocol.name);
+        if (manual != null) return manual;
+        const diff = Date.now() - self.config.network.startTime;
+        return Math.floor(diff / self.config.network.blockTimeMS);
+      },
+      knownLastPage,
+      this.config.syncProtocol.pollingInterval,
+    );
+  }
+
+  @bound
+  previousInterval(nextIntervalStart: Page): PageRange<Page> {
+    return this.intervalFromStart(
+      nextIntervalStart - this.config.syncProtocol.stepSize - 1,
+    );
+  }
+
+  @bound
+  nextInterval(prevIntervalEnd: Page): PageRange<Page> {
+    return this.intervalFromStart(prevIntervalEnd + 1);
+  }
+
+  @bound
+  intervalFromStart(start: Page): PageRange<Page> {
+    return {
+      from: start,
+      to: start + this.config.syncProtocol.stepSize,
+    };
+  }
+}

package/src/sync-protocols/test/state.ts

@@ -0,0 +1,154 @@
+import { applyDelay, blockNumberRelation } from "../common/utils.ts";
+import type { Operation } from "effection";
+import { call } from "effection";
+import { getPage } from "@effectstream/db";
+import type { PoolClient } from "pg";
+import { bound, type BlockNumber } from "@effectstream/utils";
+import { type LastPage, SyncState } from "../base/state.ts";
+import type { RootOutput, RootPage } from "../types.ts";
+import type { TestFetcher } from "./fetcher.ts";
+import { bufferAtCap, genInputRange } from "../common/page-helpers.ts";
+import type { Input, Output, Page } from "./types.ts";
+import { toMsTimestamp } from "./types.ts";
+import type {
+  ConfigNetworkType,
+  SyncProtocolWithNetwork,
+} from "@effectstream/config";
+
+type TestConfig = Extract<
+  SyncProtocolWithNetwork,
+  { networkType: ConfigNetworkType.TEST }
+>;
+
+/**
+ * Synthetic test chain state. Implements BOTH roles:
+ * - as a TEST_MAIN clock, `toRootOutput` produces root blocks;
+ * - as a TEST_PARALLEL source, `mergeDatum` folds its primitives into the root.
+ * The merge only calls the method appropriate to the chain's position (main at
+ * index 0), so implementing both is safe.
+ */
+export class TestSyncState extends SyncState<
+  Input,
+  Output,
+  Page,
+  RootOutput,
+  RootPage,
+  TestFetcher
+> {
+  constructor(
+    lastPage: LastPage<Page, RootPage>,
+    readonly config: TestConfig,
+    fetcher: TestFetcher,
+    dbConn: PoolClient,
+  ) {
+    super(
+      config.syncProtocol.name,
+      lastPage,
+      fetcher,
+      blockNumberRelation,
+      dbConn,
+    );
+  }
+
+  @bound
+  override toRootOutput(data: Output): RootOutput {
+    return {
+      blockInfo: data.blockHashes.map((h) => ({
+        protocol_name: this.config.syncProtocol.name,
+        block_number: Number(data.raw.blockNumber),
+        blockHash: h,
+      })),
+      blockNumber: Number(data.raw.blockNumber),
+      timestamp: this.toRootPage(data),
+      resumePages: [],
+      primitives: data.primitives.map((p) => ({
+        ...p,
+        source: this.config.syncProtocol.name,
+      })),
+    };
+  }
+
+  @bound
+  override toRootPage(data: Output): RootPage {
+    return applyDelay(
+      toMsTimestamp(data.raw.timestamp),
+      (this.config.syncProtocol as { delayMs?: number }).delayMs ?? 0,
+    );
+  }
+
+  @bound
+  override *stateToInput(): Operation<Input | undefined> {
+    if (bufferAtCap(this, this.config.syncProtocol)) return undefined;
+    return yield* genInputRange(
+      this as TestSyncState,
+      this.config.syncProtocol.startBlockHeight as BlockNumber,
+      {
+        name: this.config.syncProtocol.name,
+        startPage: this.config.syncProtocol.startBlockHeight as BlockNumber,
+      },
+      this.getNamespace(),
+    );
+  }
+
+  @bound
+  override toPage(
+    _input: Input,
+    _data: Output[],
+  ): Page {
+    return Number(_input.to);
+  }
+
+  @bound
+  override mergeDatum(
+    ourOutput: Output,
+    rootOutput: RootOutput,
+  ): void {
+    const primitives = ourOutput.primitives.map((p) => ({
+      ...p,
+      source: this.config.syncProtocol.name,
+    }));
+    const blockInfo = ourOutput.blockHashes.map((h) => ({
+      protocol_name: this.config.syncProtocol.name,
+      block_number: Number(ourOutput.raw.blockNumber),
+      blockHash: h,
+    }));
+    rootOutput.primitives.push(...primitives);
+    rootOutput.blockInfo.push(...blockInfo);
+  }
+
+  @bound
+  override outputToLastPage(data: Output): LastPage<Page, RootPage> {
+    const block = Number(data.raw.blockNumber);
+    return {
+      own: block,
+      ownBlockNumber: block as BlockNumber,
+      root: this.toRootPage(data),
+    };
+  }
+
+  @bound
+  override getNamespace(): string[] {
+    return [this.config.network.name, this.config.syncProtocol.name];
+  }
+
+  static *restoreState(
+    dbConn: PoolClient,
+    config: TestConfig,
+    fetcher: TestFetcher,
+  ): Operation<TestSyncState> {
+    const [result] = yield* call(async () =>
+      await getPage.run({
+        protocol_name: config.syncProtocol.name,
+      }, dbConn)
+    );
+    const page: any = result
+      ? result.page as unknown as LastPage<Page, RootPage>
+      : undefined;
+    return new TestSyncState(
+      page,
+      config,
+      fetcher,
+      dbConn,
+    );
+  }
+}

package/src/sync-protocols/test/types.ts

@@ -0,0 +1,36 @@
+import type { BlockNumber, TimestampMs } from "@effectstream/utils";
+import type { PageSyncRange } from "../common/page-helpers.ts";
+import type {
+  ConfigNetworkType,
+  ConfigSyncProtocolType,
+  FlattenSyncProtocolIOFor,
+  SyncProtocolWithNetwork,
+} from "@effectstream/config";
+
+export type Page = BlockNumber;
+
+export type ConfigType = Extract<
+  SyncProtocolWithNetwork,
+  { networkType: ConfigNetworkType.TEST }
+>;
+
+/** Primitives a TEST_PARALLEL chain emits (one per configured event in range). */
+export type PrimitiveType = FlattenSyncProtocolIOFor<
+  ConfigSyncProtocolType.TEST_PARALLEL
+>;
+
+export type Input = PageSyncRange<Page>;
+
+export type Output = {
+  raw: {
+    timestamp: bigint;
+    hash: string;
+    blockNumber: number;
+  };
+  primitives: PrimitiveType[];
+  blockHashes: string[];
+};
+
+export function toMsTimestamp(timestamp: bigint): TimestampMs {
+  return Number(timestamp);
+}

package/src/sync-protocols/types.ts

@@ -10,6 +10,7 @@ import type { AvailSyncState } from "./avail/state.ts";
 import type { BitcoinSyncState } from "./bitcoin/state.ts";
 import type { CelestiaSyncState } from "./celestia/state.ts";
 import type { NearSyncState } from "./near/state.ts";
+import type { TestSyncState } from "./test/state.ts";
 
 // TODO: move folders
 export type RootOutput = ChainBlock;
@@ -24,7 +25,8 @@ export type AllSyncProtocols =
   | AvailSyncState
   | BitcoinSyncState
   | CelestiaSyncState
-  | NearSyncState;
+  | NearSyncState
+  | TestSyncState;
 export type ISyncProtocol = UnionToIntersection<AllSyncProtocols>;
 
 type toPaginated<T> = T extends { fetcher: PaginatedFetcher<infer Page> } ? T