@effectstream/sync 0.101.0 → 0.102.0

package/src/sync-protocols/solana/fetcher.ts

@@ -0,0 +1,358 @@
+import {
+  ConfigSyncProtocolType,
+  type PrimitiveEntry,
+} from "@effectstream/config";
+import { BaseDataFetcher } from "../base/fetcher.ts";
+import type { DataFetched } from "../base/fetcher.ts";
+import type {
+  LastPage,
+  OutputAndCleanup,
+  RootConversion,
+} from "../base/state.ts";
+import type { RootOutput, RootPage } from "../types.ts";
+import type {
+  ConfigType,
+  Input,
+  Output,
+  Page,
+  PrimitiveType,
+} from "./types.ts";
+import { resolveAccountKeys, SolanaClient } from "./SolanaClient.ts";
+import { requestTimeoutOf } from "../common/http.ts";
+import { extractProgramLogs } from "./program-logs.ts";
+import { call, sleep, type Operation } from "effection";
+import { bound } from "@effectstream/utils";
+
+/**
+ * Attempts per slot before `readData` reports partial progress and lets the
+ * fetch loop retry the range. Small on purpose — the outer loop is the real
+ * retry mechanism; see `getBlockWithRetry`.
+ */
+const BLOCK_FETCH_ATTEMPTS = 3;
+/** Linear backoff between those attempts (250ms, then 500ms). */
+const BLOCK_FETCH_RETRY_DELAY_MS = 250;
+
+export class SolanaFetcher extends BaseDataFetcher<
+  Input,
+  Output,
+  RootOutput,
+  Page,
+  RootPage
+> {
+  readonly client: SolanaClient;
+
+  /**
+   * Last non-null `blockTime` seen, used to fill in a block whose timestamp the
+   * RPC couldn't compute. Carried across `readData` calls so only a null on the
+   * very first block ever fetched has no fallback.
+   */
+  private lastKnownBlockTime: number | undefined;
+
+  constructor(
+    readonly config: ConfigType,
+  ) {
+    super(config.syncProtocol.name);
+    this.client = new SolanaClient(
+      config.network.rpcUrl,
+      requestTimeoutOf(config.syncProtocol),
+    );
+  }
+
+  @bound
+  override *readData(
+    data: Input,
+    rootConversion: RootConversion<Output, RootOutput, RootPage>,
+    lastPage: LastPage<Page, RootPage> | undefined,
+  ): Operation<DataFetched<Output, Page, RootPage>> {
+    const outputs: OutputAndCleanup<Output>[] = [];
+
+    console.log(
+      `[Solana] Fetching slots from ${data.from} to ${data.to}.${
+        data.isPresync ? " [presync]" : ""
+      }`,
+    );
+
+    // Highest slot we have positively resolved — either read a block for, or had
+    // the RPC confirm was skipped. The page may never advance past this, or we
+    // would silently step over a slot we never actually read.
+    let scannedThrough: number | undefined;
+    let fetchFailure: { slot: number; error: unknown } | undefined;
+
+    for (let slot = Number(data.from); slot <= Number(data.to); slot++) {
+      let block;
+      try {
+        block = yield* this.getBlockWithRetry(slot);
+      } catch (error) {
+        // Keep the blocks already gathered rather than discarding the chunk:
+        // everything below `slot` is resolved, so we report progress up to
+        // there and the next poll resumes exactly at the failed slot.
+        fetchFailure = { slot, error };
+        break;
+      }
+      scannedThrough = slot;
+
+      // Skipped slots (no block produced) are skipped gracefully
+      if (!block) {
+        continue;
+      }
+
+      // `blockTime` is the merge key for this chain, so it can't be null.
+      // Mapping null to 0 (the previous behaviour) sorted the block to the unix
+      // epoch and broke the monotonicity the merge relies on. Solana guarantees
+      // blockTime is non-decreasing, so reusing the last known value keeps the
+      // block in the stream, in order, in the same bucket as its predecessor.
+      const blockTime = block.blockTime ?? this.lastKnownBlockTime;
+      if (blockTime == null) {
+        console.warn(
+          `[Solana] slot ${slot} has no blockTime and no earlier block to derive one from — skipping.`,
+        );
+        continue;
+      }
+      this.lastKnownBlockTime = blockTime;
+
+      const primitives = yield* this.readPrimitives(
+        slot,
+        block,
+        this.config.primitives,
+      );
+
+      outputs.push({
+        output: {
+          slot,
+          blockhash: block.blockhash,
+          blockTime,
+          blockHeight: block.blockHeight,
+          parentSlot: block.parentSlot,
+          // `meta` is null for transactions the RPC couldn't decode; keep the
+          // entry (it still occupies a tx index) but don't dereference it.
+          transactions: (block.transactions ?? []).map((tx) => ({
+            err: tx.meta?.err ?? null,
+            logMessages: tx.meta?.logMessages ?? null,
+            preBalances: tx.meta?.preBalances ?? [],
+            postBalances: tx.meta?.postBalances ?? [],
+          })),
+          primitives,
+        },
+        cleanup: () => {},
+      });
+    }
+
+    if (fetchFailure != null) {
+      console.warn(
+        `[Solana] slot ${fetchFailure.slot} could not be fetched after ` +
+          `${BLOCK_FETCH_ATTEMPTS} attempts: ${String(fetchFailure.error)}. ` +
+          `Keeping ${outputs.length} block(s) up to slot ${scannedThrough ?? "-"}; ` +
+          `the next poll resumes at ${fetchFailure.slot}.`,
+      );
+      if (scannedThrough == null) {
+        // Nothing at all was resolved, so there is no progress to report and
+        // nothing to advance to. Throw: the fetch loop counts it in
+        // `consecutiveErrors`, leaves `lastSuccessfulFetchMs` alone and retries
+        // this exact range after `pollingInterval`. That is what keeps a
+        // genuinely unreachable RPC visible to /health instead of looking idle.
+        throw fetchFailure.error;
+      }
+    }
+
+    if (outputs.length === 0) {
+      if (!lastPage) {
+        throw new Error(
+          `[Solana] Could not fetch any blocks from ${data.from} to ${data.to} and no previous page was found.`,
+        );
+      }
+      // Advance the page even though nothing was found. `stateToInput` derives
+      // `from` from `lastPage.own`, so a page that never moves re-requests the
+      // same window forever — a range of entirely skipped slots (a Solana
+      // outage, or any leaderless stretch) would stall the fetcher permanently
+      // with no recovery once blocks resume. This is the "Page != Data"
+      // invariant in the package CLAUDE.md: pages track scan progress, not
+      // content.
+      //
+      // The root timestamp is carried over unchanged: no block means no new
+      // time to report, and the merge requires root to be non-decreasing.
+      //
+      // NOTE: near/fetcher.ts:94 has the identical stall — it's the pattern this
+      // was copied from. Not fixed here to keep this PR scoped to Solana.
+      //
+      // `scannedThrough`, not `data.to`: if a fetch failed part-way we must not
+      // advance over the slot we never read.
+      return {
+        output: [],
+        lastPage: {
+          ...lastPage,
+          own: scannedThrough as Page,
+          ownBlockNumber: scannedThrough as Page,
+        },
+      };
+    }
+
+    const lastOutput = outputs[outputs.length - 1].output;
+    // Page to the highest RESOLVED slot rather than the last one that carried
+    // data: trailing skipped slots were positively confirmed empty, so
+    // re-scanning them next poll is wasted work. On a clean full scan this is
+    // `data.to`, matching bitcoin/evm. The root timestamp still comes from the
+    // newest block we actually have, which is what the merge gates on.
+    const page = (scannedThrough ?? lastOutput.slot) as Page;
+    return {
+      output: outputs,
+      lastPage: {
+        ownBlockNumber: page,
+        own: page,
+        root: rootConversion.toRootPage(lastOutput),
+      },
+    };
+  }
+
+  /**
+   * `getBlock` with a few immediate retries, so a transient RPC blip does not
+   * cost the whole chunk.
+   *
+   * Deliberately BOUNDED. Retrying here forever would look like "we always get
+   * the block eventually", but it recreates the exact failure the request
+   * timeout was added to kill (sync CLAUDE.md finding #4): `readData` would
+   * never return, so the fetch loop never reaches its `catch`,
+   * `consecutiveErrors` stays 0, `lastSuccessfulFetchMs` freezes, and /health
+   * reports a happy node that has silently stopped producing blocks.
+   *
+   * Unbounded retry still happens — one level up, where it is observable. The
+   * fetch loop re-requests the same range every `pollingInterval` indefinitely,
+   * and page ranges are idempotent, so no slot is ever skipped. This helper only
+   * decides how much we retry *before* handing that decision back.
+   */
+  @bound
+  *getBlockWithRetry(slot: number) {
+    let lastError: unknown;
+    for (let attempt = 1; attempt <= BLOCK_FETCH_ATTEMPTS; attempt++) {
+      try {
+        return yield* call(() => this.client.getBlock(slot));
+      } catch (error) {
+        lastError = error;
+        if (attempt < BLOCK_FETCH_ATTEMPTS) {
+          yield* sleep(BLOCK_FETCH_RETRY_DELAY_MS * attempt);
+        }
+      }
+    }
+    throw lastError;
+  }
+
+  @bound
+  *readPrimitives(
+    slot: number,
+    block: {
+      transactions: {
+        transaction: {
+          message: { accountKeys: string[] };
+          signatures: string[];
+        };
+        meta: {
+          err: unknown | null;
+          logMessages: string[] | null;
+          postBalances: number[];
+          loadedAddresses?: {
+            writable: string[];
+            readonly: string[];
+          } | null;
+        } | null;
+      }[];
+    },
+    primitiveEntries: Extract<
+      PrimitiveEntry,
+      { syncProtocol: ConfigSyncProtocolType.SOLANA_RPC_PARALLEL }
+    >[],
+  ): Operation<PrimitiveType[]> {
+    if (primitiveEntries.length === 0) return [];
+
+    const allPrimitives: PrimitiveType[] = [];
+
+    for (
+      let txIndex = 0;
+      txIndex < block.transactions.length;
+      txIndex++
+    ) {
+      const tx = block.transactions[txIndex];
+      // A reverted transaction has no on-chain effect: its logs describe work
+      // that was rolled back and its postBalances are the pre-state. Emitting
+      // primitives for it would drive state transitions off events that never
+      // happened. `meta == null` means the RPC couldn't decode the tx — equally
+      // unusable, so skip both.
+      if (!tx.meta || tx.meta.err) continue;
+      // Balances are indexed over static keys PLUS lookup-table addresses, so a
+      // watched address pulled in via an ALT is only findable in the resolved
+      // list. Legacy transactions resolve to the static keys unchanged.
+      const accountKeys = resolveAccountKeys(
+        tx.transaction.message.accountKeys,
+        tx.meta.loadedAddresses,
+      );
+      const logs = tx.meta.logMessages ?? [];
+      const postBalances = tx.meta.postBalances ?? [];
+      const txHash = tx.transaction.signatures[0] ?? "";
+
+      for (const entry of primitiveEntries) {
+        const prim = entry.primitive;
+
+        // ── SOLANA:AccountBalance — watch an address's lamport balance ──
+        if (prim.address) {
+          const idx = accountKeys.indexOf(prim.address);
+          if (idx === -1) continue;
+          allPrimitives.push({
+            syncProtocol: {
+              name: entry.syncProtocol,
+              blockNumber: slot,
+              transactionHash: txHash,
+              contractAddress: prim.address,
+              logIndex: txIndex,
+            },
+            primitive: prim.name,
+            output: {
+              payloadType: "solana:balance",
+              payload: {
+                address: prim.address,
+                lamports: postBalances[idx] ?? 0,
+                slot,
+              },
+            },
+          });
+          continue;
+        }
+
+        // ── SOLANA:ProgramLog — logs the watched programId actually emitted ──
+        if (prim.programId) {
+          // Source of truth is the log stream's invoke/success framing, NOT
+          // accountKeys: naming a program as an account doesn't invoke it, and
+          // a program reached through a lookup table isn't in accountKeys at
+          // all. See program-logs.ts.
+          const programLogs = extractProgramLogs(logs, prim.programId);
+          if (programLogs == null) continue;
+          // Filter by eventType if specified — against this program's own lines
+          // only, so another program can't trigger it by echoing the string.
+          if (prim.eventType) {
+            const hasMatchingLog = programLogs.some((log) =>
+              log.includes(prim.eventType!)
+            );
+            if (!hasMatchingLog) continue;
+          }
+          allPrimitives.push({
+            syncProtocol: {
+              name: entry.syncProtocol,
+              blockNumber: slot,
+              transactionHash: txHash,
+              contractAddress: prim.programId,
+              logIndex: txIndex,
+            },
+            primitive: prim.name,
+            output: {
+              payloadType: "solana:transaction",
+              payload: {
+                programId: prim.programId,
+                slot,
+                logMessages: programLogs,
+              },
+            },
+          });
+        }
+      }
+    }
+
+    return allPrimitives;
+  }
+}

package/src/sync-protocols/solana/program-logs.test.ts

@@ -0,0 +1,113 @@
+import { expect, test } from "bun:test";
+import { extractProgramLogs } from "./program-logs.ts";
+
+const WATCHED = "Watch1111111111111111111111111111111111111";
+const OTHER = "0ther1111111111111111111111111111111111111";
+const SYSTEM = "11111111111111111111111111111111";
+
+test("returns null when the program never appears at all", () => {
+  expect(extractProgramLogs([
+    `Program ${OTHER} invoke [1]`,
+    "Program log: unrelated",
+    `Program ${OTHER} success`,
+  ], WATCHED)).toBeNull();
+});
+
+test("SPOOF: a program merely named in another program's log line is not invoked", () => {
+  // The attack the old `accountKeys.includes()` + substring match allowed: the
+  // watched program is referenced as a bare account key (so it shows up in
+  // accountKeys) while someone else's program emits the expected text.
+  const logs = [
+    `Program ${OTHER} invoke [1]`,
+    `Program log: ${WATCHED} EFFECTSTREAM_COUNTER|attacker|9999|1`,
+    `Program ${OTHER} success`,
+  ];
+  expect(extractProgramLogs(logs, WATCHED)).toBeNull();
+});
+
+test("collects only the watched program's own lines at top level", () => {
+  expect(extractProgramLogs([
+    `Program ${WATCHED} invoke [1]`,
+    "Program log: mine",
+    `Program ${WATCHED} consumed 1234 of 200000 compute units`,
+    `Program ${WATCHED} success`,
+  ], WATCHED)).toEqual(["Program log: mine"]);
+});
+
+test("CPI: an inner program's lines are not attributed to the outer one", () => {
+  const logs = [
+    `Program ${WATCHED} invoke [1]`,
+    "Program log: outer before",
+    `Program ${OTHER} invoke [2]`,
+    "Program log: inner",
+    `Program ${OTHER} consumed 10 of 100 compute units`,
+    `Program ${OTHER} success`,
+    "Program log: outer after",
+    `Program ${WATCHED} consumed 50 of 200 compute units`,
+    `Program ${WATCHED} success`,
+  ];
+  expect(extractProgramLogs(logs, WATCHED))
+    .toEqual(["Program log: outer before", "Program log: outer after"]);
+  expect(extractProgramLogs(logs, OTHER)).toEqual(["Program log: inner"]);
+});
+
+test("CPI: a watched program called by another still reports its own lines", () => {
+  expect(extractProgramLogs([
+    `Program ${OTHER} invoke [1]`,
+    "Program log: caller",
+    `Program ${WATCHED} invoke [2]`,
+    "Program log: callee",
+    `Program ${WATCHED} success`,
+    `Program ${OTHER} success`,
+  ], WATCHED)).toEqual(["Program log: callee"]);
+});
+
+test("invoked-but-silent yields [] — distinct from null", () => {
+  // The System Program logs only its framing; the primitive should still fire.
+  const result = extractProgramLogs([
+    `Program ${SYSTEM} invoke [1]`,
+    `Program ${SYSTEM} success`,
+  ], SYSTEM);
+  expect(result).not.toBeNull();
+  expect(result).toEqual([]);
+});
+
+test("multiple invocations of the same program accumulate", () => {
+  expect(extractProgramLogs([
+    `Program ${WATCHED} invoke [1]`,
+    "Program log: first",
+    `Program ${WATCHED} success`,
+    `Program ${WATCHED} invoke [1]`,
+    "Program log: second",
+    `Program ${WATCHED} success`,
+  ], WATCHED)).toEqual(["Program log: first", "Program log: second"]);
+});
+
+test("a failed inner frame still pops", () => {
+  expect(extractProgramLogs([
+    `Program ${WATCHED} invoke [1]`,
+    `Program ${OTHER} invoke [2]`,
+    "Program log: inner",
+    `Program ${OTHER} failed: custom program error: 0x1`,
+    "Program log: outer recovers",
+    `Program ${WATCHED} success`,
+  ], WATCHED)).toEqual(["Program log: outer recovers"]);
+});
+
+test("truncated logs leaving an unbalanced stack don't throw", () => {
+  expect(() => extractProgramLogs([
+    `Program ${WATCHED} success`,
+    `Program ${WATCHED} invoke [1]`,
+    "Program log: mine",
+    "Log truncated",
+  ], WATCHED)).not.toThrow();
+});
+
+test("Program data and return lines are kept as content", () => {
+  expect(extractProgramLogs([
+    `Program ${WATCHED} invoke [1]`,
+    "Program data: SGVsbG8=",
+    `Program return: ${WATCHED} AQID`,
+    `Program ${WATCHED} success`,
+  ], WATCHED)).toEqual(["Program data: SGVsbG8=", `Program return: ${WATCHED} AQID`]);
+});

package/src/sync-protocols/solana/program-logs.ts

@@ -0,0 +1,84 @@
+/**
+ * Attributing Solana log lines to the program that actually emitted them.
+ *
+ * A transaction's `meta.logMessages` is a flat, structured stream covering every
+ * program the transaction touched:
+ *
+ * ```
+ * Program A invoke [1]
+ * Program log: from A
+ * Program B invoke [2]          ← A calls B (CPI)
+ * Program log: from B
+ * Program B consumed 1234 of 200000 compute units
+ * Program B success
+ * Program log: from A again
+ * Program A success
+ * ```
+ *
+ * Two things follow, and the naive `accountKeys.includes(programId)` +
+ * substring-match approach gets both wrong:
+ *
+ *  1. **Presence in `accountKeys` does not mean invocation.** Any transaction
+ *     may list an arbitrary account key without calling it. Only an
+ *     `invoke [N]` line proves the program ran — which is why this module is
+ *     the authorization boundary for the `SOLANA:ProgramLog` primitive. Without
+ *     it, an attacker names the watched program as a bare account key, has
+ *     their *own* program emit the expected text, and the primitive reports it
+ *     as the watched program's event with an attacker-chosen payload.
+ *  2. **Log lines belong to the innermost frame.** Forwarding every line in the
+ *     transaction hands the state machine other programs' output under the
+ *     watched program's name.
+ *
+ * Reading the stream also fixes an ALT blind spot for free: a program invoked
+ * via an address lookup table is absent from `message.accountKeys` (which holds
+ * static keys only) but always appears in its `invoke` line.
+ */
+
+/** `Program <id> invoke [<depth>]` — pushes a frame. */
+const INVOKE_RE = /^Program (\S+) invoke \[\d+\]$/;
+/** `Program <id> success` — pops a frame. */
+const SUCCESS_RE = /^Program (\S+) success$/;
+/** `Program <id> failed: <reason>` — pops a frame. */
+const FAILED_RE = /^Program (\S+) failed: /;
+/** `Program <id> consumed <n> of <m> compute units` — metering noise, not output. */
+const CONSUMED_RE = /^Program (\S+) consumed \d+ of \d+ compute units$/;
+
+/**
+ * The log lines `programId` itself emitted in this transaction, or `null` if it
+ * was never invoked.
+ *
+ * An empty array is meaningful and distinct from `null`: the program ran but
+ * emitted no output of its own (the System Program, for instance, logs only its
+ * `invoke`/`success` framing). Callers that care about *invocation* should check
+ * for `null`; callers that need *content* should also check `.length`.
+ */
+export function extractProgramLogs(
+  logMessages: readonly string[],
+  programId: string,
+): string[] | null {
+  // Frames currently open, innermost last. Truncated logs can leave this
+  // unbalanced, so pops are tolerant of an empty stack.
+  const frames: string[] = [];
+  const collected: string[] = [];
+  let invoked = false;
+
+  for (const line of logMessages) {
+    const invoke = INVOKE_RE.exec(line);
+    if (invoke) {
+      frames.push(invoke[1]);
+      if (invoke[1] === programId) invoked = true;
+      continue;
+    }
+    if (SUCCESS_RE.test(line) || FAILED_RE.test(line)) {
+      frames.pop();
+      continue;
+    }
+    if (CONSUMED_RE.test(line)) continue;
+
+    if (frames[frames.length - 1] === programId) {
+      collected.push(line);
+    }
+  }
+
+  return invoked ? collected : null;
+}