@effectstream/runtime 0.100.0

package/src/main.ts

@@ -0,0 +1,317 @@
+import {
+  type AllSyncProtocols,
+  type ChainBlock,
+  genSyncProtocols,
+} from "@effectstream/sync";
+import {
+  acquireDBMutex,
+  createDynamicTables,
+  getConnection,
+  releaseDBMutex,
+  resetPublicTables,
+  runSnapshotLoop,
+} from "@effectstream/db";
+import { EventBroker } from "@effectstream/event-server";
+import {
+  BuiltinEvents,
+  EventManager,
+} from "@effectstream/event-client";
+import { startMerge, startSync } from "@effectstream/sync";
+import { ComponentNames, log, SeverityNumber } from "@effectstream/log";
+import {
+  createChannel,
+  each,
+  type Operation,
+  sleep,
+  spawn,
+  until,
+} from "effection";
+import { initTelemetry } from "./telemetry.ts";
+import { processFinalizedBlock } from "./process-blocks.ts";
+import { startHttpServer } from "./api/http-server.ts";
+import type { StartConfig } from "./types.ts";
+import type { Client } from "pg";
+import type { EffectstreamBlockHash } from "@effectstream/utils";
+import { applySystemMigrations } from "./version-migrations.ts";
+import { getLastBlockHeight, getVersionInfo } from "@effectstream/db/version";
+import { type SyncProtocolWithNetwork, ConfigNetworkType } from "@effectstream/config";
+import { builtInPrimitivesMap } from "@effectstream/sm";
+import { validateAndSnapshotConfig } from "./config-snapshot.ts";
+
+export function* init() {
+  // Prevent transient network errors (broken pipes, closed connections) from
+  // crashing the process via unhandled promise rejections. The sync loop's
+  // tryYield already retries on error — this just keeps the process alive.
+  globalThis.addEventListener("unhandledrejection", (event) => {
+    log.local(
+      ComponentNames.EFFECTSTREAM_RUNTIME,
+      "unhandled-rejection",
+      SeverityNumber.WARN,
+      (l) => l("Suppressed unhandled rejection:", event.reason),
+    );
+    event.preventDefault();
+  });
+
+  // initialize OpenTelemetry
+  yield* initTelemetry();
+}
+
+/**
+ * Main entry point to start the Paima Engine Node.
+ *
+ * This will launch the networks/primitives synchronization sub-processes,
+ * the HTTP server, and the merge and effectstream-block generation process.
+ *
+ * @param config - Paima Engine Node configuration object.
+ */
+export function* start(config: StartConfig): Operation<void> {
+  const { syncInfo } = config;
+
+  const dbConn = getConnection();
+
+  const syncProtocols = yield* startup(dbConn as any, // Client,
+    syncInfo, config);
+
+  log.remote(
+    ComponentNames.EFFECTSTREAM_RUNTIME,
+    [],
+    SeverityNumber.INFO,
+    (log) => log("start sync", syncProtocols.map(p => p.name)),
+  );
+  for (const syncProtocol of syncProtocols) {
+    yield* startSync(syncProtocol);
+  }
+
+  // Create MQTT Broker
+  new EventBroker("effectstream-engine").createServer();
+
+  yield* spawn(function* () {
+    yield* startHttpServer(
+      dbConn,
+      syncProtocols,
+      config.apiRouter,
+      config.grammar,
+    );
+  });
+
+  const ntpConfig = syncInfo.find(s => s.networkType === ConfigNetworkType.NTP);
+  const ntpBlockTimeMS = (ntpConfig?.network as { blockTimeMS: number } | undefined)?.blockTimeMS;
+  const lagThresholdMs = ntpBlockTimeMS != null ? ntpBlockTimeMS * 10 : undefined;
+
+  const finalizedBlockStream = createChannel<ChainBlock>();
+
+  yield* spawn(() => startMerge(syncProtocols, finalizedBlockStream));
+
+  const heartbeatIntervalMs = 60_000;
+  yield* spawn(function* () {
+    while (true) {
+      yield* sleep(heartbeatIntervalMs);
+      const now = Date.now();
+      const status = syncProtocols.map((p) => {
+        const page = p.lastPage;
+        if (page == null) return `${p.name}: waiting for first sync`;
+        const ageMs = now - (page.root as number);
+        let line = `${p.name}: block ${page.ownBlockNumber} | buf ${p.bufferedData.size()} | age ${(ageMs / 1000).toFixed(1)}s`;
+        if (p.consecutiveErrors > 0) {
+          const sinceLast = p.lastErrorTimestamp > 0
+            ? ` ${((now - p.lastErrorTimestamp) / 1000).toFixed(0)}s ago`
+            : "";
+          line += ` | ERRORS: ${p.consecutiveErrors}${sinceLast}`;
+        } else if (p.lastSuccessfulFetchMs > 0) {
+          const idleMs = now - p.lastSuccessfulFetchMs;
+          if (idleMs > heartbeatIntervalMs * 2) {
+            line += ` | IDLE: ${(idleMs / 1000).toFixed(0)}s`;
+          }
+        }
+        return line;
+      });
+      log.local(
+        ComponentNames.EFFECTSTREAM_SYNC,
+        "heartbeat",
+        SeverityNumber.INFO,
+        (l) => l(status.join(" | ")),
+      );
+    }
+  });
+
+  let blockHash: EffectstreamBlockHash | null = null;
+  if (config.snapshotConfig) {
+    yield* spawn(() => runSnapshotLoop(config.snapshotConfig!));
+  }
+
+  for (const value of yield* each(finalizedBlockStream)) {
+    // We request a dbClient for a non-shared dbConn object.
+    // For PGLite, this is not enough, as the can only be one connection at a time.
+    // So we request a DBMutex as well.
+    let dbClient: Client | undefined;
+    try {
+      yield* acquireDBMutex(`processing-blocks:${value.blockNumber}`);
+      dbClient = yield* until((dbConn as any).connect()); // Client,
+
+      blockHash = yield* processFinalizedBlock(
+        value,
+        config,
+        dbClient as any, // Client,
+        blockHash,
+      );
+    } finally {
+      releaseDBMutex(`processing-blocks:${value.blockNumber}`);
+      if (dbClient) {
+        (dbClient as any).release(); // Client,
+      }
+    }
+
+    // Used to emit & log the block range for each protocol.
+    const contentBlocksForProtocol = getRangesForSyncProtocols(value);
+  
+    yield* until(
+      emitLatestBlocks(
+        value.blockNumber,
+        value.timestamp,
+        contentBlocksForProtocol,
+      ),
+    );
+
+    const lagMs = Date.now() - value.timestamp;
+    const lagSuffix = lagThresholdMs != null && lagMs > lagThresholdMs
+      ? ` | lag: ${(lagMs / 1000).toFixed(1)}s`
+      : "";
+    log.local(
+      ComponentNames.EFFECTSTREAM_SYNC,
+      "block-merge",
+      SeverityNumber.INFO,
+      (log) =>
+        log(
+          `finalized block ${value.blockNumber} @ ${
+            blockHash?.slice(0, 8)
+          }...${lagSuffix} | ${JSON.stringify(contentBlocksForProtocol)}`,
+        ),
+    );
+    yield* each.next();
+  }
+}
+
+function getRangesForSyncProtocols(value: ChainBlock): Record<string, [number, number]> {
+  const contentBlocksForProtocol: Record<string, [number, number]> = {};
+  for (const block of value.blockInfo) {
+    if (!contentBlocksForProtocol[block.protocol_name]) {
+      contentBlocksForProtocol[block.protocol_name] = [block.block_number, block.block_number];
+    }
+    contentBlocksForProtocol[block.protocol_name] = [
+      Math.min(contentBlocksForProtocol[block.protocol_name][0], block.block_number),
+      Math.max(contentBlocksForProtocol[block.protocol_name][1], block.block_number),
+    ];
+  }
+  return contentBlocksForProtocol;
+}
+
+async function emitLatestBlocks(
+  rollUpBlockHeight: number,
+  rollUpBlockTimestamp: number,
+  syncChains: Record<string, [number, number]>,
+) {
+  return await Promise.all([
+    EventManager.Instance.sendMessage(BuiltinEvents.RollupBlock, {
+      block: rollUpBlockHeight,
+      timestamp: rollUpBlockTimestamp,
+    }),
+    ...Object.entries(syncChains).map(([chainName, [_, toBlock]]) =>
+      EventManager.Instance.sendMessage(BuiltinEvents.SyncChains, {
+        chain: chainName,
+        block: toBlock,
+        rollup: rollUpBlockHeight
+      })
+    ),
+  ]);
+}
+
+function* startup(
+  dbConn: Client,
+  syncInfo: SyncProtocolWithNetwork[],
+  config: StartConfig,
+): Operation<AllSyncProtocols[]> {
+  const versionInfo = yield* getVersionInfo(dbConn);
+  const lastBlockHeight = yield* getLastBlockHeight(versionInfo, dbConn);
+  // Create Runtime Primitives Instances
+  syncInfo.forEach((syncProtocol) => {
+    syncProtocol.primitives.forEach((primitive, primitiveIndex) => {
+      processPrimitives(
+        syncProtocol.primitives,
+        primitiveIndex,
+        config.userDefinedPrimitives
+      );
+    });
+  });
+
+  
+  yield* acquireDBMutex(`startup-node`);
+
+  // Dev-only reset of user-owned public tables
+  if (config.dev?.resetPublicData) {
+    yield* resetPublicTables(dbConn as any); // Client,
+  }
+
+  // When the node is started, we apply system migrations.
+  // Either system initial migrations, or migrations given a Paima Engine Update.
+  yield* applySystemMigrations(
+    config.appVersion,
+    versionInfo,
+    lastBlockHeight,
+    dbConn,
+    config.migrations,
+  );
+
+  // Validate that immutable config fields (e.g. NTP startTime, startBlockHeight)
+  // have not changed since the last run. Persists a snapshot on first start.
+  // Must run after system migrations so the snapshot table exists.
+  yield* validateAndSnapshotConfig(syncInfo, dbConn);
+
+  const syncProtocols = yield* genSyncProtocols(dbConn as any, // Client,
+    syncInfo);
+
+  yield* createDynamicTables(
+    versionInfo,
+    lastBlockHeight,
+    dbConn as any, // Client,
+    syncProtocols,
+  );
+
+  releaseDBMutex(`startup-node`);
+  return syncProtocols;
+}
+
+// Convert the primitive config to the final primitive instance
+const processPrimitives = (
+  primitives: {primitive: any, id: string}[],
+  primitiveIndex: number,
+  userDefinedPrimitives?: Record<string, any>,
+) => {
+    const primitiveType = primitives[primitiveIndex].primitive.type;
+    const primitiveUniqueName = primitives[primitiveIndex].id;
+    const primitiveConfig = primitives[primitiveIndex].primitive;
+    const isBuiltInPrimitive = primitiveType in builtInPrimitivesMap;
+    const isUserDefinedPrimitive = userDefinedPrimitives && primitiveType in userDefinedPrimitives;
+    if (isBuiltInPrimitive && isUserDefinedPrimitive) {
+      throw new Error(`User defined primitive cannot have the same name as a built-in primitive.
+                       Built-in values: ${Object.keys(builtInPrimitivesMap).join(", ")}`);
+    }
+    if (!isBuiltInPrimitive && !isUserDefinedPrimitive) {
+      throw new Error(`PrimitiveUniqueName "${primitiveUniqueName}" is not built-in and not user-defined.
+                       Available values: ${Object.keys([
+                        ...Object.keys(builtInPrimitivesMap),
+                        ...Object.keys(userDefinedPrimitives || {}),
+                      ]).join(", ")}`);
+    }
+    let p = null;
+    const classConfig = {
+      ...primitiveConfig,
+      instanceName: primitiveUniqueName,
+    }
+    if (isBuiltInPrimitive) {
+      p = new builtInPrimitivesMap[primitiveType as keyof typeof builtInPrimitivesMap](classConfig as any) ;
+    } else if (isUserDefinedPrimitive) {
+      p = new userDefinedPrimitives[primitiveType as keyof typeof userDefinedPrimitives](classConfig);
+    }
+    // Update the primitive with the final configuration
+    primitives[primitiveIndex].primitive = p.getConfig();
+}

package/src/mod.ts

@@ -0,0 +1,3 @@
+export { init, start } from "./main.ts";
+export * from "./types.ts";
+export * from "./api/pagination.ts";

package/src/process-blocks.ts

@@ -0,0 +1,302 @@
+import type { ChainBlock } from "@effectstream/sync";
+import { call, type Operation, until } from "effection";
+import type { Pool } from "pg";
+import { type BaseStfInput, primitiveTransitionFunction } from "@effectstream/sm";
+import { PreparedQuery } from "@pgtyped/runtime";
+import type {
+  ExecPromise,
+  QueuedUpdate,
+  SyncStateUpdateStream,
+} from "@effectstream/coroutine";
+import {
+  blockHeightDone,
+  deleteScheduled,
+  getFutureGameInputByBlockHeight,
+  getFutureGameInputByMaxTimestamp,
+  insertGameInputResult,
+  removePages,
+  saveLastBlock,
+} from "@effectstream/db";
+import { Buffer } from "node:buffer";
+import { ComponentNames, log, SeverityNumber } from "@effectstream/log";
+import type { StartConfig } from "./types.ts";
+import type { BlockNumber, EffectstreamBlockHash } from "@effectstream/utils";
+import { generateEffectstreamBlockHash, Prando } from "@effectstream/crypto";
+import { applyUserMigrations } from "./version-migrations.ts";
+
+/** Helper to check if a SyncStateUpdateStream object is a WorldResolve */
+function isWorldResolve(value: any): value is QueuedUpdate {
+  return value && Array.isArray(value);
+}
+/** Helper to check if a SyncStateUpdateStream object is a promise */
+function isPromise(value: any): value is ExecPromise<any> {
+  return value && typeof value === "object" && "promise" in value;
+}
+
+/**
+ * We need to process all the SQL calls of an STF update in an all-or-nothing manner
+ * STF updates can fail (since the data for them comes from arbitrary onchain data)
+ * But we can't allow a single user's bad transaction to DOS the game for everybody else
+ * So failures should be isolated to just the specific input, and not the full block
+ * (recall: without this, in pgsql, if a query fails during a db transaction, the entire dbTx becomes invalid)
+ */
+async function tryOrRollback<T>(
+  dbTx: Pool,
+  func: () => Promise<T>,
+): Promise<{ status: "success" | "error"; value: T | undefined }> {
+  const checkpointName = `game_state_transition`;
+  await dbTx.query(`SAVEPOINT ${checkpointName}`);
+  try {
+    const value = await func();
+    return { status: "success", value };
+  } catch (err) {
+    await dbTx.query(`ROLLBACK TO SAVEPOINT ${checkpointName}`);
+    log.remote(
+      ComponentNames.EFFECTSTREAM_SYNC,
+      "block-processing",
+      SeverityNumber.INFO,
+      (log) =>
+        log(`Database error on ${checkpointName}. Rolling back.` + String(err)),
+    );
+    return { status: "error", value: undefined };
+  } finally {
+    await dbTx.query(`RELEASE SAVEPOINT ${checkpointName}`);
+  }
+}
+
+/**
+ * This function is used to execute a generator step by step.
+ * Each step returns either a Query or a StateMachineExecution.
+ * The results are processed, then the generator is resumed.
+ */
+function* executeGeneratorStepByStep(
+  generator: SyncStateUpdateStream<void>,
+  dbConn: Pool,
+): Operation<any> {
+  let result = generator.next();
+
+  // NOTE: sync generators cannot execute promises
+  //       so we pause the execution, and resolve them here.
+  // NOTE: there are 2 types of operations we need to handle
+  //       1. state machine executions, these return a series of queries to run.
+  //       2. generic promises required in the primitives (e.g., verifySignature)
+  while (!result.done) {
+    // We resolve the generators promises here.
+    // As generators cannot execute promises.
+    const operations: unknown[] = [];
+    const value = result.value;
+
+    if (isWorldResolve(value)) {
+      const [queryIR, params] = value;
+      const query = new PreparedQuery(queryIR);
+      const queryResult = yield* call(() => query.run(params, dbConn));
+      operations.push(queryResult);
+    } else if (isPromise(value)) {
+      // This means that we have non-database promises in the generator.
+      // Each time yield is called, we catch the intention and resolve it.
+      const queryResult = yield* until(value.promise);
+      // We wrap the result, but it will be flattened by the generator.
+      operations.push([queryResult]);
+    } else {
+      console.error("Yielding unhandled type", result);
+      throw new Error("Unhandled type in generator");
+    }
+    result = generator.next(operations.flat());
+  }
+  return result.value;
+}
+
+/**
+ * This function is the main entry point for processing a produced block.
+ * It is called when a block can be processed and finalized.
+ * It runs the entire pipeline in a transaction, with sub-transactions for each StateMachineExecution.
+ *
+ * Process Order:
+ * 1. Create a temporal block record
+ * 2. Process the migrations for this block height
+ * 4. Extract the scheduled data from the primitives in the block
+ * 4. Process the scheduled data for this block height, and time
+ * 5. Mark the block as done, and add the hash.
+ * 6. Commit the transaction
+ *
+ * IMPORTANT: This function must receive a non-shared dbConn object.
+ *            as it will be used in a transaction.
+ */
+export function* processFinalizedBlock(
+  value: ChainBlock,
+  config: StartConfig,
+  dbConn: Pool,
+  previousBlockHash: EffectstreamBlockHash | null,
+): Operation<EffectstreamBlockHash> {
+  const { gameStateTransitions, migrations } = config;
+  const blockHash: EffectstreamBlockHash = generateEffectstreamBlockHash(
+    value,
+    previousBlockHash,
+  );
+  try {
+    /* STEP 0: Start the transaction. */
+    yield* until(dbConn.query("BEGIN"));
+    const randomGenerator = new Prando(blockHash);
+
+    /* STEP 1: Create a temporal block record */
+    yield* call(() =>
+      saveLastBlock.run({
+        // TODO: Check these values
+        block_height: value.blockNumber,
+        ver: 0,
+        main_chain_block_hash: Buffer.from(value.blockNumber.toString()),
+        seed: randomGenerator.seed.toString(),
+        ms_timestamp: new Date(value.timestamp),
+      }, dbConn)
+    );
+
+    /* STEP 2: Process the migrations. */
+    if (migrations) {
+      yield* applyUserMigrations(
+        value.blockNumber,
+        dbConn as any, // Client,
+        migrations,
+      );
+    }
+
+    /* STEP 3: Process the primitives. */
+    for (const primitive of value.primitives) {
+      const generator = primitiveTransitionFunction(
+        value.blockNumber,
+        primitive,
+      );
+      yield* executeGeneratorStepByStep(generator, dbConn);
+    }
+
+    /* STEP 4: Extract the scheduled data. */
+    const scheduledData1 = yield* call(() =>
+      getFutureGameInputByBlockHeight.run({
+        block_height: value.blockNumber,
+      }, dbConn)
+    );
+    /* STEP 4.b: Extract the scheduled data by timestamp. */
+    const scheduledData2 = yield* call(() =>
+      getFutureGameInputByMaxTimestamp.run({
+        max_timestamp: new Date(value.timestamp),
+      }, dbConn)
+    );
+
+    /* STEP 5: Process the scheduled data in the State Machine. */
+    // TODO What should be the order of the scheduled data - per id?
+    const scheduledData = [...scheduledData1, ...scheduledData2];
+    let index_in_block = 0;
+
+    if (gameStateTransitions && scheduledData.length > 0) {
+      randomGenerator.skip();
+      for (const data of scheduledData) {
+        let success = true;
+        try {
+          const input: BaseStfInput = {
+            blockTimestamp: value.timestamp,
+            blockHeight: value.blockNumber,
+            conciseInput: data.input_data,
+            signerAddress: data.from_address,
+            signerAddressType: data.from_address_type,
+            randomGenerator,
+            // TODO: We might want to add this to the scheduled data.
+            //
+            //      Or do we resolve it here. Account might change.
+            //      Should we preserve the original accountId or the signer's
+            //      current accountId?
+            accountId: undefined, // data.accountId,
+          };
+          const gameSTFGenerator = gameStateTransitions(
+            value.blockNumber,
+            input,
+          );
+          yield* executeGeneratorStepByStep(gameSTFGenerator, dbConn);
+        } catch (err) {
+          success = false;
+          log.remote(
+            ComponentNames.EFFECTSTREAM_SYNC,
+            "block-processing",
+            SeverityNumber.ERROR,
+            (log) =>
+              log(
+                `Error processing block ${value.blockNumber}: ${
+                  String(err)
+                }. Payload: ${JSON.stringify(data)}`,
+              ),
+          );
+        }
+        const conciseInputHash = `0x${
+          Array(64).fill(0).map(() =>
+            Math.floor(Math.random() * 16).toString(16)
+          )
+            .join("")
+        }`;
+        yield* until(
+          insertGameInputResult.run({
+            id: data.id,
+            success,
+            effectstream_tx_hash: Buffer.from(conciseInputHash),
+            index_in_block,
+            block_height: value.blockNumber,
+          }, dbConn),
+        );
+        index_in_block++;
+
+        // Remove the scheduled data from the database.
+        yield* until(
+          deleteScheduled.run({
+            id: data.id,
+          }, dbConn),
+        );
+      }
+    }
+
+    /* STEP 6: Mark the block as done. */
+    yield* call(() =>
+      blockHeightDone.run({
+        block_hash: Buffer.from(blockHash),
+        block_height: value.blockNumber,
+      }, dbConn)
+    );
+
+    /* STEP 7: Update page state for all sync protocols consumed
+    *          This is to mark what blocks where consumed by this specific
+    *          block, to avoid re-processing the same blocks more than once
+    *          if the sync protocol is restarted.
+    */
+
+    // Get latest block for each sync protocol.
+    const latestBlocks: Record<string, BlockNumber> = {};
+    for (const blockInfo of value.blockInfo) {
+      if (
+        !latestBlocks[blockInfo.protocol_name] ||
+        latestBlocks[blockInfo.protocol_name] < blockInfo.block_number
+      ) {
+        latestBlocks[blockInfo.protocol_name] = blockInfo.block_number;
+      }
+    }
+
+    for (const [protocolName, blockNumber] of Object.entries(latestBlocks)) {
+      yield* until(
+        removePages.run({
+          protocol_name: protocolName,
+          page_number: blockNumber,
+        }, dbConn),
+      );
+    }
+
+    /* STEP 8: Commit the transaction. */
+    yield* until(dbConn.query("COMMIT"));
+  } catch (err) {
+    yield* until(dbConn.query("ROLLBACK"));
+    log.remote(
+      ComponentNames.EFFECTSTREAM_SYNC,
+      "block-processing",
+      SeverityNumber.ERROR,
+      (log) =>
+        log(`Error processing block ${value.blockNumber}: ${String(err)}`),
+    );
+    // We cannot recover from this error.
+    throw err;
+  }
+  return blockHash;
+}

package/src/telemetry.ts

@@ -0,0 +1,37 @@
+import { NodeSDK } from "@opentelemetry/sdk-node";
+import { getNodeAutoInstrumentations } from "@opentelemetry/auto-instrumentations-node";
+import opentelemetry from "@opentelemetry/api";
+import { defaultOtelSetup, PaimaTelemetryContext } from "@effectstream/log";
+import type { Operation } from "effection";
+import fs from "node:fs";
+import path from "node:path";
+import { parse } from "jsonc-parser";
+
+// TODO: this is hacky, but there is no proper way to do this unfortunately
+// We're blocked by any of the following
+// 1. Allowing `type: jsonc` imports: https://github.com/denoland/deno/issues/20374#issuecomment-2543118482
+// 2. Allowing `type: tesxt` imports: https://github.com/denoland/deno/issues/25354
+const version = "0.3.0";
+// 3. import.meta.dirname is undefined when loaded as a jsr package
+// const DenoConfig = parse(fs.readFileSync(
+//   path.resolve(import.meta.dirname!, "../", "./deno.json"),
+//   "utf-8",
+// ));
+
+export function* initTelemetry(): Operation<void> {
+  const sdk = new NodeSDK({
+    ...defaultOtelSetup("effectstream-node", version), // DenoConfig.version),
+    // TODO: set OTEL_EXPORTER_OTLP_PROTOCOL to json for @effectstream/collector support until it supports protobuf
+    instrumentations: [getNodeAutoInstrumentations()],
+  });
+
+  const tracer = opentelemetry.trace.getTracer("paima", version); // DenoConfig.version);
+  yield* PaimaTelemetryContext.set(
+    {
+      tracer,
+      span: undefined,
+    },
+  );
+
+  sdk.start();
+}