@effectstream/runtime 0.100.13 → 0.100.15

package/README.md

@@ -0,0 +1,89 @@
+# @effectstream/runtime
+
+The state-machine runtime — the loop that ties sync, state machine,
+database, events, and HTTP API together inside an EffectStream node.
+Boot it with `init()` then drive it with `start(config)` and your node
+is up.
+
+## Install
+
+```bash
+bun add @effectstream/runtime
+# or
+npm install @effectstream/runtime
+```
+
+`@effectstream/runtime` is the canonical import for `init` and `start`,
+used directly by every template. The same surface is also reachable
+through `@effectstream/node-sdk/runtime` if you prefer the umbrella.
+
+## Standalone usage
+
+This package **owns the node's process model**. You don't pick parts of
+it; you call `init()` once at boot and then `start(config)`. The config
+brings together everything else:
+
+```typescript
+import { init, start } from "@effectstream/runtime";
+import { Stm } from "@effectstream/sm";
+import { config } from "./config.dev.ts";
+
+await init();
+
+const gameStm = new Stm(grammar);
+gameStm.addStateTransition("join", function* () { /* ... */ });
+
+await start({
+  config,
+  gameStateTransitions: [gameStm],
+  apiRouter: undefined,           // optional Fastify route plugin
+  dbMigrations: [],               // SQL migrations
+});
+```
+
+While running, the runtime:
+
+- Reads finalized blocks via `@effectstream/sync`.
+- Routes batcher inputs through your registered `Stm`s.
+- Commits all yielded SQL inside a per-block transaction.
+- Publishes lifecycle and app events via `@effectstream/event-server`.
+- Serves the optional Fastify API.
+- Emits OpenTelemetry traces and logs through `@effectstream/log`.
+
+## Inside EffectStream
+
+`@effectstream/runtime` is the conductor: it doesn't define any chain
+integrations or queries itself, but every other node package only gets
+exercised when this loop is running. The `StartConfig` shape determines
+which DB migrations apply, which state machines fire, and whether the
+runtime exposes a Fastify router.
+
+## Key exports
+
+- `init()` — `Operation<void>`. One-shot setup: OpenTelemetry, config validation, version pinning. Call before `start`.
+- `start(config: StartConfig)` — `Operation<void>`. Run the node loop until cancelled.
+
+Types and helpers re-exported alongside `init` / `start`:
+
+- `DBMigrations` — versioned SQL migrations passed into `start`.
+- `StartConfig`, `StartConfigGameStateTransitions`, `StartConfigApiRouter` — `start`'s config types. Templates type-check against these implicitly but don't usually import them by name.
+- `PrimitiveConstructor<T>` — extension point for new primitives.
+- `VERSION` — `${number}.${number}.${number}` literal type for version pinning.
+- Pagination helpers re-exported from `./api/pagination.ts`.
+
+## Examples
+
+The templates under
+[`templates/`](https://github.com/PaimaStudios/paima-engine/tree/main/templates)
+are full working `init()` + `start()` examples. The simplest is
+[`templates/minimal/`](https://github.com/PaimaStudios/paima-engine/tree/main/templates/minimal).
+
+End-to-end EVM sync test:
+[`e2e/evm/sync/`](https://github.com/PaimaStudios/paima-engine/tree/main/e2e/evm/sync).
+
+Runnable: [`test/examples.test.ts`](./test/examples.test.ts).
+
+## Links
+
+- Docs: https://effectstream.github.io/docs/packages/node/runtime
+- Source: https://github.com/PaimaStudios/paima-engine/tree/main/packages/node-sdk/runtime

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@effectstream/runtime",
-  "version": "0.100.13",
+  "version": "0.100.15",
   "license": "MIT",
   "homepage": "https://effectstream.github.io/docs/",
   "repository": {
@@ -13,17 +13,17 @@
   },
   "exports": "./src/mod.ts",
   "dependencies": {
-    "@effectstream/concise": "0.100.13",
-    "@effectstream/config": "0.100.13",
-    "@effectstream/coroutine": "0.100.13",
-    "@effectstream/crypto": "0.100.13",
-    "@effectstream/db": "0.100.13",
-    "@effectstream/event-client": "0.100.13",
-    "@effectstream/event-server": "0.100.13",
-    "@effectstream/log": "0.100.13",
-    "@effectstream/sm": "0.100.13",
-    "@effectstream/sync": "0.100.13",
-    "@effectstream/utils": "0.100.13",
+    "@effectstream/concise": "0.100.15",
+    "@effectstream/config": "0.100.15",
+    "@effectstream/coroutine": "0.100.15",
+    "@effectstream/crypto": "0.100.15",
+    "@effectstream/db": "0.100.15",
+    "@effectstream/event-client": "0.100.15",
+    "@effectstream/event-server": "0.100.15",
+    "@effectstream/log": "0.100.15",
+    "@effectstream/sm": "0.100.15",
+    "@effectstream/sync": "0.100.15",
+    "@effectstream/utils": "0.100.15",
     "@fastify/cors": "^11.0.1",
     "@metamask/json-rpc-engine": "^10.0.3",
     "pg": "8.17.2",

package/src/api/http-server.ts

@@ -46,7 +46,7 @@ import {
   type TypePaginationQuerySchema,
 } from "./pagination.ts";
 import { PrimitiveRegistry } from "@effectstream/sm";
-import { ConfigNetworkType } from "@effectstream/config";
+import { ConfigNetworkType, getWriteNamespace, usePaimaStaticConfig } from "@effectstream/config";
 
 function tableListContains(
   list: Array<{ table_name: string | null }>,
@@ -470,26 +470,36 @@ export const startHttpServer = function* (
       return cleanedProtocols;
     });
 
+    const staticConfigForRoute = yield* usePaimaStaticConfig();
+    const securityNamespaceForRoute = getWriteNamespace(
+      staticConfigForRoute.securityNamespace,
+    );
+
     server.get("/config", {
       schema: {
         tags: ["developer"],
         response: {
-          200: Type.Array(Type.Object({
-            networkType: Type.String(),
-            syncProtocolType: Type.String(),
-            syncProtocol: Type.Object({}, { additionalProperties: true }),
-            network: Type.Object({}, { additionalProperties: true }),
-            primitives: Type.Array(
-              Type.Object({}, { additionalProperties: true }),
-            ),
-          }, { additionalProperties: true })),
+          200: Type.Object({
+            securityNamespace: Type.Union([Type.String(), Type.Null()]),
+            syncProtocols: Type.Array(Type.Object({
+              networkType: Type.String(),
+              syncProtocolType: Type.String(),
+              syncProtocol: Type.Object({}, { additionalProperties: true }),
+              network: Type.Object({}, { additionalProperties: true }),
+              primitives: Type.Array(
+                Type.Object({}, { additionalProperties: true }),
+              ),
+            }, { additionalProperties: true })),
+          }),
         },
       },
     }, () => {
       const config = syncProtocols.map((syncProtocol) => syncProtocol.config)
         .flat();
-      const cleanedConfig = clearBigInts(config);
-      return cleanedConfig;
+      return {
+        securityNamespace: securityNamespaceForRoute,
+        syncProtocols: clearBigInts(config),
+      };
     });
   }
 

package/src/main.ts

@@ -7,6 +7,7 @@ import {
   acquireDBMutex,
   createDynamicTables,
   getConnection,
+  getLastNonEmptyBlockHash,
   releaseDBMutex,
   resetPublicTables,
   runSnapshotLoop,
@@ -28,31 +29,19 @@ import {
   until,
 } from "effection";
 import { initTelemetry } from "./telemetry.ts";
-import { processFinalizedBlock } from "./process-blocks.ts";
+import { type PendingEvent, processFinalizedBlock } from "./process-blocks.ts";
 import { startHttpServer } from "./api/http-server.ts";
 import type { StartConfig } from "./types.ts";
 import type { Client } from "pg";
 import type { EffectstreamBlockHash } from "@effectstream/utils";
 import { applySystemMigrations } from "./version-migrations.ts";
 import { getLastBlockHeight, getVersionInfo } from "@effectstream/db/version";
-import { type SyncProtocolWithNetwork, ConfigNetworkType } from "@effectstream/config";
+import { ConfigNetworkType, usePaimaStaticConfig } from "@effectstream/config";
+import type { SecurityNamespace, SyncProtocolWithNetwork } from "@effectstream/config";
 import { builtInPrimitivesMap } from "@effectstream/sm";
 import { validateAndSnapshotConfig } from "./config-snapshot.ts";
 
 export function* init() {
-  // Prevent transient network errors (broken pipes, closed connections) from
-  // crashing the process via unhandled promise rejections. The sync loop's
-  // tryYield already retries on error — this just keeps the process alive.
-  globalThis.addEventListener("unhandledrejection", (event) => {
-    log.local(
-      ComponentNames.EFFECTSTREAM_RUNTIME,
-      "unhandled-rejection",
-      SeverityNumber.WARN,
-      (l) => l("Suppressed unhandled rejection:", event.reason),
-    );
-    event.preventDefault();
-  });
-
   // initialize OpenTelemetry
   yield* initTelemetry();
 }
@@ -137,7 +126,10 @@ export function* start(config: StartConfig): Operation<void> {
     }
   });
 
-  let blockHash: EffectstreamBlockHash | null = null;
+  const [lastHashRow] = yield* until(getLastNonEmptyBlockHash.run(undefined, dbConn));
+  let blockHash: EffectstreamBlockHash | null = lastHashRow
+    ? lastHashRow.effectstream_block_hash!.toString() as EffectstreamBlockHash
+    : null;
   if (config.snapshotConfig) {
     yield* spawn(() => runSnapshotLoop(config.snapshotConfig!));
   }
@@ -147,16 +139,25 @@ export function* start(config: StartConfig): Operation<void> {
     // For PGLite, this is not enough, as the can only be one connection at a time.
     // So we request a DBMutex as well.
     let dbClient: Client | undefined;
+    // Custom app events emitted by STFs during this block. Collected by
+    // processFinalizedBlock and flushed below ONLY after it returns — i.e.
+    // strictly after the block's COMMIT. See plan invariant I1.
+    let blockAppEvents: PendingEvent[] = [];
     try {
       yield* acquireDBMutex(`processing-blocks:${value.blockNumber}`);
       dbClient = yield* until((dbConn as any).connect()); // Client,
 
-      blockHash = yield* processFinalizedBlock(
+      const result = yield* processFinalizedBlock(
         value,
         config,
         dbClient as any, // Client,
         blockHash,
       );
+      const resultHash = result.blockHash;
+      blockAppEvents = result.events;
+      if (resultHash !== "0x0") {
+        blockHash = resultHash;
+      }
     } finally {
       releaseDBMutex(`processing-blocks:${value.blockNumber}`);
       if (dbClient) {
@@ -166,14 +167,38 @@ export function* start(config: StartConfig): Operation<void> {
 
     // Used to emit & log the block range for each protocol.
     const contentBlocksForProtocol = getRangesForSyncProtocols(value);
-  
-    yield* until(
-      emitLatestBlocks(
-        value.blockNumber,
-        value.timestamp,
-        contentBlocksForProtocol,
-      ),
+
+    // Fire-and-forget: do not stall the sync loop on broker acks.
+    //
+    // Opifex's TcpClient assigns packet IDs synchronously in client.publish()
+    // and writes to the socket via Node's net.Socket, which has its own write
+    // queue — so concurrent publish() calls land on the wire in call order.
+    // MQTT 3.1.1 §4.6 then guarantees in-order delivery to each subscriber
+    // per (publisher, topic, QoS). We don't need a wrapper queue.
+    //
+    // TODO(scaling): the broker runs in-process (see `new EventBroker(...)`
+    // above). Fan-out to N subscribers consumes the same event loop as block
+    // processing. At ~thousands of subscribers, consider:
+    //   1) moving the broker to a worker / separate process,
+    //   2) a publisher-side circuit breaker if ctx.unresolvedPublish.size > N
+    //      (drop events with a counter for observability).
+    // Today: localhost, in-process, expected O(10²) subscribers — non-issue.
+    emitLatestBlocks(
+      value.blockNumber,
+      value.timestamp,
+      contentBlocksForProtocol,
     );
+    for (const { event, payload } of blockAppEvents) {
+      EventManager.Instance.sendMessage(event, payload as any).catch((err) => {
+        log.local(
+          ComponentNames.EFFECTSTREAM_RUNTIME,
+          "event-publish",
+          SeverityNumber.WARN,
+          (l) =>
+            l(`publish ${event.path.join("/")} failed: ${String(err)}`),
+        );
+      });
+    }
 
     const lagMs = Date.now() - value.timestamp;
     const lagSuffix = lagThresholdMs != null && lagMs > lagThresholdMs
@@ -208,24 +233,40 @@ function getRangesForSyncProtocols(value: ChainBlock): Record<string, [number, n
   return contentBlocksForProtocol;
 }
 
-async function emitLatestBlocks(
+/**
+ * Publish built-in block-level events.
+ *
+ * Fire-and-forget for the same reasons as the app-event flush above
+ * (do not stall the sync loop; ordering preserved by Opifex + MQTT 3.1.1).
+ *
+ * Errors are caught locally so they don't escape to the global
+ * `unhandledrejection` handler without context.
+ */
+function emitLatestBlocks(
   rollUpBlockHeight: number,
   rollUpBlockTimestamp: number,
   syncChains: Record<string, [number, number]>,
-) {
-  return await Promise.all([
-    EventManager.Instance.sendMessage(BuiltinEvents.RollupBlock, {
-      block: rollUpBlockHeight,
-      timestamp: rollUpBlockTimestamp,
-    }),
-    ...Object.entries(syncChains).map(([chainName, [_, toBlock]]) =>
-      EventManager.Instance.sendMessage(BuiltinEvents.SyncChains, {
-        chain: chainName,
-        block: toBlock,
-        rollup: rollUpBlockHeight
-      })
-    ),
-  ]);
+): void {
+  const logFailure = (topic: string) => (err: unknown) =>
+    log.local(
+      ComponentNames.EFFECTSTREAM_RUNTIME,
+      "event-publish",
+      SeverityNumber.WARN,
+      (l) => l(`publish ${topic} failed: ${String(err)}`),
+    );
+
+  EventManager.Instance.sendMessage(BuiltinEvents.RollupBlock, {
+    block: rollUpBlockHeight,
+    timestamp: rollUpBlockTimestamp,
+  }).catch(logFailure("RollupBlock"));
+
+  for (const [chainName, [_, toBlock]] of Object.entries(syncChains)) {
+    EventManager.Instance.sendMessage(BuiltinEvents.SyncChains, {
+      chain: chainName,
+      block: toBlock,
+      rollup: rollUpBlockHeight,
+    }).catch(logFailure(`SyncChains/${chainName}`));
+  }
 }
 
 function* startup(
@@ -235,12 +276,16 @@ function* startup(
 ): Operation<AllSyncProtocols[]> {
   const versionInfo = yield* getVersionInfo(dbConn);
   const lastBlockHeight = yield* getLastBlockHeight(versionInfo, dbConn);
+  // Pull the security namespace from the static config so primitives that
+  // re-verify batched signatures can access it synchronously.
+  const staticConfig = yield* usePaimaStaticConfig();
   // Create Runtime Primitives Instances
   syncInfo.forEach((syncProtocol) => {
     syncProtocol.primitives.forEach((primitive, primitiveIndex) => {
       processPrimitives(
         syncProtocol.primitives,
         primitiveIndex,
+        staticConfig.securityNamespace,
         config.userDefinedPrimitives
       );
     });
@@ -287,6 +332,7 @@ function* startup(
 const processPrimitives = (
   primitives: {primitive: any, id: string}[],
   primitiveIndex: number,
+  securityNamespace: SecurityNamespace | undefined,
   userDefinedPrimitives?: Record<string, any>,
 ) => {
     const primitiveType = primitives[primitiveIndex].primitive.type;
@@ -309,6 +355,7 @@ const processPrimitives = (
     const classConfig = {
       ...primitiveConfig,
       instanceName: primitiveUniqueName,
+      securityNamespace,
     }
     if (isBuiltInPrimitive) {
       p = new builtInPrimitivesMap[primitiveType as keyof typeof builtInPrimitivesMap](classConfig as any) ;

package/src/process-blocks.ts

@@ -1,7 +1,7 @@
 import type { ChainBlock } from "@effectstream/sync";
 import { call, type Operation, until } from "effection";
 import type { Pool } from "pg";
-import { type BaseStfInput, primitiveTransitionFunction } from "@effectstream/sm";
+import { type BaseStfInput, type EmitFn, primitiveTransitionFunction } from "@effectstream/sm";
 import { PreparedQuery } from "@pgtyped/runtime";
 import type {
   ExecPromise,
@@ -10,6 +10,7 @@ import type {
 } from "@effectstream/coroutine";
 import {
   blockHeightDone,
+  deleteEmptyBlocks,
   deleteScheduled,
   getFutureGameInputByBlockHeight,
   getFutureGameInputByMaxTimestamp,
@@ -23,6 +24,24 @@ import type { StartConfig } from "./types.ts";
 import type { BlockNumber, EffectstreamBlockHash } from "@effectstream/utils";
 import { generateEffectstreamBlockHash, Prando } from "@effectstream/crypto";
 import { applyUserMigrations } from "./version-migrations.ts";
+import type { EventPathAndDef } from "@effectstream/event-client";
+
+/**
+ * A pending app event collected during STF execution, flushed to MQTT by
+ * the caller (main.ts) only after the block-level `COMMIT` succeeds.
+ *
+ * See plan invariants I1 (post-COMMIT delivery) and I2 (rollback drops events)
+ * in /Users/edwardalvarado/.claude/plans/wild-kindling-reddy.md.
+ */
+export type PendingEvent = {
+  event: EventPathAndDef;
+  payload: Record<string, unknown>;
+};
+
+export type ProcessFinalizedBlockResult = {
+  blockHash: EffectstreamBlockHash;
+  events: PendingEvent[];
+};
 
 /** Helper to check if a SyncStateUpdateStream object is a WorldResolve */
 function isWorldResolve(value: any): value is QueuedUpdate {
@@ -127,12 +146,17 @@ export function* processFinalizedBlock(
   config: StartConfig,
   dbConn: Pool,
   previousBlockHash: EffectstreamBlockHash | null,
-): Operation<EffectstreamBlockHash> {
+): Operation<ProcessFinalizedBlockResult> {
   const { gameStateTransitions, migrations } = config;
   const blockHash: EffectstreamBlockHash = generateEffectstreamBlockHash(
     value,
     previousBlockHash,
   );
+  // Per-block event buffer. Only events from STF runs that succeeded
+  // (and survived to the post-COMMIT return) end up here. On block-level
+  // ROLLBACK below, this buffer goes out of scope unflushed.
+  const blockEvents: PendingEvent[] = [];
+  let hasOperations = false;
   try {
     /* STEP 0: Start the transaction. */
     yield* until(dbConn.query("BEGIN"));
@@ -151,8 +175,9 @@ export function* processFinalizedBlock(
     );
 
     /* STEP 2: Process the migrations. */
+    let hasMigrations = false;
     if (migrations) {
-      yield* applyUserMigrations(
+      hasMigrations = yield* applyUserMigrations(
         value.blockNumber,
         dbConn as any, // Client,
         migrations,
@@ -190,6 +215,18 @@ export function* processFinalizedBlock(
       randomGenerator.skip();
       for (const data of scheduledData) {
         let success = true;
+        // Per-input event buffer. If the STF throws below, this buffer is
+        // dropped (events from rolled-back inputs never reach MQTT — I2).
+        // On success, we promote into `blockEvents` after the catch block.
+        const inputEvents: PendingEvent[] = [];
+        const emit: EmitFn = (event, payload) => {
+          inputEvents.push({
+            event,
+            // Auto-inject blockHeight (I5). registerEvents prepends it as the
+            // first indexed field; apps never set it themselves.
+            payload: { ...payload, blockHeight: value.blockNumber },
+          });
+        };
         try {
           const input: BaseStfInput = {
             blockTimestamp: value.timestamp,
@@ -198,6 +235,7 @@ export function* processFinalizedBlock(
             signerAddress: data.from_address,
             signerAddressType: data.from_address_type,
             randomGenerator,
+            emit,
             // TODO: We might want to add this to the scheduled data.
             //
             //      Or do we resolve it here. Account might change.
@@ -210,6 +248,12 @@ export function* processFinalizedBlock(
             input,
           );
           yield* executeGeneratorStepByStep(gameSTFGenerator, dbConn);
+          // STF succeeded: promote buffered events to the block buffer.
+          // Failure path falls through the catch below and `inputEvents`
+          // simply goes out of scope.
+          if (inputEvents.length > 0) {
+            blockEvents.push(...inputEvents);
+          }
         } catch (err) {
           success = false;
           log.remote(
@@ -251,9 +295,15 @@ export function* processFinalizedBlock(
     }
 
     /* STEP 6: Mark the block as done. */
+    hasOperations = hasMigrations ||
+      value.primitives.length > 0 ||
+      scheduledData.length > 0;
+    if (!hasOperations) {
+      yield* call(() => deleteEmptyBlocks.run(undefined, dbConn));
+    }
     yield* call(() =>
       blockHeightDone.run({
-        block_hash: Buffer.from(blockHash),
+        block_hash: Buffer.from(hasOperations ? blockHash : "0x0"),
         block_height: value.blockNumber,
       }, dbConn)
     );
@@ -298,5 +348,9 @@ export function* processFinalizedBlock(
     // We cannot recover from this error.
     throw err;
   }
-  return blockHash;
+
+  return { 
+    blockHash: hasOperations ? blockHash : "0x0" as PaimaBlockHash, 
+    events: blockEvents 
+  };
 }

package/src/version-migrations.ts

@@ -95,7 +95,7 @@ export function* applyUserMigrations(
   currentBlockHeight: number,
   dbConn: Client,
   migrationOrder: DBMigrations[] | undefined,
-): Operation<void> {
+): Operation<boolean> {
   const migrations = yield* getAllUserMigrations(
     EFFECTSTREAM_ENGINE_VERSION,
     migrationOrder,
@@ -114,6 +114,7 @@ export function* applyUserMigrations(
       ),
     );
   }
+  return migrations.length > 0;
 }
 
 function* getAllUserMigrations(

package/test/examples.test.ts

@@ -0,0 +1,14 @@
+// Examples for the README — verify the public surface resolves.
+
+import { test, expect } from "bun:test";
+import * as runtime from "../src/mod.ts";
+
+test("README: init and start are exported as generator functions", () => {
+  expect(typeof runtime.init).toBe("function");
+  expect(typeof runtime.start).toBe("function");
+});
+
+test("README: pagination helpers are re-exported", async () => {
+  const pagination = await import("../src/api/pagination.ts");
+  expect(Object.keys(pagination).length).toBeGreaterThan(0);
+});