@latticexyz/store-indexer 2.0.0-next.14 → 2.0.0-next.16

package/src/koa-middleware/helloWorld.ts

@@ -0,0 +1,12 @@
+import { Middleware } from "koa";
+
+export function helloWorld(): Middleware {
+  return async function helloWorldMiddleware(ctx, next): Promise<void> {
+    if (ctx.path === "/") {
+      ctx.status = 200;
+      ctx.body = "emit HelloWorld();";
+      return;
+    }
+    await next();
+  };
+}

package/src/koa-middleware/sentry.ts

@@ -0,0 +1,101 @@
+import * as Sentry from "@sentry/node";
+import { ProfilingIntegration } from "@sentry/profiling-node";
+import { stripUrlQueryAndFragment } from "@sentry/utils";
+import debug from "debug";
+import Koa from "koa";
+import compose from "koa-compose";
+
+export function errorHandler(): Koa.Middleware {
+  return async function errorHandlerMiddleware(ctx, next) {
+    try {
+      await next();
+    } catch (err) {
+      Sentry.withScope((scope) => {
+        scope.addEventProcessor((event) => {
+          return Sentry.addRequestDataToEvent(event, ctx.request);
+        });
+        Sentry.captureException(err);
+      });
+      throw err;
+    }
+  };
+}
+
+export function requestHandler(): Koa.Middleware {
+  return async function requestHandlerMiddleware(ctx, next) {
+    await Sentry.runWithAsyncContext(async () => {
+      const hub = Sentry.getCurrentHub();
+      hub.configureScope((scope) =>
+        scope.addEventProcessor((event) =>
+          Sentry.addRequestDataToEvent(event, ctx.request, {
+            include: {
+              user: false,
+            },
+          })
+        )
+      );
+      await next();
+    });
+  };
+}
+
+export function tracing(): Koa.Middleware {
+  // creates a Sentry transaction per request
+  return async function tracingMiddleware(ctx, next) {
+    const reqMethod = (ctx.method || "").toUpperCase();
+    const reqUrl = ctx.url && stripUrlQueryAndFragment(ctx.url);
+
+    // Connect to trace of upstream app
+    let traceparentData;
+    if (ctx.request.get("sentry-trace")) {
+      traceparentData = Sentry.extractTraceparentData(ctx.request.get("sentry-trace"));
+    }
+
+    const transaction = Sentry.startTransaction({
+      name: `${reqMethod} ${reqUrl}`,
+      op: "http.server",
+      ...traceparentData,
+    });
+
+    ctx.__sentry_transaction = transaction;
+
+    // We put the transaction on the scope so users can attach children to it
+    Sentry.getCurrentHub().configureScope((scope) => {
+      scope.setSpan(transaction);
+    });
+
+    ctx.res.on("finish", () => {
+      // Push `transaction.finish` to the next event loop so open spans have a chance to finish before the transaction closes
+      setImmediate(() => {
+        // If you're using koa router, set the matched route as transaction name
+        if (ctx._matchedRoute) {
+          const mountPath = ctx.mountPath || "";
+          transaction.setName(`${reqMethod} ${mountPath}${ctx._matchedRoute}`);
+        }
+
+        transaction.setHttpStatus(ctx.status);
+        transaction.finish();
+      });
+    });
+
+    await next();
+  };
+}
+
+export function sentry(dsn: string): Koa.Middleware {
+  debug("Initializing Sentry");
+  Sentry.init({
+    dsn,
+    integrations: [
+      // Automatically instrument Node.js libraries and frameworks
+      ...Sentry.autoDiscoverNodePerformanceMonitoringIntegrations(),
+      new ProfilingIntegration(),
+    ],
+    // Performance Monitoring
+    tracesSampleRate: 1.0,
+    // Set sampling rate for profiling - this is relative to tracesSampleRate
+    profilesSampleRate: 1.0,
+  });
+
+  return compose([errorHandler(), requestHandler(), tracing()]);
+}

package/src/postgres/apiRoutes.ts

@@ -0,0 +1,58 @@
+import { Sql } from "postgres";
+import { Middleware } from "koa";
+import Router from "@koa/router";
+import compose from "koa-compose";
+import { input } from "@latticexyz/store-sync/indexer-client";
+import { storeTables } from "@latticexyz/store-sync";
+import { queryLogs } from "./queryLogs";
+import { recordToLog } from "./recordToLog";
+import { debug, error } from "../debug";
+import { createBenchmark } from "@latticexyz/common";
+import { compress } from "../koa-middleware/compress";
+
+export function apiRoutes(database: Sql): Middleware {
+  const router = new Router();
+
+  router.get("/api/logs", compress(), async (ctx) => {
+    const benchmark = createBenchmark("postgres:logs");
+    let options: ReturnType<typeof input.parse>;
+
+    try {
+      options = input.parse(typeof ctx.query.input === "string" ? JSON.parse(ctx.query.input) : {});
+    } catch (e) {
+      ctx.status = 400;
+      ctx.body = JSON.stringify(e);
+      debug(e);
+      return;
+    }
+
+    try {
+      options.filters = options.filters.length > 0 ? [...options.filters, { tableId: storeTables.Tables.tableId }] : [];
+      const records = await queryLogs(database, options ?? {}).execute();
+      benchmark("query records");
+      const logs = records.map(recordToLog);
+      benchmark("map records to logs");
+
+      if (records.length === 0) {
+        ctx.status = 404;
+        ctx.body = "no logs found";
+        error(
+          `no logs found for chainId ${options.chainId}, address ${options.address}, filters ${JSON.stringify(
+            options.filters
+          )}`
+        );
+        return;
+      }
+
+      const blockNumber = records[0].chainBlockNumber;
+      ctx.body = JSON.stringify({ blockNumber, logs });
+      ctx.status = 200;
+    } catch (e) {
+      ctx.status = 500;
+      ctx.body = JSON.stringify(e);
+      error(e);
+    }
+  });
+
+  return compose([router.routes(), router.allowedMethods()]) as Middleware;
+}

package/src/postgres/common.ts

@@ -0,0 +1,21 @@
+import { Hex } from "viem";
+
+export type RecordData = {
+  address: Hex;
+  tableId: Hex;
+  keyBytes: Hex;
+  staticData: Hex | null;
+  encodedLengths: Hex | null;
+  dynamicData: Hex | null;
+  recordBlockNumber: string;
+  logIndex: number;
+};
+
+export type RecordMetadata = {
+  indexerVersion: string;
+  chainId: string;
+  chainBlockNumber: string;
+  totalRows: number;
+};
+
+export type Record = RecordData & RecordMetadata;

package/src/postgres/deprecated/createQueryAdapter.ts

@@ -0,0 +1,56 @@
+import { getAddress } from "viem";
+import { PgDatabase } from "drizzle-orm/pg-core";
+import { TableWithRecords, isTableRegistrationLog, logToTable, storeTables } from "@latticexyz/store-sync";
+import { decodeKey, decodeValueArgs } from "@latticexyz/protocol-parser";
+import { QueryAdapter } from "@latticexyz/store-sync/trpc-indexer";
+import { debug } from "../../debug";
+import { getLogs } from "./getLogs";
+import { groupBy } from "@latticexyz/common/utils";
+
+/**
+ * Creates a query adapter for the tRPC server/client to query data from Postgres.
+ *
+ * @param {PgDatabase<any>} database Postgres database object from Drizzle
+ * @returns {Promise<QueryAdapter>} A set of methods used by tRPC endpoints.
+ * @deprecated
+ */
+export async function createQueryAdapter(database: PgDatabase<any>): Promise<QueryAdapter> {
+  const adapter: QueryAdapter = {
+    async getLogs(opts) {
+      return getLogs(database, opts);
+    },
+    async findAll(opts) {
+      const filters = opts.filters ?? [];
+      const { blockNumber, logs } = await getLogs(database, {
+        ...opts,
+        // make sure we're always retrieving `store.Tables` table, so we can decode table values
+        filters: filters.length > 0 ? [...filters, { tableId: storeTables.Tables.tableId }] : [],
+      });
+
+      const tables = logs.filter(isTableRegistrationLog).map(logToTable);
+
+      const logsByTable = groupBy(logs, (log) => `${getAddress(log.address)}:${log.args.tableId}`);
+
+      const tablesWithRecords: TableWithRecords[] = tables.map((table) => {
+        const tableLogs = logsByTable.get(`${getAddress(table.address)}:${table.tableId}`) ?? [];
+        const records = tableLogs.map((log) => ({
+          key: decodeKey(table.keySchema, log.args.keyTuple),
+          value: decodeValueArgs(table.valueSchema, log.args),
+        }));
+
+        return {
+          ...table,
+          records,
+        };
+      });
+
+      debug("findAll: decoded %d logs across %d tables", logs.length, tables.length);
+
+      return {
+        blockNumber,
+        tables: tablesWithRecords,
+      };
+    },
+  };
+  return adapter;
+}

package/src/postgres/deprecated/getLogs.ts

@@ -0,0 +1,81 @@
+import { PgDatabase } from "drizzle-orm/pg-core";
+import { Hex } from "viem";
+import { StorageAdapterLog, SyncFilter } from "@latticexyz/store-sync";
+import { tables } from "@latticexyz/store-sync/postgres";
+import { and, asc, eq, or } from "drizzle-orm";
+import { bigIntMax } from "@latticexyz/common/utils";
+import { recordToLog } from "../recordToLog";
+import { createBenchmark } from "@latticexyz/common";
+
+/**
+ * @deprecated
+ */
+export async function getLogs(
+  database: PgDatabase<any>,
+  {
+    chainId,
+    address,
+    filters = [],
+  }: {
+    readonly chainId: number;
+    readonly address?: Hex;
+    readonly filters?: readonly SyncFilter[];
+  }
+): Promise<{ blockNumber: bigint; logs: (StorageAdapterLog & { eventName: "Store_SetRecord" })[] }> {
+  const benchmark = createBenchmark("drizzleGetLogs");
+
+  const conditions = filters.length
+    ? filters.map((filter) =>
+        and(
+          address != null ? eq(tables.recordsTable.address, address) : undefined,
+          eq(tables.recordsTable.tableId, filter.tableId),
+          filter.key0 != null ? eq(tables.recordsTable.key0, filter.key0) : undefined,
+          filter.key1 != null ? eq(tables.recordsTable.key1, filter.key1) : undefined
+        )
+      )
+    : address != null
+    ? [eq(tables.recordsTable.address, address)]
+    : [];
+  benchmark("parse config");
+
+  // Query for the block number that the indexer (i.e. chain) is at, in case the
+  // indexer is further along in the chain than a given store/table's last updated
+  // block number. We'll then take the highest block number between the indexer's
+  // chain state and all the records in the query (in case the records updated
+  // between these queries). Using just the highest block number from the queries
+  // could potentially signal to the client an older-than-necessary block number,
+  // for stores/tables that haven't seen recent activity.
+  // TODO: move the block number query into the records query for atomicity so we don't have to merge them here
+  const chainState = await database
+    .select()
+    .from(tables.configTable)
+    .where(eq(tables.configTable.chainId, chainId))
+    .limit(1)
+    .execute()
+    // Get the first record in a way that returns a possible `undefined`
+    // TODO: move this to `.findFirst` after upgrading drizzle or `rows[0]` after enabling `noUncheckedIndexedAccess: true`
+    .then((rows) => rows.find(() => true));
+  const indexerBlockNumber = chainState?.blockNumber ?? 0n;
+  benchmark("query chainState");
+
+  const records = await database
+    .select()
+    .from(tables.recordsTable)
+    .where(or(...conditions))
+    .orderBy(
+      asc(tables.recordsTable.blockNumber)
+      // TODO: add logIndex (https://github.com/latticexyz/mud/issues/1979)
+    );
+  benchmark("query records");
+
+  const blockNumber = records.reduce((max, record) => bigIntMax(max, record.blockNumber ?? 0n), indexerBlockNumber);
+  benchmark("find block number");
+
+  const logs = records
+    // TODO: add this to the query, assuming we can optimize with an index
+    .filter((record) => !record.isDeleted)
+    .map(recordToLog);
+  benchmark("map records to logs");
+
+  return { blockNumber, logs };
+}

package/src/postgres/queryLogs.ts

@@ -0,0 +1,71 @@
+import { isNotNull } from "@latticexyz/common/utils";
+import { PendingQuery, Row, Sql } from "postgres";
+import { hexToBytes } from "viem";
+import { z } from "zod";
+import { input } from "@latticexyz/store-sync/indexer-client";
+import { transformSchemaName } from "@latticexyz/store-sync/postgres";
+import { Record } from "./common";
+
+const schemaName = transformSchemaName("mud");
+
+function and(sql: Sql, conditions: PendingQuery<Row[]>[]): PendingQuery<Row[]> {
+  return sql`(${conditions.reduce((query, condition) => sql`${query} AND ${condition}`)})`;
+}
+
+function or(sql: Sql, conditions: PendingQuery<Row[]>[]): PendingQuery<Row[]> {
+  return sql`(${conditions.reduce((query, condition) => sql`${query} OR ${condition}`)})`;
+}
+
+export function queryLogs(sql: Sql, opts: z.infer<typeof input>): PendingQuery<Record[]> {
+  const conditions = opts.filters.length
+    ? opts.filters.map((filter) =>
+        and(
+          sql,
+          [
+            opts.address != null ? sql`address = ${hexToBytes(opts.address)}` : null,
+            sql`table_id = ${hexToBytes(filter.tableId)}`,
+            filter.key0 != null ? sql`key0 = ${hexToBytes(filter.key0)}` : null,
+            filter.key1 != null ? sql`key1 = ${hexToBytes(filter.key1)}` : null,
+          ].filter(isNotNull)
+        )
+      )
+    : opts.address != null
+    ? [sql`address = ${hexToBytes(opts.address)}`]
+    : [];
+
+  const where = sql`WHERE ${and(
+    sql,
+    [sql`is_deleted != true`, conditions.length ? or(sql, conditions) : null].filter(isNotNull)
+  )}`;
+
+  // TODO: implement bytea <> hex columns via custom types: https://github.com/porsager/postgres#custom-types
+  return sql<Record[]>`
+    WITH
+      config AS (
+        SELECT
+          version AS "indexerVersion",
+          chain_id AS "chainId",
+          block_number AS "chainBlockNumber"
+        FROM ${sql(`${schemaName}.config`)}
+        LIMIT 1
+      ),
+      records AS (
+        SELECT
+          '0x' || encode(address, 'hex') AS address,
+          '0x' || encode(table_id, 'hex') AS "tableId",
+          '0x' || encode(key_bytes, 'hex') AS "keyBytes",
+          '0x' || encode(static_data, 'hex') AS "staticData",
+          '0x' || encode(encoded_lengths, 'hex') AS "encodedLengths",
+          '0x' || encode(dynamic_data, 'hex') AS "dynamicData",
+          block_number AS "recordBlockNumber",
+          log_index AS "logIndex"
+        FROM ${sql(`${schemaName}.records`)}
+        ${where}
+        ORDER BY block_number, log_index ASC
+      )
+    SELECT
+      (SELECT COUNT(*) FROM records) AS "totalRows",
+      *
+    FROM config, records
+  `;
+}

package/src/postgres/recordToLog.ts

@@ -0,0 +1,19 @@
+import { StorageAdapterLog } from "@latticexyz/store-sync";
+import { decodeDynamicField } from "@latticexyz/protocol-parser";
+import { RecordData } from "./common";
+
+export function recordToLog(
+  record: Omit<RecordData, "recordBlockNumber">
+): StorageAdapterLog & { eventName: "Store_SetRecord" } {
+  return {
+    address: record.address,
+    eventName: "Store_SetRecord",
+    args: {
+      tableId: record.tableId,
+      keyTuple: decodeDynamicField("bytes32[]", record.keyBytes),
+      staticData: record.staticData ?? "0x",
+      encodedLengths: record.encodedLengths ?? "0x",
+      dynamicData: record.dynamicData ?? "0x",
+    },
+  } as const;
+}

package/src/sqlite/apiRoutes.ts

@@ -0,0 +1,47 @@
+import { Middleware } from "koa";
+import Router from "@koa/router";
+import compose from "koa-compose";
+import { input } from "@latticexyz/store-sync/indexer-client";
+import { storeTables, tablesWithRecordsToLogs } from "@latticexyz/store-sync";
+import { debug } from "../debug";
+import { createBenchmark } from "@latticexyz/common";
+import { compress } from "../koa-middleware/compress";
+import { getTablesWithRecords } from "./getTablesWithRecords";
+import { BaseSQLiteDatabase } from "drizzle-orm/sqlite-core";
+
+export function apiRoutes(database: BaseSQLiteDatabase<"sync", any>): Middleware {
+  const router = new Router();
+
+  router.get("/api/logs", compress(), async (ctx) => {
+    const benchmark = createBenchmark("sqlite:logs");
+
+    let options: ReturnType<typeof input.parse>;
+
+    try {
+      options = input.parse(typeof ctx.query.input === "string" ? JSON.parse(ctx.query.input) : {});
+    } catch (error) {
+      ctx.status = 400;
+      ctx.body = JSON.stringify(error);
+      debug(error);
+      return;
+    }
+
+    try {
+      options.filters = options.filters.length > 0 ? [...options.filters, { tableId: storeTables.Tables.tableId }] : [];
+      benchmark("parse config");
+      const { blockNumber, tables } = getTablesWithRecords(database, options);
+      benchmark("query tables with records");
+      const logs = tablesWithRecordsToLogs(tables);
+      benchmark("convert records to logs");
+
+      ctx.body = JSON.stringify({ blockNumber: blockNumber?.toString() ?? "-1", logs });
+      ctx.status = 200;
+    } catch (error) {
+      ctx.status = 500;
+      ctx.body = JSON.stringify(error);
+      debug(error);
+    }
+  });
+
+  return compose([router.routes(), router.allowedMethods()]) as Middleware;
+}

package/src/sqlite/createQueryAdapter.ts

@@ -1,10 +1,7 @@
-import { eq } from "drizzle-orm";
 import { BaseSQLiteDatabase } from "drizzle-orm/sqlite-core";
-import { buildTable, chainState, getTables } from "@latticexyz/store-sync/sqlite";
 import { QueryAdapter } from "@latticexyz/store-sync/trpc-indexer";
-import { debug } from "../debug";
-import { getAddress } from "viem";
-import { decodeDynamicField } from "@latticexyz/protocol-parser";
+import { getTablesWithRecords } from "./getTablesWithRecords";
+import { tablesWithRecordsToLogs } from "@latticexyz/store-sync";
 
 /**
  * Creates a storage adapter for the tRPC server/client to query data from SQLite.
@@ -14,48 +11,13 @@ import { decodeDynamicField } from "@latticexyz/protocol-parser";
  */
 export async function createQueryAdapter(database: BaseSQLiteDatabase<"sync", any>): Promise<QueryAdapter> {
   const adapter: QueryAdapter = {
-    async findAll({ chainId, address, filters = [] }) {
-      // If _any_ filter has a table ID, this will filter down all data to just those tables. Which mean we can't yet mix table filters with key-only filters.
-      // TODO: improve this so we can express this in the query (need to be able to query data across tables more easily)
-      const tableIds = Array.from(new Set(filters.map((filter) => filter.tableId)));
-      const tables = getTables(database)
-        .filter((table) => address == null || getAddress(address) === getAddress(table.address))
-        .filter((table) => !tableIds.length || tableIds.includes(table.tableId));
-
-      const tablesWithRecords = tables.map((table) => {
-        const sqliteTable = buildTable(table);
-        const records = database.select().from(sqliteTable).where(eq(sqliteTable.__isDeleted, false)).all();
-        const filteredRecords = !filters.length
-          ? records
-          : records.filter((record) => {
-              const keyTuple = decodeDynamicField("bytes32[]", record.__key);
-              return filters.some(
-                (filter) =>
-                  filter.tableId === table.tableId &&
-                  (filter.key0 == null || filter.key0 === keyTuple[0]) &&
-                  (filter.key1 == null || filter.key1 === keyTuple[1])
-              );
-            });
-        return {
-          ...table,
-          records: filteredRecords.map((record) => ({
-            key: Object.fromEntries(Object.entries(table.keySchema).map(([name]) => [name, record[name]])),
-            value: Object.fromEntries(Object.entries(table.valueSchema).map(([name]) => [name, record[name]])),
-          })),
-        };
-      });
-
-      const metadata = database.select().from(chainState).where(eq(chainState.chainId, chainId)).all();
-      const { lastUpdatedBlockNumber } = metadata[0] ?? {};
-
-      const result = {
-        blockNumber: lastUpdatedBlockNumber ?? null,
-        tables: tablesWithRecords,
-      };
-
-      debug("findAll", chainId, address, result);
-
-      return result;
+    async getLogs(opts) {
+      const { blockNumber, tables } = getTablesWithRecords(database, opts);
+      const logs = tablesWithRecordsToLogs(tables);
+      return { blockNumber: blockNumber ?? 0n, logs };
+    },
+    async findAll(opts) {
+      return getTablesWithRecords(database, opts);
     },
   };
   return adapter;

package/src/sqlite/getTablesWithRecords.ts

@@ -0,0 +1,75 @@
+import { asc, eq } from "drizzle-orm";
+import { BaseSQLiteDatabase } from "drizzle-orm/sqlite-core";
+import { buildTable, chainState, getTables } from "@latticexyz/store-sync/sqlite";
+import { Hex, getAddress } from "viem";
+import { decodeDynamicField } from "@latticexyz/protocol-parser";
+import { SyncFilter, TableWithRecords } from "@latticexyz/store-sync";
+
+// TODO: refactor sqlite and replace this with getLogs to match postgres (https://github.com/latticexyz/mud/issues/1970)
+
+/**
+ * @deprecated
+ * */
+export function getTablesWithRecords(
+  database: BaseSQLiteDatabase<"sync", any>,
+  {
+    chainId,
+    address,
+    filters = [],
+  }: {
+    readonly chainId: number;
+    readonly address?: Hex;
+    readonly filters?: readonly SyncFilter[];
+  }
+): { blockNumber: bigint | null; tables: readonly TableWithRecords[] } {
+  const metadata = database
+    .select()
+    .from(chainState)
+    .where(eq(chainState.chainId, chainId))
+    .limit(1)
+    .all()
+    .find(() => true);
+
+  // If _any_ filter has a table ID, this will filter down all data to just those tables. Which mean we can't yet mix table filters with key-only filters.
+  // TODO: improve this so we can express this in the query (need to be able to query data across tables more easily)
+  const tableIds = Array.from(new Set(filters.map((filter) => filter.tableId)));
+  const tables = getTables(database)
+    .filter((table) => address == null || getAddress(address) === getAddress(table.address))
+    .filter((table) => !tableIds.length || tableIds.includes(table.tableId));
+
+  const tablesWithRecords = tables.map((table) => {
+    const sqliteTable = buildTable(table);
+    const records = database
+      .select()
+      .from(sqliteTable)
+      .where(eq(sqliteTable.__isDeleted, false))
+      .orderBy(
+        asc(sqliteTable.__lastUpdatedBlockNumber)
+        // TODO: add logIndex (https://github.com/latticexyz/mud/issues/1979)
+      )
+      .all();
+    const filteredRecords = !filters.length
+      ? records
+      : records.filter((record) => {
+          const keyTuple = decodeDynamicField("bytes32[]", record.__key);
+          return filters.some(
+            (filter) =>
+              filter.tableId === table.tableId &&
+              (filter.key0 == null || filter.key0 === keyTuple[0]) &&
+              (filter.key1 == null || filter.key1 === keyTuple[1])
+          );
+        });
+    return {
+      ...table,
+      records: filteredRecords.map((record) => ({
+        key: Object.fromEntries(Object.entries(table.keySchema).map(([name]) => [name, record[name]])),
+        value: Object.fromEntries(Object.entries(table.valueSchema).map(([name]) => [name, record[name]])),
+      })),
+    };
+  });
+
+  return {
+    blockNumber: metadata?.lastUpdatedBlockNumber ?? null,
+    tables: tablesWithRecords,
+  };
+}

package/dist/chunk-LOFAZSVJ.js

@@ -1,2 +0,0 @@
-import e from"debug";var o=e("mud:store-indexer");export{o as a};
-//# sourceMappingURL=chunk-LOFAZSVJ.js.map

package/dist/chunk-LOFAZSVJ.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/debug.ts"],"sourcesContent":["import createDebug from \"debug\";\n\nexport const debug = createDebug(\"mud:store-indexer\");\n"],"mappings":"AAAA,OAAOA,MAAiB,QAEjB,IAAMC,EAAQD,EAAY,mBAAmB","names":["createDebug","debug"]}