@latticexyz/store-indexer 2.0.0-snapshot-test-32d38619 → 2.0.0

package/src/koa-middleware/compress.ts

@@ -0,0 +1,48 @@
+import { Middleware } from "koa";
+import { Readable, Stream } from "node:stream";
+import accepts from "accepts";
+import { Zlib, createBrotliCompress, createDeflate, createGzip } from "node:zlib";
+import { includes } from "@latticexyz/common/utils";
+
+// Loosely based on https://github.com/holic/koa-compress/blob/master/lib/index.js
+// with better handling of streams better with occasional flushing
+
+const encodings = {
+  br: createBrotliCompress,
+  gzip: createGzip,
+  deflate: createDeflate,
+} as const;
+
+const encodingNames = Object.keys(encodings) as (keyof typeof encodings)[];
+
+function flushEvery<stream extends Zlib & Readable>(stream: stream, bytesThreshold: number): stream {
+  let bytesSinceFlush = 0;
+  stream.on("data", (data) => {
+    bytesSinceFlush += data.length;
+    if (bytesSinceFlush > bytesThreshold) {
+      bytesSinceFlush = 0;
+      stream.flush();
+    }
+  });
+  return stream;
+}
+
+type CompressOptions = {
+  flushThreshold?: number;
+};
+
+export function compress({ flushThreshold = 1024 * 4 }: CompressOptions = {}): Middleware {
+  return async function compressMiddleware(ctx, next) {
+    ctx.vary("Accept-Encoding");
+
+    await next();
+
+    const encoding = accepts(ctx.req).encoding(encodingNames);
+    if (!includes(encodingNames, encoding)) return;
+
+    const compressed = flushEvery(encodings[encoding](), flushThreshold);
+
+    ctx.set("Content-Encoding", encoding);
+    ctx.body = ctx.body instanceof Stream ? ctx.body.pipe(compressed) : compressed.end(ctx.body);
+  };
+}

package/src/koa-middleware/healthcheck.ts

@@ -0,0 +1,37 @@
+import { Middleware } from "koa";
+
+type HealthcheckOptions = {
+  isHealthy?: () => boolean;
+  isReady?: () => boolean;
+};
+
+/**
+ * Middleware to add Kubernetes healthcheck endpoints
+ */
+export function healthcheck({ isHealthy, isReady }: HealthcheckOptions = {}): Middleware {
+  return async function healthcheckMiddleware(ctx, next): Promise<void> {
+    if (ctx.path === "/healthz") {
+      if (isHealthy == null || isHealthy()) {
+        ctx.status = 200;
+        ctx.body = "healthy";
+      } else {
+        ctx.status = 503;
+        ctx.body = "not healthy";
+      }
+      return;
+    }
+
+    if (ctx.path === "/readyz") {
+      if (isReady == null || isReady()) {
+        ctx.status = 200;
+        ctx.body = "ready";
+      } else {
+        ctx.status = 503;
+        ctx.body = "not ready";
+      }
+      return;
+    }
+
+    await next();
+  };
+}

package/src/koa-middleware/helloWorld.ts

@@ -0,0 +1,12 @@
+import { Middleware } from "koa";
+
+export function helloWorld(): Middleware {
+  return async function helloWorldMiddleware(ctx, next): Promise<void> {
+    if (ctx.path === "/") {
+      ctx.status = 200;
+      ctx.body = "emit HelloWorld();";
+      return;
+    }
+    await next();
+  };
+}

package/src/koa-middleware/sentry.ts

@@ -0,0 +1,101 @@
+import * as Sentry from "@sentry/node";
+import { ProfilingIntegration } from "@sentry/profiling-node";
+import { stripUrlQueryAndFragment } from "@sentry/utils";
+import debug from "debug";
+import Koa from "koa";
+import compose from "koa-compose";
+
+export function errorHandler(): Koa.Middleware {
+  return async function errorHandlerMiddleware(ctx, next) {
+    try {
+      await next();
+    } catch (err) {
+      Sentry.withScope((scope) => {
+        scope.addEventProcessor((event) => {
+          return Sentry.addRequestDataToEvent(event, ctx.request);
+        });
+        Sentry.captureException(err);
+      });
+      throw err;
+    }
+  };
+}
+
+export function requestHandler(): Koa.Middleware {
+  return async function requestHandlerMiddleware(ctx, next) {
+    await Sentry.runWithAsyncContext(async () => {
+      const hub = Sentry.getCurrentHub();
+      hub.configureScope((scope) =>
+        scope.addEventProcessor((event) =>
+          Sentry.addRequestDataToEvent(event, ctx.request, {
+            include: {
+              user: false,
+            },
+          }),
+        ),
+      );
+      await next();
+    });
+  };
+}
+
+export function tracing(): Koa.Middleware {
+  // creates a Sentry transaction per request
+  return async function tracingMiddleware(ctx, next) {
+    const reqMethod = (ctx.method || "").toUpperCase();
+    const reqUrl = ctx.url && stripUrlQueryAndFragment(ctx.url);
+
+    // Connect to trace of upstream app
+    let traceparentData;
+    if (ctx.request.get("sentry-trace")) {
+      traceparentData = Sentry.extractTraceparentData(ctx.request.get("sentry-trace"));
+    }
+
+    const transaction = Sentry.startTransaction({
+      name: `${reqMethod} ${reqUrl}`,
+      op: "http.server",
+      ...traceparentData,
+    });
+
+    ctx.__sentry_transaction = transaction;
+
+    // We put the transaction on the scope so users can attach children to it
+    Sentry.getCurrentHub().configureScope((scope) => {
+      scope.setSpan(transaction);
+    });
+
+    ctx.res.on("finish", () => {
+      // Push `transaction.finish` to the next event loop so open spans have a chance to finish before the transaction closes
+      setImmediate(() => {
+        // If you're using koa router, set the matched route as transaction name
+        if (ctx._matchedRoute) {
+          const mountPath = ctx.mountPath || "";
+          transaction.setName(`${reqMethod} ${mountPath}${ctx._matchedRoute}`);
+        }
+
+        transaction.setHttpStatus(ctx.status);
+        transaction.finish();
+      });
+    });
+
+    await next();
+  };
+}
+
+export function sentry(dsn: string): Koa.Middleware {
+  debug("Initializing Sentry");
+  Sentry.init({
+    dsn,
+    integrations: [
+      // Automatically instrument Node.js libraries and frameworks
+      ...Sentry.autoDiscoverNodePerformanceMonitoringIntegrations(),
+      new ProfilingIntegration(),
+    ],
+    // Performance Monitoring
+    tracesSampleRate: 1.0,
+    // Set sampling rate for profiling - this is relative to tracesSampleRate
+    profilesSampleRate: 1.0,
+  });
+
+  return compose([errorHandler(), requestHandler(), tracing()]);
+}

package/src/postgres/apiRoutes.ts

@@ -0,0 +1,58 @@
+import { Sql } from "postgres";
+import { Middleware } from "koa";
+import Router from "@koa/router";
+import compose from "koa-compose";
+import { input } from "@latticexyz/store-sync/indexer-client";
+import { storeTables } from "@latticexyz/store-sync";
+import { queryLogs } from "./queryLogs";
+import { recordToLog } from "./recordToLog";
+import { debug, error } from "../debug";
+import { createBenchmark } from "@latticexyz/common";
+import { compress } from "../koa-middleware/compress";
+
+export function apiRoutes(database: Sql): Middleware {
+  const router = new Router();
+
+  router.get("/api/logs", compress(), async (ctx) => {
+    const benchmark = createBenchmark("postgres:logs");
+    let options: ReturnType<typeof input.parse>;
+
+    try {
+      options = input.parse(typeof ctx.query.input === "string" ? JSON.parse(ctx.query.input) : {});
+    } catch (e) {
+      ctx.status = 400;
+      ctx.body = JSON.stringify(e);
+      debug(e);
+      return;
+    }
+
+    try {
+      options.filters = options.filters.length > 0 ? [...options.filters, { tableId: storeTables.Tables.tableId }] : [];
+      const records = await queryLogs(database, options ?? {}).execute();
+      benchmark("query records");
+      const logs = records.map(recordToLog);
+      benchmark("map records to logs");
+
+      if (records.length === 0) {
+        ctx.status = 404;
+        ctx.body = "no logs found";
+        error(
+          `no logs found for chainId ${options.chainId}, address ${options.address}, filters ${JSON.stringify(
+            options.filters,
+          )}`,
+        );
+        return;
+      }
+
+      const blockNumber = records[0].chainBlockNumber;
+      ctx.body = JSON.stringify({ blockNumber, logs });
+      ctx.status = 200;
+    } catch (e) {
+      ctx.status = 500;
+      ctx.body = JSON.stringify(e);
+      error(e);
+    }
+  });
+
+  return compose([router.routes(), router.allowedMethods()]) as Middleware;
+}

package/src/postgres/common.ts

@@ -0,0 +1,21 @@
+import { Hex } from "viem";
+
+export type RecordData = {
+  address: Hex;
+  tableId: Hex;
+  keyBytes: Hex;
+  staticData: Hex | null;
+  encodedLengths: Hex | null;
+  dynamicData: Hex | null;
+  recordBlockNumber: string;
+  logIndex: number;
+};
+
+export type RecordMetadata = {
+  indexerVersion: string;
+  chainId: string;
+  chainBlockNumber: string;
+  totalRows: number;
+};
+
+export type Record = RecordData & RecordMetadata;

package/src/postgres/deprecated/createQueryAdapter.ts

@@ -0,0 +1,57 @@
+import { getAddress } from "viem";
+import { PgDatabase } from "drizzle-orm/pg-core";
+import { TableWithRecords, isTableRegistrationLog, logToTable, storeTables } from "@latticexyz/store-sync";
+import { decodeKey, decodeValueArgs } from "@latticexyz/protocol-parser/internal";
+import { QueryAdapter } from "@latticexyz/store-sync/trpc-indexer";
+import { debug } from "../../debug";
+import { getLogs } from "./getLogs";
+import { groupBy } from "@latticexyz/common/utils";
+
+/**
+ * Creates a query adapter for the tRPC server/client to query data from Postgres.
+ *
+ * @param {PgDatabase<any>} database Postgres database object from Drizzle
+ * @returns {Promise<QueryAdapter>} A set of methods used by tRPC endpoints.
+ * @deprecated
+ */
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export async function createQueryAdapter(database: PgDatabase<any>): Promise<QueryAdapter> {
+  const adapter: QueryAdapter = {
+    async getLogs(opts) {
+      return getLogs(database, opts);
+    },
+    async findAll(opts) {
+      const filters = opts.filters ?? [];
+      const { blockNumber, logs } = await getLogs(database, {
+        ...opts,
+        // make sure we're always retrieving `store.Tables` table, so we can decode table values
+        filters: filters.length > 0 ? [...filters, { tableId: storeTables.Tables.tableId }] : [],
+      });
+
+      const tables = logs.filter(isTableRegistrationLog).map(logToTable);
+
+      const logsByTable = groupBy(logs, (log) => `${getAddress(log.address)}:${log.args.tableId}`);
+
+      const tablesWithRecords: TableWithRecords[] = tables.map((table) => {
+        const tableLogs = logsByTable.get(`${getAddress(table.address)}:${table.tableId}`) ?? [];
+        const records = tableLogs.map((log) => ({
+          key: decodeKey(table.keySchema, log.args.keyTuple),
+          value: decodeValueArgs(table.valueSchema, log.args),
+        }));
+
+        return {
+          ...table,
+          records,
+        };
+      });
+
+      debug("findAll: decoded %d logs across %d tables", logs.length, tables.length);
+
+      return {
+        blockNumber,
+        tables: tablesWithRecords,
+      };
+    },
+  };
+  return adapter;
+}

package/src/postgres/deprecated/getLogs.ts

@@ -0,0 +1,82 @@
+import { PgDatabase } from "drizzle-orm/pg-core";
+import { Hex } from "viem";
+import { StorageAdapterLog, SyncFilter } from "@latticexyz/store-sync";
+import { tables } from "@latticexyz/store-sync/postgres";
+import { and, asc, eq, or } from "drizzle-orm";
+import { bigIntMax } from "@latticexyz/common/utils";
+import { recordToLog } from "../recordToLog";
+import { createBenchmark } from "@latticexyz/common";
+
+/**
+ * @deprecated
+ */
+export async function getLogs(
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  database: PgDatabase<any>,
+  {
+    chainId,
+    address,
+    filters = [],
+  }: {
+    readonly chainId: number;
+    readonly address?: Hex;
+    readonly filters?: readonly SyncFilter[];
+  },
+): Promise<{ blockNumber: bigint; logs: (StorageAdapterLog & { eventName: "Store_SetRecord" })[] }> {
+  const benchmark = createBenchmark("drizzleGetLogs");
+
+  const conditions = filters.length
+    ? filters.map((filter) =>
+        and(
+          address != null ? eq(tables.recordsTable.address, address) : undefined,
+          eq(tables.recordsTable.tableId, filter.tableId),
+          filter.key0 != null ? eq(tables.recordsTable.key0, filter.key0) : undefined,
+          filter.key1 != null ? eq(tables.recordsTable.key1, filter.key1) : undefined,
+        ),
+      )
+    : address != null
+      ? [eq(tables.recordsTable.address, address)]
+      : [];
+  benchmark("parse config");
+
+  // Query for the block number that the indexer (i.e. chain) is at, in case the
+  // indexer is further along in the chain than a given store/table's last updated
+  // block number. We'll then take the highest block number between the indexer's
+  // chain state and all the records in the query (in case the records updated
+  // between these queries). Using just the highest block number from the queries
+  // could potentially signal to the client an older-than-necessary block number,
+  // for stores/tables that haven't seen recent activity.
+  // TODO: move the block number query into the records query for atomicity so we don't have to merge them here
+  const chainState = await database
+    .select()
+    .from(tables.configTable)
+    .where(eq(tables.configTable.chainId, chainId))
+    .limit(1)
+    .execute()
+    // Get the first record in a way that returns a possible `undefined`
+    // TODO: move this to `.findFirst` after upgrading drizzle or `rows[0]` after enabling `noUncheckedIndexedAccess: true`
+    .then((rows) => rows.find(() => true));
+  const indexerBlockNumber = chainState?.blockNumber ?? 0n;
+  benchmark("query chainState");
+
+  const records = await database
+    .select()
+    .from(tables.recordsTable)
+    .where(or(...conditions))
+    .orderBy(
+      asc(tables.recordsTable.blockNumber),
+      // TODO: add logIndex (https://github.com/latticexyz/mud/issues/1979)
+    );
+  benchmark("query records");
+
+  const blockNumber = records.reduce((max, record) => bigIntMax(max, record.blockNumber ?? 0n), indexerBlockNumber);
+  benchmark("find block number");
+
+  const logs = records
+    // TODO: add this to the query, assuming we can optimize with an index
+    .filter((record) => !record.isDeleted)
+    .map(recordToLog);
+  benchmark("map records to logs");
+
+  return { blockNumber, logs };
+}

package/src/postgres/queryLogs.ts

@@ -0,0 +1,71 @@
+import { isNotNull } from "@latticexyz/common/utils";
+import { PendingQuery, Row, Sql } from "postgres";
+import { hexToBytes } from "viem";
+import { z } from "zod";
+import { input } from "@latticexyz/store-sync/indexer-client";
+import { transformSchemaName } from "@latticexyz/store-sync/postgres";
+import { Record } from "./common";
+
+const schemaName = transformSchemaName("mud");
+
+function and(sql: Sql, conditions: PendingQuery<Row[]>[]): PendingQuery<Row[]> {
+  return sql`(${conditions.reduce((query, condition) => sql`${query} AND ${condition}`)})`;
+}
+
+function or(sql: Sql, conditions: PendingQuery<Row[]>[]): PendingQuery<Row[]> {
+  return sql`(${conditions.reduce((query, condition) => sql`${query} OR ${condition}`)})`;
+}
+
+export function queryLogs(sql: Sql, opts: z.infer<typeof input>): PendingQuery<Record[]> {
+  const conditions = opts.filters.length
+    ? opts.filters.map((filter) =>
+        and(
+          sql,
+          [
+            opts.address != null ? sql`address = ${hexToBytes(opts.address)}` : null,
+            sql`table_id = ${hexToBytes(filter.tableId)}`,
+            filter.key0 != null ? sql`key0 = ${hexToBytes(filter.key0)}` : null,
+            filter.key1 != null ? sql`key1 = ${hexToBytes(filter.key1)}` : null,
+          ].filter(isNotNull),
+        ),
+      )
+    : opts.address != null
+      ? [sql`address = ${hexToBytes(opts.address)}`]
+      : [];
+
+  const where = sql`WHERE ${and(
+    sql,
+    [sql`is_deleted != true`, conditions.length ? or(sql, conditions) : null].filter(isNotNull),
+  )}`;
+
+  // TODO: implement bytea <> hex columns via custom types: https://github.com/porsager/postgres#custom-types
+  return sql<Record[]>`
+    WITH
+      config AS (
+        SELECT
+          version AS "indexerVersion",
+          chain_id AS "chainId",
+          block_number AS "chainBlockNumber"
+        FROM ${sql(`${schemaName}.config`)}
+        LIMIT 1
+      ),
+      records AS (
+        SELECT
+          '0x' || encode(address, 'hex') AS address,
+          '0x' || encode(table_id, 'hex') AS "tableId",
+          '0x' || encode(key_bytes, 'hex') AS "keyBytes",
+          '0x' || encode(static_data, 'hex') AS "staticData",
+          '0x' || encode(encoded_lengths, 'hex') AS "encodedLengths",
+          '0x' || encode(dynamic_data, 'hex') AS "dynamicData",
+          block_number AS "recordBlockNumber",
+          log_index AS "logIndex"
+        FROM ${sql(`${schemaName}.records`)}
+        ${where}
+        ORDER BY block_number, log_index ASC
+      )
+    SELECT
+      (SELECT COUNT(*) FROM records) AS "totalRows",
+      *
+    FROM config, records
+  `;
+}

package/src/postgres/recordToLog.ts

@@ -0,0 +1,19 @@
+import { StorageAdapterLog } from "@latticexyz/store-sync";
+import { decodeDynamicField } from "@latticexyz/protocol-parser/internal";
+import { RecordData } from "./common";
+
+export function recordToLog(
+  record: Omit<RecordData, "recordBlockNumber">,
+): StorageAdapterLog & { eventName: "Store_SetRecord" } {
+  return {
+    address: record.address,
+    eventName: "Store_SetRecord",
+    args: {
+      tableId: record.tableId,
+      keyTuple: decodeDynamicField("bytes32[]", record.keyBytes),
+      staticData: record.staticData ?? "0x",
+      encodedLengths: record.encodedLengths ?? "0x",
+      dynamicData: record.dynamicData ?? "0x",
+    },
+  } as const;
+}

package/src/sqlite/apiRoutes.ts

@@ -0,0 +1,48 @@
+import { Middleware } from "koa";
+import Router from "@koa/router";
+import compose from "koa-compose";
+import { input } from "@latticexyz/store-sync/indexer-client";
+import { storeTables, tablesWithRecordsToLogs } from "@latticexyz/store-sync";
+import { debug } from "../debug";
+import { createBenchmark } from "@latticexyz/common";
+import { compress } from "../koa-middleware/compress";
+import { getTablesWithRecords } from "./getTablesWithRecords";
+import { BaseSQLiteDatabase } from "drizzle-orm/sqlite-core";
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export function apiRoutes(database: BaseSQLiteDatabase<"sync", any>): Middleware {
+  const router = new Router();
+
+  router.get("/api/logs", compress(), async (ctx) => {
+    const benchmark = createBenchmark("sqlite:logs");
+
+    let options: ReturnType<typeof input.parse>;
+
+    try {
+      options = input.parse(typeof ctx.query.input === "string" ? JSON.parse(ctx.query.input) : {});
+    } catch (error) {
+      ctx.status = 400;
+      ctx.body = JSON.stringify(error);
+      debug(error);
+      return;
+    }
+
+    try {
+      options.filters = options.filters.length > 0 ? [...options.filters, { tableId: storeTables.Tables.tableId }] : [];
+      benchmark("parse config");
+      const { blockNumber, tables } = getTablesWithRecords(database, options);
+      benchmark("query tables with records");
+      const logs = tablesWithRecordsToLogs(tables);
+      benchmark("convert records to logs");
+
+      ctx.body = JSON.stringify({ blockNumber: blockNumber?.toString() ?? "-1", logs });
+      ctx.status = 200;
+    } catch (error) {
+      ctx.status = 500;
+      ctx.body = JSON.stringify(error);
+      debug(error);
+    }
+  });
+
+  return compose([router.routes(), router.allowedMethods()]) as Middleware;
+}

package/src/sqlite/createQueryAdapter.ts

@@ -1,8 +1,7 @@
-import { eq } from "drizzle-orm";
 import { BaseSQLiteDatabase } from "drizzle-orm/sqlite-core";
-import { buildTable, chainState, getTables } from "@latticexyz/store-sync/sqlite";
 import { QueryAdapter } from "@latticexyz/store-sync/trpc-indexer";
-import { debug } from "../debug";
+import { getTablesWithRecords } from "./getTablesWithRecords";
+import { tablesWithRecordsToLogs } from "@latticexyz/store-sync";
 
 /**
  * Creates a storage adapter for the tRPC server/client to query data from SQLite.
@@ -10,34 +9,16 @@ import { debug } from "../debug";
  * @param {BaseSQLiteDatabase<"sync", any>} database SQLite database object from Drizzle
  * @returns {Promise<QueryAdapter>} A set of methods used by tRPC endpoints.
  */
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
 export async function createQueryAdapter(database: BaseSQLiteDatabase<"sync", any>): Promise<QueryAdapter> {
   const adapter: QueryAdapter = {
-    async findAll(chainId, address) {
-      const tables = getTables(database).filter((table) => table.address === address);
-
-      const tablesWithRecords = tables.map((table) => {
-        const sqliteTable = buildTable(table);
-        const records = database.select().from(sqliteTable).where(eq(sqliteTable.__isDeleted, false)).all();
-        return {
-          ...table,
-          records: records.map((record) => ({
-            key: Object.fromEntries(Object.entries(table.keySchema).map(([name]) => [name, record[name]])),
-            value: Object.fromEntries(Object.entries(table.valueSchema).map(([name]) => [name, record[name]])),
-          })),
-        };
-      });
-
-      const metadata = database.select().from(chainState).where(eq(chainState.chainId, chainId)).all();
-      const { lastUpdatedBlockNumber } = metadata[0] ?? {};
-
-      const result = {
-        blockNumber: lastUpdatedBlockNumber ?? null,
-        tables: tablesWithRecords,
-      };
-
-      debug("findAll", chainId, address, result);
-
-      return result;
+    async getLogs(opts) {
+      const { blockNumber, tables } = getTablesWithRecords(database, opts);
+      const logs = tablesWithRecordsToLogs(tables);
+      return { blockNumber: blockNumber ?? 0n, logs };
+    },
+    async findAll(opts) {
+      return getTablesWithRecords(database, opts);
     },
   };
   return adapter;