@latticexyz/store-indexer 2.0.12-main-9be2bb86 → 2.0.12-main-e43c0938

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@latticexyz/store-indexer",
-  "version": "2.0.12-main-9be2bb86",
+  "version": "2.0.12-main-e43c0938",
   "description": "Minimal Typescript indexer for Store",
   "repository": {
     "type": "git",
@@ -12,13 +12,15 @@
   "exports": {
     ".": "./dist/index.js"
   },
-  "types": "src/index.ts",
   "bin": {
     "postgres-decoded-indexer": "./dist/bin/postgres-decoded-indexer.js",
     "postgres-frontend": "./dist/bin/postgres-frontend.js",
     "postgres-indexer": "./dist/bin/postgres-indexer.js",
     "sqlite-indexer": "./dist/bin/sqlite-indexer.js"
   },
+  "files": [
+    "dist"
+  ],
   "dependencies": {
     "@koa/cors": "^4.0.0",
     "@koa/router": "^12.0.1",
@@ -41,11 +43,11 @@
     "trpc-koa-adapter": "^1.1.3",
     "viem": "2.9.20",
     "zod": "^3.22.2",
-    "@latticexyz/block-logs-stream": "2.0.12-main-9be2bb86",
-    "@latticexyz/common": "2.0.12-main-9be2bb86",
-    "@latticexyz/protocol-parser": "2.0.12-main-9be2bb86",
-    "@latticexyz/store": "2.0.12-main-9be2bb86",
-    "@latticexyz/store-sync": "2.0.12-main-9be2bb86"
+    "@latticexyz/block-logs-stream": "2.0.12-main-e43c0938",
+    "@latticexyz/common": "2.0.12-main-e43c0938",
+    "@latticexyz/protocol-parser": "2.0.12-main-e43c0938",
+    "@latticexyz/store": "2.0.12-main-e43c0938",
+    "@latticexyz/store-sync": "2.0.12-main-e43c0938"
   },
   "devDependencies": {
     "@types/accepts": "^1.3.7",
@@ -79,7 +81,7 @@
     "start:sqlite": "tsx bin/sqlite-indexer",
     "start:sqlite:local": "SQLITE_FILENAME=anvil.db RPC_HTTP_URL=http://127.0.0.1:8545 pnpm start:sqlite",
     "start:sqlite:testnet": "SQLITE_FILENAME=testnet.db RPC_HTTP_URL=https://rpc.holesky.redstone.xyz pnpm start:sqlite",
-    "test": "tsc --noEmit --skipLibCheck",
+    "test": "tsc --noEmit",
     "test:ci": "pnpm run test"
   }
 }

package/src/debug.ts

@@ -1,10 +0,0 @@
-import createDebug from "debug";
-
-export const debug = createDebug("mud:store-indexer");
-export const error = createDebug("mud:store-indexer");
-
-// Pipe debug output to stdout instead of stderr
-debug.log = console.debug.bind(console);
-
-// Pipe error output to stderr
-error.log = console.error.bind(console);

package/src/index.ts

@@ -1 +0,0 @@
-// Nothing to export yet

package/src/koa-middleware/compress.ts

@@ -1,48 +0,0 @@
-import { Middleware } from "koa";
-import { Readable, Stream } from "node:stream";
-import accepts from "accepts";
-import { Zlib, createBrotliCompress, createDeflate, createGzip } from "node:zlib";
-import { includes } from "@latticexyz/common/utils";
-
-// Loosely based on https://github.com/holic/koa-compress/blob/master/lib/index.js
-// with better handling of streams better with occasional flushing
-
-const encodings = {
-  br: createBrotliCompress,
-  gzip: createGzip,
-  deflate: createDeflate,
-} as const;
-
-const encodingNames = Object.keys(encodings) as (keyof typeof encodings)[];
-
-function flushEvery<stream extends Zlib & Readable>(stream: stream, bytesThreshold: number): stream {
-  let bytesSinceFlush = 0;
-  stream.on("data", (data) => {
-    bytesSinceFlush += data.length;
-    if (bytesSinceFlush > bytesThreshold) {
-      bytesSinceFlush = 0;
-      stream.flush();
-    }
-  });
-  return stream;
-}
-
-type CompressOptions = {
-  flushThreshold?: number;
-};
-
-export function compress({ flushThreshold = 1024 * 4 }: CompressOptions = {}): Middleware {
-  return async function compressMiddleware(ctx, next) {
-    ctx.vary("Accept-Encoding");
-
-    await next();
-
-    const encoding = accepts(ctx.req).encoding(encodingNames);
-    if (!includes(encodingNames, encoding)) return;
-
-    const compressed = flushEvery(encodings[encoding](), flushThreshold);
-
-    ctx.set("Content-Encoding", encoding);
-    ctx.body = ctx.body instanceof Stream ? ctx.body.pipe(compressed) : compressed.end(ctx.body);
-  };
-}

package/src/koa-middleware/healthcheck.ts

@@ -1,37 +0,0 @@
-import { Middleware } from "koa";
-
-type HealthcheckOptions = {
-  isHealthy?: () => boolean;
-  isReady?: () => boolean;
-};
-
-/**
- * Middleware to add Kubernetes healthcheck endpoints
- */
-export function healthcheck({ isHealthy, isReady }: HealthcheckOptions = {}): Middleware {
-  return async function healthcheckMiddleware(ctx, next): Promise<void> {
-    if (ctx.path === "/healthz") {
-      if (isHealthy == null || isHealthy()) {
-        ctx.status = 200;
-        ctx.body = "healthy";
-      } else {
-        ctx.status = 503;
-        ctx.body = "not healthy";
-      }
-      return;
-    }
-
-    if (ctx.path === "/readyz") {
-      if (isReady == null || isReady()) {
-        ctx.status = 200;
-        ctx.body = "ready";
-      } else {
-        ctx.status = 503;
-        ctx.body = "not ready";
-      }
-      return;
-    }
-
-    await next();
-  };
-}

package/src/koa-middleware/helloWorld.ts

@@ -1,12 +0,0 @@
-import { Middleware } from "koa";
-
-export function helloWorld(): Middleware {
-  return async function helloWorldMiddleware(ctx, next): Promise<void> {
-    if (ctx.path === "/") {
-      ctx.status = 200;
-      ctx.body = "emit HelloWorld();";
-      return;
-    }
-    await next();
-  };
-}

package/src/koa-middleware/metrics.ts

@@ -1,85 +0,0 @@
-import { Middleware } from "koa";
-import promClient from "prom-client";
-
-type MetricsOptions = {
-  isHealthy?: () => boolean;
-  isReady?: () => boolean;
-  getLatestStoredBlockNumber?: () => Promise<bigint | undefined>;
-  getDistanceFromFollowBlock?: () => Promise<bigint>;
-  followBlockTag?: "latest" | "safe" | "finalized";
-};
-
-/**
- * Middleware to add Prometheus metrics endpoints
- */
-export function metrics({
-  isHealthy,
-  isReady,
-  getLatestStoredBlockNumber,
-  getDistanceFromFollowBlock,
-  followBlockTag,
-}: MetricsOptions = {}): Middleware {
-  promClient.collectDefaultMetrics();
-  if (isHealthy != null) {
-    new promClient.Gauge({
-      name: "health_status",
-      help: "Health status (0 = unhealthy, 1 = healthy)",
-      collect(): void {
-        this.set(Number(isHealthy()));
-      },
-    });
-  }
-
-  if (isReady != null) {
-    new promClient.Gauge({
-      name: "readiness_status",
-      help: "Readiness status (whether the service is ready to receive requests, 0 = not ready, 1 = ready)",
-      collect(): void {
-        this.set(Number(isReady()));
-      },
-    });
-  }
-
-  if (getLatestStoredBlockNumber != null) {
-    new promClient.Gauge({
-      name: "latest_stored_block_number",
-      help: "Latest block number stored in the database",
-      async collect(): Promise<void> {
-        this.set(Number(await getLatestStoredBlockNumber()));
-      },
-    });
-  }
-
-  if (followBlockTag != null) {
-    const blockTagGauge = new promClient.Gauge({
-      name: "follow_block_tag",
-      help: "Block tag the indexer is following (0 = finalized, 1 = safe, 2 = latest)",
-    });
-    const blockTagToValue = {
-      finalized: 0,
-      safe: 1,
-      latest: 2,
-    };
-    blockTagGauge.set(blockTagToValue[followBlockTag]);
-  }
-
-  if (getDistanceFromFollowBlock != null) {
-    new promClient.Gauge({
-      name: "distance_from_follow_block",
-      help: "Block distance from the block tag this the indexer is following",
-      async collect(): Promise<void> {
-        this.set(Number(await getDistanceFromFollowBlock()));
-      },
-    });
-  }
-
-  return async function metricsMiddleware(ctx, next): Promise<void> {
-    if (ctx.path === "/metrics") {
-      ctx.status = 200;
-      ctx.body = await promClient.register.metrics();
-      return;
-    }
-
-    await next();
-  };
-}

package/src/koa-middleware/sentry.ts

@@ -1,101 +0,0 @@
-import * as Sentry from "@sentry/node";
-import { ProfilingIntegration } from "@sentry/profiling-node";
-import { stripUrlQueryAndFragment } from "@sentry/utils";
-import debug from "debug";
-import Koa from "koa";
-import compose from "koa-compose";
-
-export function errorHandler(): Koa.Middleware {
-  return async function errorHandlerMiddleware(ctx, next) {
-    try {
-      await next();
-    } catch (err) {
-      Sentry.withScope((scope) => {
-        scope.addEventProcessor((event) => {
-          return Sentry.addRequestDataToEvent(event, ctx.request);
-        });
-        Sentry.captureException(err);
-      });
-      throw err;
-    }
-  };
-}
-
-export function requestHandler(): Koa.Middleware {
-  return async function requestHandlerMiddleware(ctx, next) {
-    await Sentry.runWithAsyncContext(async () => {
-      const hub = Sentry.getCurrentHub();
-      hub.configureScope((scope) =>
-        scope.addEventProcessor((event) =>
-          Sentry.addRequestDataToEvent(event, ctx.request, {
-            include: {
-              user: false,
-            },
-          }),
-        ),
-      );
-      await next();
-    });
-  };
-}
-
-export function tracing(): Koa.Middleware {
-  // creates a Sentry transaction per request
-  return async function tracingMiddleware(ctx, next) {
-    const reqMethod = (ctx.method || "").toUpperCase();
-    const reqUrl = ctx.url && stripUrlQueryAndFragment(ctx.url);
-
-    // Connect to trace of upstream app
-    let traceparentData;
-    if (ctx.request.get("sentry-trace")) {
-      traceparentData = Sentry.extractTraceparentData(ctx.request.get("sentry-trace"));
-    }
-
-    const transaction = Sentry.startTransaction({
-      name: `${reqMethod} ${reqUrl}`,
-      op: "http.server",
-      ...traceparentData,
-    });
-
-    ctx.__sentry_transaction = transaction;
-
-    // We put the transaction on the scope so users can attach children to it
-    Sentry.getCurrentHub().configureScope((scope) => {
-      scope.setSpan(transaction);
-    });
-
-    ctx.res.on("finish", () => {
-      // Push `transaction.finish` to the next event loop so open spans have a chance to finish before the transaction closes
-      setImmediate(() => {
-        // If you're using koa router, set the matched route as transaction name
-        if (ctx._matchedRoute) {
-          const mountPath = ctx.mountPath || "";
-          transaction.setName(`${reqMethod} ${mountPath}${ctx._matchedRoute}`);
-        }
-
-        transaction.setHttpStatus(ctx.status);
-        transaction.finish();
-      });
-    });
-
-    await next();
-  };
-}
-
-export function sentry(dsn: string): Koa.Middleware {
-  debug("Initializing Sentry");
-  Sentry.init({
-    dsn,
-    integrations: [
-      // Automatically instrument Node.js libraries and frameworks
-      ...Sentry.autoDiscoverNodePerformanceMonitoringIntegrations(),
-      new ProfilingIntegration(),
-    ],
-    // Performance Monitoring
-    tracesSampleRate: 1.0,
-    // Set sampling rate for profiling - this is relative to tracesSampleRate
-    profilesSampleRate: 1.0,
-  });
-
-  return compose([errorHandler(), requestHandler(), tracing()]);
-}

package/src/postgres/apiRoutes.ts

@@ -1,81 +0,0 @@
-import { Sql } from "postgres";
-import { Middleware } from "koa";
-import Router from "@koa/router";
-import compose from "koa-compose";
-import { input } from "@latticexyz/store-sync/indexer-client";
-import { storeTables } from "@latticexyz/store-sync";
-import { queryLogs } from "./queryLogs";
-import { recordToLog } from "./recordToLog";
-import { debug, error } from "../debug";
-import { createBenchmark } from "@latticexyz/common";
-import { compress } from "../koa-middleware/compress";
-
-export function apiRoutes(database: Sql): Middleware {
-  const router = new Router();
-
-  router.get("/api/logs", compress(), async (ctx) => {
-    const benchmark = createBenchmark("postgres:logs");
-    let options: ReturnType<typeof input.parse>;
-
-    try {
-      options = input.parse(typeof ctx.query.input === "string" ? JSON.parse(ctx.query.input) : {});
-    } catch (e) {
-      ctx.status = 400;
-      ctx.set("Content-Type", "application/json");
-      ctx.body = JSON.stringify(e);
-      debug(e);
-      return;
-    }
-
-    try {
-      options.filters = options.filters.length > 0 ? [...options.filters, { tableId: storeTables.Tables.tableId }] : [];
-      const records = await queryLogs(database, options ?? {}).execute();
-      benchmark("query records");
-      const logs = records.map(recordToLog);
-      benchmark("map records to logs");
-
-      // Ideally we would immediately return an error if the request is for a Store that the indexer
-      // is not configured to index. Since we don't have easy access to this information here,
-      // we return an error if there are no logs found for a given Store, since that would never
-      // be the case for a Store that is being indexed (since there would at least be records for the
-      // Tables table with tables created during Store initialization).
-      if (records.length === 0) {
-        ctx.status = 404;
-        ctx.body = "no logs found";
-        error(
-          `no logs found for chainId ${options.chainId}, address ${options.address}, filters ${JSON.stringify(
-            options.filters,
-          )}`,
-        );
-        return;
-      }
-
-      const blockNumber = records[0].chainBlockNumber;
-      ctx.status = 200;
-
-      // max age is set to several multiples of the uncached response time (currently ~10s, but using 60s for wiggle room) to ensure only ~one origin request at a time
-      // and stale-while-revalidate below means that the cache is refreshed under the hood while still responding fast (cached)
-      const maxAgeSeconds = 60 * 5;
-      // we set stale-while-revalidate to the time elapsed by the number of blocks we can fetch from the RPC in the same amount of time as an uncached response
-      // meaning it would take ~the same about of time to get an uncached response from the origin as it would to catch up from the currently cached response
-      // if an uncached response takes ~10 seconds, we have ~10s to catch up, so let's say we can do enough RPC calls to fetch 4000 blocks
-      // with a block per 2 seconds, that means we can serve a stale/cached response for 8000 seconds before we should require the response be returned by the origin
-      const staleWhileRevalidateSeconds = 4000 * 2;
-
-      ctx.set(
-        "Cache-Control",
-        `public, max-age=${maxAgeSeconds}, stale-while-revalidate=${staleWhileRevalidateSeconds}`,
-      );
-
-      ctx.set("Content-Type", "application/json");
-      ctx.body = JSON.stringify({ blockNumber, logs });
-    } catch (e) {
-      ctx.status = 500;
-      ctx.set("Content-Type", "application/json");
-      ctx.body = JSON.stringify(e);
-      error(e);
-    }
-  });
-
-  return compose([router.routes(), router.allowedMethods()]) as Middleware;
-}

package/src/postgres/common.ts

@@ -1,21 +0,0 @@
-import { Hex } from "viem";
-
-export type RecordData = {
-  address: Hex;
-  tableId: Hex;
-  keyBytes: Hex;
-  staticData: Hex | null;
-  encodedLengths: Hex | null;
-  dynamicData: Hex | null;
-  recordBlockNumber: string;
-  logIndex: number;
-};
-
-export type RecordMetadata = {
-  indexerVersion: string;
-  chainId: string;
-  chainBlockNumber: string;
-  totalRows: number;
-};
-
-export type Record = RecordData & RecordMetadata;

package/src/postgres/deprecated/createQueryAdapter.ts

@@ -1,57 +0,0 @@
-import { getAddress } from "viem";
-import { PgDatabase } from "drizzle-orm/pg-core";
-import { TableWithRecords, isTableRegistrationLog, logToTable, storeTables } from "@latticexyz/store-sync";
-import { decodeKey, decodeValueArgs } from "@latticexyz/protocol-parser/internal";
-import { QueryAdapter } from "@latticexyz/store-sync/trpc-indexer";
-import { debug } from "../../debug";
-import { getLogs } from "./getLogs";
-import { groupBy } from "@latticexyz/common/utils";
-
-/**
- * Creates a query adapter for the tRPC server/client to query data from Postgres.
- *
- * @param {PgDatabase<any>} database Postgres database object from Drizzle
- * @returns {Promise<QueryAdapter>} A set of methods used by tRPC endpoints.
- * @deprecated
- */
-// eslint-disable-next-line @typescript-eslint/no-explicit-any
-export async function createQueryAdapter(database: PgDatabase<any>): Promise<QueryAdapter> {
-  const adapter: QueryAdapter = {
-    async getLogs(opts) {
-      return getLogs(database, opts);
-    },
-    async findAll(opts) {
-      const filters = opts.filters ?? [];
-      const { blockNumber, logs } = await getLogs(database, {
-        ...opts,
-        // make sure we're always retrieving `store.Tables` table, so we can decode table values
-        filters: filters.length > 0 ? [...filters, { tableId: storeTables.Tables.tableId }] : [],
-      });
-
-      const tables = logs.filter(isTableRegistrationLog).map(logToTable);
-
-      const logsByTable = groupBy(logs, (log) => `${getAddress(log.address)}:${log.args.tableId}`);
-
-      const tablesWithRecords: TableWithRecords[] = tables.map((table) => {
-        const tableLogs = logsByTable.get(`${getAddress(table.address)}:${table.tableId}`) ?? [];
-        const records = tableLogs.map((log) => ({
-          key: decodeKey(table.keySchema, log.args.keyTuple),
-          value: decodeValueArgs(table.valueSchema, log.args),
-        }));
-
-        return {
-          ...table,
-          records,
-        };
-      });
-
-      debug("findAll: decoded %d logs across %d tables", logs.length, tables.length);
-
-      return {
-        blockNumber,
-        tables: tablesWithRecords,
-      };
-    },
-  };
-  return adapter;
-}

package/src/postgres/deprecated/getLogs.ts

@@ -1,82 +0,0 @@
-import { PgDatabase } from "drizzle-orm/pg-core";
-import { Hex } from "viem";
-import { StorageAdapterLog, SyncFilter } from "@latticexyz/store-sync";
-import { tables } from "@latticexyz/store-sync/postgres";
-import { and, asc, eq, or } from "drizzle-orm";
-import { bigIntMax } from "@latticexyz/common/utils";
-import { recordToLog } from "../recordToLog";
-import { createBenchmark } from "@latticexyz/common";
-
-/**
- * @deprecated
- */
-export async function getLogs(
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  database: PgDatabase<any>,
-  {
-    chainId,
-    address,
-    filters = [],
-  }: {
-    readonly chainId: number;
-    readonly address?: Hex;
-    readonly filters?: readonly SyncFilter[];
-  },
-): Promise<{ blockNumber: bigint; logs: (StorageAdapterLog & { eventName: "Store_SetRecord" })[] }> {
-  const benchmark = createBenchmark("drizzleGetLogs");
-
-  const conditions = filters.length
-    ? filters.map((filter) =>
-        and(
-          address != null ? eq(tables.recordsTable.address, address) : undefined,
-          eq(tables.recordsTable.tableId, filter.tableId),
-          filter.key0 != null ? eq(tables.recordsTable.key0, filter.key0) : undefined,
-          filter.key1 != null ? eq(tables.recordsTable.key1, filter.key1) : undefined,
-        ),
-      )
-    : address != null
-      ? [eq(tables.recordsTable.address, address)]
-      : [];
-  benchmark("parse config");
-
-  // Query for the block number that the indexer (i.e. chain) is at, in case the
-  // indexer is further along in the chain than a given store/table's last updated
-  // block number. We'll then take the highest block number between the indexer's
-  // chain state and all the records in the query (in case the records updated
-  // between these queries). Using just the highest block number from the queries
-  // could potentially signal to the client an older-than-necessary block number,
-  // for stores/tables that haven't seen recent activity.
-  // TODO: move the block number query into the records query for atomicity so we don't have to merge them here
-  const chainState = await database
-    .select()
-    .from(tables.configTable)
-    .where(eq(tables.configTable.chainId, chainId))
-    .limit(1)
-    .execute()
-    // Get the first record in a way that returns a possible `undefined`
-    // TODO: move this to `.findFirst` after upgrading drizzle or `rows[0]` after enabling `noUncheckedIndexedAccess: true`
-    .then((rows) => rows.find(() => true));
-  const indexerBlockNumber = chainState?.blockNumber ?? 0n;
-  benchmark("query chainState");
-
-  const records = await database
-    .select()
-    .from(tables.recordsTable)
-    .where(or(...conditions))
-    .orderBy(
-      asc(tables.recordsTable.blockNumber),
-      // TODO: add logIndex (https://github.com/latticexyz/mud/issues/1979)
-    );
-  benchmark("query records");
-
-  const blockNumber = records.reduce((max, record) => bigIntMax(max, record.blockNumber ?? 0n), indexerBlockNumber);
-  benchmark("find block number");
-
-  const logs = records
-    // TODO: add this to the query, assuming we can optimize with an index
-    .filter((record) => !record.isDeleted)
-    .map(recordToLog);
-  benchmark("map records to logs");
-
-  return { blockNumber, logs };
-}

package/src/postgres/queryLogs.ts

@@ -1,71 +0,0 @@
-import { isNotNull } from "@latticexyz/common/utils";
-import { PendingQuery, Row, Sql } from "postgres";
-import { hexToBytes } from "viem";
-import { z } from "zod";
-import { input } from "@latticexyz/store-sync/indexer-client";
-import { transformSchemaName } from "@latticexyz/store-sync/postgres";
-import { Record } from "./common";
-
-const schemaName = transformSchemaName("mud");
-
-function and(sql: Sql, conditions: PendingQuery<Row[]>[]): PendingQuery<Row[]> {
-  return sql`(${conditions.reduce((query, condition) => sql`${query} AND ${condition}`)})`;
-}
-
-function or(sql: Sql, conditions: PendingQuery<Row[]>[]): PendingQuery<Row[]> {
-  return sql`(${conditions.reduce((query, condition) => sql`${query} OR ${condition}`)})`;
-}
-
-export function queryLogs(sql: Sql, opts: z.infer<typeof input>): PendingQuery<Record[]> {
-  const conditions = opts.filters.length
-    ? opts.filters.map((filter) =>
-        and(
-          sql,
-          [
-            opts.address != null ? sql`address = ${hexToBytes(opts.address)}` : null,
-            sql`table_id = ${hexToBytes(filter.tableId)}`,
-            filter.key0 != null ? sql`key0 = ${hexToBytes(filter.key0)}` : null,
-            filter.key1 != null ? sql`key1 = ${hexToBytes(filter.key1)}` : null,
-          ].filter(isNotNull),
-        ),
-      )
-    : opts.address != null
-      ? [sql`address = ${hexToBytes(opts.address)}`]
-      : [];
-
-  const where = sql`WHERE ${and(
-    sql,
-    [sql`is_deleted != true`, conditions.length ? or(sql, conditions) : null].filter(isNotNull),
-  )}`;
-
-  // TODO: implement bytea <> hex columns via custom types: https://github.com/porsager/postgres#custom-types
-  return sql<Record[]>`
-    WITH
-      config AS (
-        SELECT
-          version AS "indexerVersion",
-          chain_id AS "chainId",
-          block_number AS "chainBlockNumber"
-        FROM ${sql(`${schemaName}.config`)}
-        LIMIT 1
-      ),
-      records AS (
-        SELECT
-          '0x' || encode(address, 'hex') AS address,
-          '0x' || encode(table_id, 'hex') AS "tableId",
-          '0x' || encode(key_bytes, 'hex') AS "keyBytes",
-          '0x' || encode(static_data, 'hex') AS "staticData",
-          '0x' || encode(encoded_lengths, 'hex') AS "encodedLengths",
-          '0x' || encode(dynamic_data, 'hex') AS "dynamicData",
-          block_number AS "recordBlockNumber",
-          log_index AS "logIndex"
-        FROM ${sql(`${schemaName}.records`)}
-        ${where}
-        ORDER BY block_number, log_index ASC
-      )
-    SELECT
-      (SELECT COUNT(*) FROM records) AS "totalRows",
-      *
-    FROM config, records
-  `;
-}

package/src/postgres/recordToLog.ts

@@ -1,19 +0,0 @@
-import { StorageAdapterLog } from "@latticexyz/store-sync";
-import { decodeDynamicField } from "@latticexyz/protocol-parser/internal";
-import { RecordData } from "./common";
-
-export function recordToLog(
-  record: Omit<RecordData, "recordBlockNumber">,
-): StorageAdapterLog & { eventName: "Store_SetRecord" } {
-  return {
-    address: record.address,
-    eventName: "Store_SetRecord",
-    args: {
-      tableId: record.tableId,
-      keyTuple: decodeDynamicField("bytes32[]", record.keyBytes),
-      staticData: record.staticData ?? "0x",
-      encodedLengths: record.encodedLengths ?? "0x",
-      dynamicData: record.dynamicData ?? "0x",
-    },
-  } as const;
-}

package/src/sqlite/apiRoutes.ts

@@ -1,48 +0,0 @@
-import { Middleware } from "koa";
-import Router from "@koa/router";
-import compose from "koa-compose";
-import { input } from "@latticexyz/store-sync/indexer-client";
-import { storeTables, tablesWithRecordsToLogs } from "@latticexyz/store-sync";
-import { debug } from "../debug";
-import { createBenchmark } from "@latticexyz/common";
-import { compress } from "../koa-middleware/compress";
-import { getTablesWithRecords } from "./getTablesWithRecords";
-import { BaseSQLiteDatabase } from "drizzle-orm/sqlite-core";
-
-// eslint-disable-next-line @typescript-eslint/no-explicit-any
-export function apiRoutes(database: BaseSQLiteDatabase<"sync", any>): Middleware {
-  const router = new Router();
-
-  router.get("/api/logs", compress(), async (ctx) => {
-    const benchmark = createBenchmark("sqlite:logs");
-
-    let options: ReturnType<typeof input.parse>;
-
-    try {
-      options = input.parse(typeof ctx.query.input === "string" ? JSON.parse(ctx.query.input) : {});
-    } catch (error) {
-      ctx.status = 400;
-      ctx.body = JSON.stringify(error);
-      debug(error);
-      return;
-    }
-
-    try {
-      options.filters = options.filters.length > 0 ? [...options.filters, { tableId: storeTables.Tables.tableId }] : [];
-      benchmark("parse config");
-      const { blockNumber, tables } = getTablesWithRecords(database, options);
-      benchmark("query tables with records");
-      const logs = tablesWithRecordsToLogs(tables);
-      benchmark("convert records to logs");
-
-      ctx.body = JSON.stringify({ blockNumber: blockNumber?.toString() ?? "-1", logs });
-      ctx.status = 200;
-    } catch (error) {
-      ctx.status = 500;
-      ctx.body = JSON.stringify(error);
-      debug(error);
-    }
-  });
-
-  return compose([router.routes(), router.allowedMethods()]) as Middleware;
-}

package/src/sqlite/createQueryAdapter.ts

@@ -1,25 +0,0 @@
-import { BaseSQLiteDatabase } from "drizzle-orm/sqlite-core";
-import { QueryAdapter } from "@latticexyz/store-sync/trpc-indexer";
-import { getTablesWithRecords } from "./getTablesWithRecords";
-import { tablesWithRecordsToLogs } from "@latticexyz/store-sync";
-
-/**
- * Creates a storage adapter for the tRPC server/client to query data from SQLite.
- *
- * @param {BaseSQLiteDatabase<"sync", any>} database SQLite database object from Drizzle
- * @returns {Promise<QueryAdapter>} A set of methods used by tRPC endpoints.
- */
-// eslint-disable-next-line @typescript-eslint/no-explicit-any
-export async function createQueryAdapter(database: BaseSQLiteDatabase<"sync", any>): Promise<QueryAdapter> {
-  const adapter: QueryAdapter = {
-    async getLogs(opts) {
-      const { blockNumber, tables } = getTablesWithRecords(database, opts);
-      const logs = tablesWithRecordsToLogs(tables);
-      return { blockNumber: blockNumber ?? 0n, logs };
-    },
-    async findAll(opts) {
-      return getTablesWithRecords(database, opts);
-    },
-  };
-  return adapter;
-}

package/src/sqlite/getTablesWithRecords.ts

@@ -1,76 +0,0 @@
-import { asc, eq } from "drizzle-orm";
-import { BaseSQLiteDatabase } from "drizzle-orm/sqlite-core";
-import { buildTable, chainState, getTables } from "@latticexyz/store-sync/sqlite";
-import { Hex, getAddress } from "viem";
-import { decodeDynamicField } from "@latticexyz/protocol-parser/internal";
-import { SyncFilter, TableWithRecords } from "@latticexyz/store-sync";
-
-// TODO: refactor sqlite and replace this with getLogs to match postgres (https://github.com/latticexyz/mud/issues/1970)
-
-/**
- * @deprecated
- * */
-export function getTablesWithRecords(
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  database: BaseSQLiteDatabase<"sync", any>,
-  {
-    chainId,
-    address,
-    filters = [],
-  }: {
-    readonly chainId: number;
-    readonly address?: Hex;
-    readonly filters?: readonly SyncFilter[];
-  },
-): { blockNumber: bigint | null; tables: readonly TableWithRecords[] } {
-  const metadata = database
-    .select()
-    .from(chainState)
-    .where(eq(chainState.chainId, chainId))
-    .limit(1)
-    .all()
-    .find(() => true);
-
-  // If _any_ filter has a table ID, this will filter down all data to just those tables. Which mean we can't yet mix table filters with key-only filters.
-  // TODO: improve this so we can express this in the query (need to be able to query data across tables more easily)
-  const tableIds = Array.from(new Set(filters.map((filter) => filter.tableId)));
-  const tables = getTables(database)
-    .filter((table) => address == null || getAddress(address) === getAddress(table.address))
-    .filter((table) => !tableIds.length || tableIds.includes(table.tableId));
-
-  const tablesWithRecords = tables.map((table) => {
-    const sqliteTable = buildTable(table);
-    const records = database
-      .select()
-      .from(sqliteTable)
-      .where(eq(sqliteTable.__isDeleted, false))
-      .orderBy(
-        asc(sqliteTable.__lastUpdatedBlockNumber),
-        // TODO: add logIndex (https://github.com/latticexyz/mud/issues/1979)
-      )
-      .all();
-    const filteredRecords = !filters.length
-      ? records
-      : records.filter((record) => {
-          const keyTuple = decodeDynamicField("bytes32[]", record.__key);
-          return filters.some(
-            (filter) =>
-              filter.tableId === table.tableId &&
-              (filter.key0 == null || filter.key0 === keyTuple[0]) &&
-              (filter.key1 == null || filter.key1 === keyTuple[1]),
-          );
-        });
-    return {
-      ...table,
-      records: filteredRecords.map((record) => ({
-        key: Object.fromEntries(Object.entries(table.keySchema).map(([name]) => [name, record[name]])),
-        value: Object.fromEntries(Object.entries(table.valueSchema).map(([name]) => [name, record[name]])),
-      })),
-    };
-  });
-
-  return {
-    blockNumber: metadata?.lastUpdatedBlockNumber ?? null,
-    tables: tablesWithRecords,
-  };
-}