@latticexyz/store-indexer 2.0.0-next.1 → 2.0.0-next.11

package/README.md

@@ -0,0 +1,53 @@
+# store-indexer
+
+A minimal Typescript indexer for [MUD Store](https://mud.dev/store) events (built on [store-sync](https://npmjs.com/package/@latticexyz/store-sync))
+
+## Usage
+
+Install and run with:
+
+```sh
+npm install @latticexyz/store-indexer
+
+npm sqlite-indexer
+# or
+npm postgres-indexer
+```
+
+or execute the one of the package bins directly:
+
+```sh
+npx -p @latticexyz/store-indexer sqlite-indexer
+# or
+npx -p @latticexyz/store-indexer postgres-indexer
+```
+
+## Configuration
+
+Each indexer can be configured with environment variables.
+
+### Common environment variables
+
+| Variable           | Description                                                | Default   |
+| ------------------ | ---------------------------------------------------------- | --------- |
+| `HOST`             | Host that the indexer server listens on                    | `0.0.0.0` |
+| `PORT`             | Port that the indexer server listens on                    | `3001`    |
+| `RPC_HTTP_URL`     | HTTP URL for Ethereum RPC to fetch data from               |           |
+| `RPC_WS_URL`       | WebSocket URL for Ethereum RPC to fetch data from          |           |
+| `START_BLOCK`      | Block number to start indexing from                        | `0`       |
+| `MAX_BLOCK_RANGE`  | Maximum number of blocks to fetch from the RPC per request | `1000`    |
+| `POLLING_INTERVAL` | How often to poll for new blocks (in milliseconds)         | `1000`    |
+
+Note that you only need one of `RPC_HTTP_URL` or `RPC_WS_URL`, but we recommend both. The WebSocket URL will be prioritized and fall back to the HTTP URL if there are any connection issues.
+
+### Postgres indexer environment variables
+
+| Variable       | Description             | Default |
+| -------------- | ----------------------- | ------- |
+| `DATABASE_URL` | Postgres connection URL |         |
+
+### SQLite indexer environment variables
+
+| Variable          | Description              | Default      |
+| ----------------- | ------------------------ | ------------ |
+| `SQLITE_FILENAME` | SQLite database filename | `indexer.db` |

package/dist/bin/postgres-indexer.js

@@ -0,0 +1,3 @@
+#!/usr/bin/env node
+import{a as b,b as g}from"../chunk-X3OEYQLT.js";import"dotenv/config";import{z as R}from"zod";import{DefaultLogger as v,eq as x}from"drizzle-orm";import{createPublicClient as D,fallback as E,webSocket as I,http as j}from"viem";import Q from"fastify";import{fastifyTRPCPlugin as q}from"@trpc/server/adapters/fastify";import{createAppRouter as H}from"@latticexyz/store-sync/trpc-indexer";import{eq as h}from"drizzle-orm";import{buildTable as O,buildInternalTables as N,getTables as U}from"@latticexyz/store-sync/postgres";import{getAddress as A}from"viem";async function T(o){return{async findAll({chainId:l,address:i,tableIds:m=[]}){let _=(await U(o)).filter(r=>i==null||A(i)===A(r.address)).filter(r=>!m.length||m.includes(r.tableId)),k=await Promise.all(_.map(async r=>{let f=O(r),B=await o.select().from(f).where(h(f.__isDeleted,!1)).execute();return{...r,records:B.map(d=>({key:Object.fromEntries(Object.entries(r.keySchema).map(([a])=>[a,d[a]])),value:Object.fromEntries(Object.entries(r.valueSchema).map(([a])=>[a,d[a]]))}))}})),p=N(),C=await o.select().from(p.chain).where(h(p.chain.chainId,l)).execute(),{lastUpdatedBlockNumber:L}=C[0]??{},u={blockNumber:L??null,tables:k};return b("findAll",l,i,u),u}}}import{isDefined as V}from"@latticexyz/common/utils";import{combineLatest as $,filter as z,first as W}from"rxjs";import{drizzle as G}from"drizzle-orm/postgres-js";import K from"postgres";import{cleanDatabase as M,postgresStorage as X,schemaVersion as y}from"@latticexyz/store-sync/postgres";import{createStoreSync as F}from"@latticexyz/store-sync";var e=g(R.object({DATABASE_URL:R.string()})),J=[e.RPC_WS_URL?I(e.RPC_WS_URL):void 0,e.RPC_HTTP_URL?j(e.RPC_HTTP_URL):void 0].filter(V),s=D({transport:E(J),pollingInterval:e.POLLING_INTERVAL}),Y=await s.getChainId(),n=G(K(e.DATABASE_URL),{logger:new v}),{storageAdapter:Z,internalTables:w}=await X({database:n,publicClient:s}),P=e.START_BLOCK;try{let t=(await n.select().from(w.chain).where(x(w.chain.chainId,Y)).execute())[0];t!=null&&(t.schemaVersion!=y?(console.log("schema version changed from",t.schemaVersion,"to",y,"cleaning database"),await M(n)):t.lastUpdatedBlockNumber!=null&&(console.log("resuming from block number",t.lastUpdatedBlockNumber+1n),P=t.lastUpdatedBlockNumber+1n))}catch{}var{latestBlockNumber$:ee,storedBlockLogs$:S}=await F({storageAdapter:Z,publicClient:s,startBlock:P,maxBlockRange:e.MAX_BLOCK_RANGE});S.subscribe();$([ee,S]).pipe(z(([o,{blockNumber:t}])=>o===t),W()).subscribe(()=>{console.log("all caught up")});var c=Q({maxParamLength:5e3});await c.register(import("@fastify/cors"));c.register(q,{prefix:"/trpc",trpcOptions:{router:H(),createContext:async()=>({queryAdapter:await T(n)})}});await c.listen({host:e.HOST,port:e.PORT});console.log(`indexer server listening on http://${e.HOST}:${e.PORT}`);
+//# sourceMappingURL=postgres-indexer.js.map

package/dist/bin/postgres-indexer.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../bin/postgres-indexer.ts","../../src/postgres/createQueryAdapter.ts"],"sourcesContent":["#!/usr/bin/env node\nimport \"dotenv/config\";\nimport { z } from \"zod\";\nimport { DefaultLogger, eq } from \"drizzle-orm\";\nimport { createPublicClient, fallback, webSocket, http, Transport } from \"viem\";\nimport fastify from \"fastify\";\nimport { fastifyTRPCPlugin } from \"@trpc/server/adapters/fastify\";\nimport { AppRouter, createAppRouter } from \"@latticexyz/store-sync/trpc-indexer\";\nimport { createQueryAdapter } from \"../src/postgres/createQueryAdapter\";\nimport { isDefined } from \"@latticexyz/common/utils\";\nimport { combineLatest, filter, first } from \"rxjs\";\nimport { drizzle } from \"drizzle-orm/postgres-js\";\nimport postgres from \"postgres\";\nimport { cleanDatabase, postgresStorage, schemaVersion } from \"@latticexyz/store-sync/postgres\";\nimport { createStoreSync } from \"@latticexyz/store-sync\";\nimport { parseEnv } from \"./parseEnv\";\n\nconst env = parseEnv(\n  z.object({\n    DATABASE_URL: z.string(),\n  })\n);\n\nconst transports: Transport[] = [\n  // prefer WS when specified\n  env.RPC_WS_URL ? webSocket(env.RPC_WS_URL) : undefined,\n  // otherwise use or fallback to HTTP\n  env.RPC_HTTP_URL ? http(env.RPC_HTTP_URL) : undefined,\n].filter(isDefined);\n\nconst publicClient = createPublicClient({\n  transport: fallback(transports),\n  pollingInterval: env.POLLING_INTERVAL,\n});\n\nconst chainId = await publicClient.getChainId();\nconst database = drizzle(postgres(env.DATABASE_URL), {\n  logger: new DefaultLogger(),\n});\n\nconst { storageAdapter, internalTables } = await postgresStorage({ database, publicClient });\n\nlet startBlock = env.START_BLOCK;\n\n// Resume from latest block stored in DB. This will throw if the DB doesn't exist yet, so we wrap in a try/catch and ignore the error.\ntry {\n  const currentChainStates = await database\n    .select()\n    .from(internalTables.chain)\n    .where(eq(internalTables.chain.chainId, chainId))\n    .execute();\n  // TODO: replace this type workaround with `noUncheckedIndexedAccess: true` when we can fix all the issues related (https://github.com/latticexyz/mud/issues/1212)\n  const currentChainState: (typeof currentChainStates)[number] | undefined = currentChainStates[0];\n\n  if (currentChainState != null) {\n    if (currentChainState.schemaVersion != schemaVersion) {\n      console.log(\n        \"schema version changed from\",\n        currentChainState.schemaVersion,\n        \"to\",\n        schemaVersion,\n        \"cleaning database\"\n      );\n      await cleanDatabase(database);\n    } else if (currentChainState.lastUpdatedBlockNumber != null) {\n      console.log(\"resuming from block number\", currentChainState.lastUpdatedBlockNumber + 1n);\n      startBlock = currentChainState.lastUpdatedBlockNumber + 1n;\n    }\n  }\n} catch (error) {\n  // ignore errors, this is optional\n}\n\nconst { latestBlockNumber$, storedBlockLogs$ } = await createStoreSync({\n  storageAdapter,\n  publicClient,\n  startBlock,\n  maxBlockRange: env.MAX_BLOCK_RANGE,\n});\n\nstoredBlockLogs$.subscribe();\n\ncombineLatest([latestBlockNumber$, storedBlockLogs$])\n  .pipe(\n    filter(\n      ([latestBlockNumber, { blockNumber: lastBlockNumberProcessed }]) => latestBlockNumber === lastBlockNumberProcessed\n    ),\n    first()\n  )\n  .subscribe(() => {\n    console.log(\"all caught up\");\n  });\n\n// @see https://fastify.dev/docs/latest/\nconst server = fastify({\n  maxParamLength: 5000,\n});\n\nawait server.register(import(\"@fastify/cors\"));\n\n// @see https://trpc.io/docs/server/adapters/fastify\nserver.register(fastifyTRPCPlugin<AppRouter>, {\n  prefix: \"/trpc\",\n  trpcOptions: {\n    router: createAppRouter(),\n    createContext: async () => ({\n      queryAdapter: await createQueryAdapter(database),\n    }),\n  },\n});\n\nawait server.listen({ host: env.HOST, port: env.PORT });\nconsole.log(`indexer server listening on http://${env.HOST}:${env.PORT}`);\n","import { eq } from \"drizzle-orm\";\nimport { PgDatabase } from \"drizzle-orm/pg-core\";\nimport { buildTable, buildInternalTables, getTables } from \"@latticexyz/store-sync/postgres\";\nimport { QueryAdapter } from \"@latticexyz/store-sync/trpc-indexer\";\nimport { debug } from \"../debug\";\nimport { getAddress } from \"viem\";\n\n/**\n * Creates a query adapter for the tRPC server/client to query data from Postgres.\n *\n * @param {PgDatabase<any>} database Postgres database object from Drizzle\n * @returns {Promise<QueryAdapter>} A set of methods used by tRPC endpoints.\n */\nexport async function createQueryAdapter(database: PgDatabase<any>): Promise<QueryAdapter> {\n  const adapter: QueryAdapter = {\n    async findAll({ chainId, address, tableIds = [] }) {\n      const tables = (await getTables(database))\n        .filter((table) => address == null || getAddress(address) === getAddress(table.address))\n        .filter((table) => !tableIds.length || tableIds.includes(table.tableId));\n\n      const tablesWithRecords = await Promise.all(\n        tables.map(async (table) => {\n          const sqliteTable = buildTable(table);\n          const records = await database.select().from(sqliteTable).where(eq(sqliteTable.__isDeleted, false)).execute();\n          return {\n            ...table,\n            records: records.map((record) => ({\n              key: Object.fromEntries(Object.entries(table.keySchema).map(([name]) => [name, record[name]])),\n              value: Object.fromEntries(Object.entries(table.valueSchema).map(([name]) => [name, record[name]])),\n            })),\n          };\n        })\n      );\n\n      const internalTables = buildInternalTables();\n      const metadata = await database\n        .select()\n        .from(internalTables.chain)\n        .where(eq(internalTables.chain.chainId, chainId))\n        .execute();\n      const { lastUpdatedBlockNumber } = metadata[0] ?? {};\n\n      const result = {\n        blockNumber: lastUpdatedBlockNumber ?? null,\n        tables: tablesWithRecords,\n      };\n\n      debug(\"findAll\", chainId, address, result);\n\n      return result;\n    },\n  };\n  return adapter;\n}\n"],"mappings":";gDACA,MAAO,gBACP,OAAS,KAAAA,MAAS,MAClB,OAAS,iBAAAC,EAAe,MAAAC,MAAU,cAClC,OAAS,sBAAAC,EAAoB,YAAAC,EAAU,aAAAC,EAAW,QAAAC,MAAuB,OACzE,OAAOC,MAAa,UACpB,OAAS,qBAAAC,MAAyB,gCAClC,OAAoB,mBAAAC,MAAuB,sCCP3C,OAAS,MAAAC,MAAU,cAEnB,OAAS,cAAAC,EAAY,uBAAAC,EAAqB,aAAAC,MAAiB,kCAG3D,OAAS,cAAAC,MAAkB,OAQ3B,eAAsBC,EAAmBC,EAAkD,CAuCzF,MAtC8B,CAC5B,MAAM,QAAQ,CAAE,QAAAC,EAAS,QAAAC,EAAS,SAAAC,EAAW,CAAC,CAAE,EAAG,CACjD,IAAMC,GAAU,MAAMC,EAAUL,CAAQ,GACrC,OAAQM,GAAUJ,GAAW,MAAQJ,EAAWI,CAAO,IAAMJ,EAAWQ,EAAM,OAAO,CAAC,EACtF,OAAQA,GAAU,CAACH,EAAS,QAAUA,EAAS,SAASG,EAAM,OAAO,CAAC,EAEnEC,EAAoB,MAAM,QAAQ,IACtCH,EAAO,IAAI,MAAOE,GAAU,CAC1B,IAAME,EAAcC,EAAWH,CAAK,EAC9BI,EAAU,MAAMV,EAAS,OAAO,EAAE,KAAKQ,CAAW,EAAE,MAAMG,EAAGH,EAAY,YAAa,EAAK,CAAC,EAAE,QAAQ,EAC5G,MAAO,CACL,GAAGF,EACH,QAASI,EAAQ,IAAKE,IAAY,CAChC,IAAK,OAAO,YAAY,OAAO,QAAQN,EAAM,SAAS,EAAE,IAAI,CAAC,CAACO,CAAI,IAAM,CAACA,EAAMD,EAAOC,CAAI,CAAC,CAAC,CAAC,EAC7F,MAAO,OAAO,YAAY,OAAO,QAAQP,EAAM,WAAW,EAAE,IAAI,CAAC,CAACO,CAAI,IAAM,CAACA,EAAMD,EAAOC,CAAI,CAAC,CAAC,CAAC,CACnG,EAAE,CACJ,CACF,CAAC,CACH,EAEMC,EAAiBC,EAAoB,EACrCC,EAAW,MAAMhB,EACpB,OAAO,EACP,KAAKc,EAAe,KAAK,EACzB,MAAMH,EAAGG,EAAe,MAAM,QAASb,CAAO,CAAC,EAC/C,QAAQ,EACL,CAAE,uBAAAgB,CAAuB,EAAID,EAAS,CAAC,GAAK,CAAC,EAE7CE,EAAS,CACb,YAAaD,GAA0B,KACvC,OAAQV,CACV,EAEA,OAAAY,EAAM,UAAWlB,EAASC,EAASgB,CAAM,EAElCA,CACT,CACF,CAEF,CD5CA,OAAS,aAAAE,MAAiB,2BAC1B,OAAS,iBAAAC,EAAe,UAAAC,EAAQ,SAAAC,MAAa,OAC7C,OAAS,WAAAC,MAAe,0BACxB,OAAOC,MAAc,WACrB,OAAS,iBAAAC,EAAe,mBAAAC,EAAiB,iBAAAC,MAAqB,kCAC9D,OAAS,mBAAAC,MAAuB,yBAGhC,IAAMC,EAAMC,EACVC,EAAE,OAAO,CACP,aAAcA,EAAE,OAAO,CACzB,CAAC,CACH,EAEMC,EAA0B,CAE9BH,EAAI,WAAaI,EAAUJ,EAAI,UAAU,EAAI,OAE7CA,EAAI,aAAeK,EAAKL,EAAI,YAAY,EAAI,MAC9C,EAAE,OAAOM,CAAS,EAEZC,EAAeC,EAAmB,CACtC,UAAWC,EAASN,CAAU,EAC9B,gBAAiBH,EAAI,gBACvB,CAAC,EAEKU,EAAU,MAAMH,EAAa,WAAW,EACxCI,EAAWC,EAAQC,EAASb,EAAI,YAAY,EAAG,CACnD,OAAQ,IAAIc,CACd,CAAC,EAEK,CAAE,eAAAC,EAAgB,eAAAC,CAAe,EAAI,MAAMC,EAAgB,CAAE,SAAAN,EAAU,aAAAJ,CAAa,CAAC,EAEvFW,EAAalB,EAAI,YAGrB,GAAI,CAOF,IAAMmB,GANqB,MAAMR,EAC9B,OAAO,EACP,KAAKK,EAAe,KAAK,EACzB,MAAMI,EAAGJ,EAAe,MAAM,QAASN,CAAO,CAAC,EAC/C,QAAQ,GAEmF,CAAC,EAE3FS,GAAqB,OACnBA,EAAkB,eAAiBE,GACrC,QAAQ,IACN,8BACAF,EAAkB,cAClB,KACAE,EACA,mBACF,EACA,MAAMC,EAAcX,CAAQ,GACnBQ,EAAkB,wBAA0B,OACrD,QAAQ,IAAI,6BAA8BA,EAAkB,uBAAyB,EAAE,EACvFD,EAAaC,EAAkB,uBAAyB,IAG9D,MAAE,CAEF,CAEA,GAAM,CAAE,mBAAAI,GAAoB,iBAAAC,CAAiB,EAAI,MAAMC,EAAgB,CACrE,eAAAV,EACA,aAAAR,EACA,WAAAW,EACA,cAAelB,EAAI,eACrB,CAAC,EAEDwB,EAAiB,UAAU,EAE3BE,EAAc,CAACH,GAAoBC,CAAgB,CAAC,EACjD,KACCG,EACE,CAAC,CAACC,EAAmB,CAAE,YAAaC,CAAyB,CAAC,IAAMD,IAAsBC,CAC5F,EACAC,EAAM,CACR,EACC,UAAU,IAAM,CACf,QAAQ,IAAI,eAAe,CAC7B,CAAC,EAGH,IAAMC,EAASC,EAAQ,CACrB,eAAgB,GAClB,CAAC,EAED,MAAMD,EAAO,SAAS,OAAO,eAAe,CAAC,EAG7CA,EAAO,SAASE,EAA8B,CAC5C,OAAQ,QACR,YAAa,CACX,OAAQC,EAAgB,EACxB,cAAe,UAAa,CAC1B,aAAc,MAAMC,EAAmBxB,CAAQ,CACjD,EACF,CACF,CAAC,EAED,MAAMoB,EAAO,OAAO,CAAE,KAAM/B,EAAI,KAAM,KAAMA,EAAI,IAAK,CAAC,EACtD,QAAQ,IAAI,sCAAsCA,EAAI,QAAQA,EAAI,MAAM","names":["z","DefaultLogger","eq","createPublicClient","fallback","webSocket","http","fastify","fastifyTRPCPlugin","createAppRouter","eq","buildTable","buildInternalTables","getTables","getAddress","createQueryAdapter","database","chainId","address","tableIds","tables","getTables","table","tablesWithRecords","sqliteTable","buildTable","records","eq","record","name","internalTables","buildInternalTables","metadata","lastUpdatedBlockNumber","result","debug","isDefined","combineLatest","filter","first","drizzle","postgres","cleanDatabase","postgresStorage","schemaVersion","createStoreSync","env","parseEnv","z","transports","webSocket","http","isDefined","publicClient","createPublicClient","fallback","chainId","database","drizzle","postgres","DefaultLogger","storageAdapter","internalTables","postgresStorage","startBlock","currentChainState","eq","schemaVersion","cleanDatabase","latestBlockNumber$","storedBlockLogs$","createStoreSync","combineLatest","filter","latestBlockNumber","lastBlockNumberProcessed","first","server","fastify","fastifyTRPCPlugin","createAppRouter","createQueryAdapter"]}

package/dist/bin/sqlite-indexer.js

@@ -0,0 +1,3 @@
+#!/usr/bin/env node
+import{a as u,b as d}from"../chunk-X3OEYQLT.js";import"dotenv/config";import B from"node:fs";import{z as T}from"zod";import{eq as I}from"drizzle-orm";import{drizzle as O}from"drizzle-orm/better-sqlite3";import w from"better-sqlite3";import{createPublicClient as Q,fallback as v,webSocket as U,http as x}from"viem";import j from"fastify";import{fastifyTRPCPlugin as q}from"@trpc/server/adapters/fastify";import{createAppRouter as D}from"@latticexyz/store-sync/trpc-indexer";import{chainState as L,schemaVersion as y,syncToSqlite as H}from"@latticexyz/store-sync/sqlite";import{eq as b}from"drizzle-orm";import{buildTable as P,chainState as h,getTables as N}from"@latticexyz/store-sync/sqlite";import{getAddress as g}from"viem";async function S(o){return{async findAll({chainId:c,address:n,tableIds:l=[]}){let _=N(o).filter(r=>n==null||g(n)===g(r.address)).filter(r=>!l.length||l.includes(r.tableId)).map(r=>{let p=P(r),E=o.select().from(p).where(b(p.__isDeleted,!1)).all();return{...r,records:E.map(f=>({key:Object.fromEntries(Object.entries(r.keySchema).map(([a])=>[a,f[a]])),value:Object.fromEntries(Object.entries(r.valueSchema).map(([a])=>[a,f[a]]))}))}}),k=o.select().from(h).where(b(h.chainId,c)).all(),{lastUpdatedBlockNumber:C}=k[0]??{},m={blockNumber:C??null,tables:_};return u("findAll",c,n,m),m}}}import{isDefined as M}from"@latticexyz/common/utils";import{combineLatest as V,filter as $,first as z}from"rxjs";var e=d(T.object({SQLITE_FILENAME:T.string().default("indexer.db")})),F=[e.RPC_WS_URL?U(e.RPC_WS_URL):void 0,e.RPC_HTTP_URL?x(e.RPC_HTTP_URL):void 0].filter(M),A=Q({transport:v(F),pollingInterval:e.POLLING_INTERVAL}),W=await A.getChainId(),s=O(new w(e.SQLITE_FILENAME)),R=e.START_BLOCK;try{let t=s.select().from(L).where(I(L.chainId,W)).all()[0];t!=null&&(t.schemaVersion!=y?(console.log("schema version changed from",t.schemaVersion,"to",y,"recreating database"),B.truncateSync(e.SQLITE_FILENAME)):t.lastUpdatedBlockNumber!=null&&(console.log("resuming from block number",t.lastUpdatedBlockNumber+1n),R=t.lastUpdatedBlockNumber+1n))}catch{}var{latestBlockNumber$:G,storedBlockLogs$:K}=await H({database:s,publicClient:A,startBlock:R,maxBlockRange:e.MAX_BLOCK_RANGE});V([G,K]).pipe($(([o,{blockNumber:t}])=>o===t),z()).subscribe(()=>{console.log("all caught up")});var i=j({maxParamLength:5e3});await i.register(import("@fastify/cors"));i.register(q,{prefix:"/trpc",trpcOptions:{router:D(),createContext:async()=>({queryAdapter:await S(s)})}});await i.listen({host:e.HOST,port:e.PORT});console.log(`indexer server listening on http://${e.HOST}:${e.PORT}`);
+//# sourceMappingURL=sqlite-indexer.js.map

package/dist/bin/sqlite-indexer.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../bin/sqlite-indexer.ts","../../src/sqlite/createQueryAdapter.ts"],"sourcesContent":["#!/usr/bin/env node\nimport \"dotenv/config\";\nimport fs from \"node:fs\";\nimport { z } from \"zod\";\nimport { eq } from \"drizzle-orm\";\nimport { drizzle } from \"drizzle-orm/better-sqlite3\";\nimport Database from \"better-sqlite3\";\nimport { createPublicClient, fallback, webSocket, http, Transport } from \"viem\";\nimport fastify from \"fastify\";\nimport { fastifyTRPCPlugin } from \"@trpc/server/adapters/fastify\";\nimport { AppRouter, createAppRouter } from \"@latticexyz/store-sync/trpc-indexer\";\nimport { chainState, schemaVersion, syncToSqlite } from \"@latticexyz/store-sync/sqlite\";\nimport { createQueryAdapter } from \"../src/sqlite/createQueryAdapter\";\nimport { isDefined } from \"@latticexyz/common/utils\";\nimport { combineLatest, filter, first } from \"rxjs\";\nimport { parseEnv } from \"./parseEnv\";\n\nconst env = parseEnv(\n  z.object({\n    SQLITE_FILENAME: z.string().default(\"indexer.db\"),\n  })\n);\n\nconst transports: Transport[] = [\n  // prefer WS when specified\n  env.RPC_WS_URL ? webSocket(env.RPC_WS_URL) : undefined,\n  // otherwise use or fallback to HTTP\n  env.RPC_HTTP_URL ? http(env.RPC_HTTP_URL) : undefined,\n].filter(isDefined);\n\nconst publicClient = createPublicClient({\n  transport: fallback(transports),\n  pollingInterval: env.POLLING_INTERVAL,\n});\n\nconst chainId = await publicClient.getChainId();\nconst database = drizzle(new Database(env.SQLITE_FILENAME));\n\nlet startBlock = env.START_BLOCK;\n\n// Resume from latest block stored in DB. This will throw if the DB doesn't exist yet, so we wrap in a try/catch and ignore the error.\ntry {\n  const currentChainStates = database.select().from(chainState).where(eq(chainState.chainId, chainId)).all();\n  // TODO: replace this type workaround with `noUncheckedIndexedAccess: true` when we can fix all the issues related (https://github.com/latticexyz/mud/issues/1212)\n  const currentChainState: (typeof currentChainStates)[number] | undefined = currentChainStates[0];\n\n  if (currentChainState != null) {\n    if (currentChainState.schemaVersion != schemaVersion) {\n      console.log(\n        \"schema version changed from\",\n        currentChainState.schemaVersion,\n        \"to\",\n        schemaVersion,\n        \"recreating database\"\n      );\n      fs.truncateSync(env.SQLITE_FILENAME);\n    } else if (currentChainState.lastUpdatedBlockNumber != null) {\n      console.log(\"resuming from block number\", currentChainState.lastUpdatedBlockNumber + 1n);\n      startBlock = currentChainState.lastUpdatedBlockNumber + 1n;\n    }\n  }\n} catch (error) {\n  // ignore errors, this is optional\n}\n\nconst { latestBlockNumber$, storedBlockLogs$ } = await syncToSqlite({\n  database,\n  publicClient,\n  startBlock,\n  maxBlockRange: env.MAX_BLOCK_RANGE,\n});\n\ncombineLatest([latestBlockNumber$, storedBlockLogs$])\n  .pipe(\n    filter(\n      ([latestBlockNumber, { blockNumber: lastBlockNumberProcessed }]) => latestBlockNumber === lastBlockNumberProcessed\n    ),\n    first()\n  )\n  .subscribe(() => {\n    console.log(\"all caught up\");\n  });\n\n// @see https://fastify.dev/docs/latest/\nconst server = fastify({\n  maxParamLength: 5000,\n});\n\nawait server.register(import(\"@fastify/cors\"));\n\n// @see https://trpc.io/docs/server/adapters/fastify\nserver.register(fastifyTRPCPlugin<AppRouter>, {\n  prefix: \"/trpc\",\n  trpcOptions: {\n    router: createAppRouter(),\n    createContext: async () => ({\n      queryAdapter: await createQueryAdapter(database),\n    }),\n  },\n});\n\nawait server.listen({ host: env.HOST, port: env.PORT });\nconsole.log(`indexer server listening on http://${env.HOST}:${env.PORT}`);\n","import { eq } from \"drizzle-orm\";\nimport { BaseSQLiteDatabase } from \"drizzle-orm/sqlite-core\";\nimport { buildTable, chainState, getTables } from \"@latticexyz/store-sync/sqlite\";\nimport { QueryAdapter } from \"@latticexyz/store-sync/trpc-indexer\";\nimport { debug } from \"../debug\";\nimport { getAddress } from \"viem\";\n\n/**\n * Creates a storage adapter for the tRPC server/client to query data from SQLite.\n *\n * @param {BaseSQLiteDatabase<\"sync\", any>} database SQLite database object from Drizzle\n * @returns {Promise<QueryAdapter>} A set of methods used by tRPC endpoints.\n */\nexport async function createQueryAdapter(database: BaseSQLiteDatabase<\"sync\", any>): Promise<QueryAdapter> {\n  const adapter: QueryAdapter = {\n    async findAll({ chainId, address, tableIds = [] }) {\n      const tables = getTables(database)\n        .filter((table) => address == null || getAddress(address) === getAddress(table.address))\n        .filter((table) => !tableIds.length || tableIds.includes(table.tableId));\n\n      const tablesWithRecords = tables.map((table) => {\n        const sqliteTable = buildTable(table);\n        const records = database.select().from(sqliteTable).where(eq(sqliteTable.__isDeleted, false)).all();\n        return {\n          ...table,\n          records: records.map((record) => ({\n            key: Object.fromEntries(Object.entries(table.keySchema).map(([name]) => [name, record[name]])),\n            value: Object.fromEntries(Object.entries(table.valueSchema).map(([name]) => [name, record[name]])),\n          })),\n        };\n      });\n\n      const metadata = database.select().from(chainState).where(eq(chainState.chainId, chainId)).all();\n      const { lastUpdatedBlockNumber } = metadata[0] ?? {};\n\n      const result = {\n        blockNumber: lastUpdatedBlockNumber ?? null,\n        tables: tablesWithRecords,\n      };\n\n      debug(\"findAll\", chainId, address, result);\n\n      return result;\n    },\n  };\n  return adapter;\n}\n"],"mappings":";gDACA,MAAO,gBACP,OAAOA,MAAQ,UACf,OAAS,KAAAC,MAAS,MAClB,OAAS,MAAAC,MAAU,cACnB,OAAS,WAAAC,MAAe,6BACxB,OAAOC,MAAc,iBACrB,OAAS,sBAAAC,EAAoB,YAAAC,EAAU,aAAAC,EAAW,QAAAC,MAAuB,OACzE,OAAOC,MAAa,UACpB,OAAS,qBAAAC,MAAyB,gCAClC,OAAoB,mBAAAC,MAAuB,sCAC3C,OAAS,cAAAC,EAAY,iBAAAC,EAAe,gBAAAC,MAAoB,gCCXxD,OAAS,MAAAC,MAAU,cAEnB,OAAS,cAAAC,EAAY,cAAAC,EAAY,aAAAC,MAAiB,gCAGlD,OAAS,cAAAC,MAAkB,OAQ3B,eAAsBC,EAAmBC,EAAkE,CAgCzG,MA/B8B,CAC5B,MAAM,QAAQ,CAAE,QAAAC,EAAS,QAAAC,EAAS,SAAAC,EAAW,CAAC,CAAE,EAAG,CAKjD,IAAMC,EAJSC,EAAUL,CAAQ,EAC9B,OAAQM,GAAUJ,GAAW,MAAQJ,EAAWI,CAAO,IAAMJ,EAAWQ,EAAM,OAAO,CAAC,EACtF,OAAQA,GAAU,CAACH,EAAS,QAAUA,EAAS,SAASG,EAAM,OAAO,CAAC,EAExC,IAAKA,GAAU,CAC9C,IAAMC,EAAcC,EAAWF,CAAK,EAC9BG,EAAUT,EAAS,OAAO,EAAE,KAAKO,CAAW,EAAE,MAAMG,EAAGH,EAAY,YAAa,EAAK,CAAC,EAAE,IAAI,EAClG,MAAO,CACL,GAAGD,EACH,QAASG,EAAQ,IAAKE,IAAY,CAChC,IAAK,OAAO,YAAY,OAAO,QAAQL,EAAM,SAAS,EAAE,IAAI,CAAC,CAACM,CAAI,IAAM,CAACA,EAAMD,EAAOC,CAAI,CAAC,CAAC,CAAC,EAC7F,MAAO,OAAO,YAAY,OAAO,QAAQN,EAAM,WAAW,EAAE,IAAI,CAAC,CAACM,CAAI,IAAM,CAACA,EAAMD,EAAOC,CAAI,CAAC,CAAC,CAAC,CACnG,EAAE,CACJ,CACF,CAAC,EAEKC,EAAWb,EAAS,OAAO,EAAE,KAAKc,CAAU,EAAE,MAAMJ,EAAGI,EAAW,QAASb,CAAO,CAAC,EAAE,IAAI,EACzF,CAAE,uBAAAc,CAAuB,EAAIF,EAAS,CAAC,GAAK,CAAC,EAE7CG,EAAS,CACb,YAAaD,GAA0B,KACvC,OAAQX,CACV,EAEA,OAAAa,EAAM,UAAWhB,EAASC,EAASc,CAAM,EAElCA,CACT,CACF,CAEF,CDjCA,OAAS,aAAAE,MAAiB,2BAC1B,OAAS,iBAAAC,EAAe,UAAAC,EAAQ,SAAAC,MAAa,OAG7C,IAAMC,EAAMC,EACVC,EAAE,OAAO,CACP,gBAAiBA,EAAE,OAAO,EAAE,QAAQ,YAAY,CAClD,CAAC,CACH,EAEMC,EAA0B,CAE9BH,EAAI,WAAaI,EAAUJ,EAAI,UAAU,EAAI,OAE7CA,EAAI,aAAeK,EAAKL,EAAI,YAAY,EAAI,MAC9C,EAAE,OAAOM,CAAS,EAEZC,EAAeC,EAAmB,CACtC,UAAWC,EAASN,CAAU,EAC9B,gBAAiBH,EAAI,gBACvB,CAAC,EAEKU,EAAU,MAAMH,EAAa,WAAW,EACxCI,EAAWC,EAAQ,IAAIC,EAASb,EAAI,eAAe,CAAC,EAEtDc,EAAad,EAAI,YAGrB,GAAI,CAGF,IAAMe,EAFqBJ,EAAS,OAAO,EAAE,KAAKK,CAAU,EAAE,MAAMC,EAAGD,EAAW,QAASN,CAAO,CAAC,EAAE,IAAI,EAEX,CAAC,EAE3FK,GAAqB,OACnBA,EAAkB,eAAiBG,GACrC,QAAQ,IACN,8BACAH,EAAkB,cAClB,KACAG,EACA,qBACF,EACAC,EAAG,aAAanB,EAAI,eAAe,GAC1Be,EAAkB,wBAA0B,OACrD,QAAQ,IAAI,6BAA8BA,EAAkB,uBAAyB,EAAE,EACvFD,EAAaC,EAAkB,uBAAyB,IAG9D,MAAE,CAEF,CAEA,GAAM,CAAE,mBAAAK,EAAoB,iBAAAC,CAAiB,EAAI,MAAMC,EAAa,CAClE,SAAAX,EACA,aAAAJ,EACA,WAAAO,EACA,cAAed,EAAI,eACrB,CAAC,EAEDuB,EAAc,CAACH,EAAoBC,CAAgB,CAAC,EACjD,KACCG,EACE,CAAC,CAACC,EAAmB,CAAE,YAAaC,CAAyB,CAAC,IAAMD,IAAsBC,CAC5F,EACAC,EAAM,CACR,EACC,UAAU,IAAM,CACf,QAAQ,IAAI,eAAe,CAC7B,CAAC,EAGH,IAAMC,EAASC,EAAQ,CACrB,eAAgB,GAClB,CAAC,EAED,MAAMD,EAAO,SAAS,OAAO,eAAe,CAAC,EAG7CA,EAAO,SAASE,EAA8B,CAC5C,OAAQ,QACR,YAAa,CACX,OAAQC,EAAgB,EACxB,cAAe,UAAa,CAC1B,aAAc,MAAMC,EAAmBrB,CAAQ,CACjD,EACF,CACF,CAAC,EAED,MAAMiB,EAAO,OAAO,CAAE,KAAM5B,EAAI,KAAM,KAAMA,EAAI,IAAK,CAAC,EACtD,QAAQ,IAAI,sCAAsCA,EAAI,QAAQA,EAAI,MAAM","names":["fs","z","eq","drizzle","Database","createPublicClient","fallback","webSocket","http","fastify","fastifyTRPCPlugin","createAppRouter","chainState","schemaVersion","syncToSqlite","eq","buildTable","chainState","getTables","getAddress","createQueryAdapter","database","chainId","address","tableIds","tablesWithRecords","getTables","table","sqliteTable","buildTable","records","eq","record","name","metadata","chainState","lastUpdatedBlockNumber","result","debug","isDefined","combineLatest","filter","first","env","parseEnv","z","transports","webSocket","http","isDefined","publicClient","createPublicClient","fallback","chainId","database","drizzle","Database","startBlock","currentChainState","chainState","eq","schemaVersion","fs","latestBlockNumber$","storedBlockLogs$","syncToSqlite","combineLatest","filter","latestBlockNumber","lastBlockNumberProcessed","first","server","fastify","fastifyTRPCPlugin","createAppRouter","createQueryAdapter"]}

package/dist/chunk-X3OEYQLT.js

@@ -0,0 +1,7 @@
+import{z as e,ZodError as c}from"zod";var t=e.intersection(e.object({HOST:e.string().default("0.0.0.0"),PORT:e.coerce.number().positive().default(3001),START_BLOCK:e.coerce.bigint().nonnegative().default(0n),MAX_BLOCK_RANGE:e.coerce.bigint().positive().default(1000n),POLLING_INTERVAL:e.coerce.number().positive().default(1e3)}),e.union([e.object({RPC_HTTP_URL:e.string(),RPC_WS_URL:e.string().optional()}),e.object({RPC_HTTP_URL:e.string().optional(),RPC_WS_URL:e.string()})]));function f(o){let r=o!==void 0?e.intersection(t,o):t;try{return r.parse(process.env)}catch(n){if(n instanceof c){let{_errors:a,...i}=n.format();console.error(`
+Missing or invalid environment variables:
+
+  ${Object.keys(i).join(`
+  `)}
+`),process.exit(1)}throw n}}import s from"debug";var _=s("mud:store-indexer");export{_ as a,f as b};
+//# sourceMappingURL=chunk-X3OEYQLT.js.map

package/dist/chunk-X3OEYQLT.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../bin/parseEnv.ts","../src/debug.ts"],"sourcesContent":["import { z, ZodError, ZodIntersection, ZodTypeAny } from \"zod\";\n\nconst commonSchema = z.intersection(\n  z.object({\n    HOST: z.string().default(\"0.0.0.0\"),\n    PORT: z.coerce.number().positive().default(3001),\n    START_BLOCK: z.coerce.bigint().nonnegative().default(0n),\n    MAX_BLOCK_RANGE: z.coerce.bigint().positive().default(1000n),\n    POLLING_INTERVAL: z.coerce.number().positive().default(1000),\n  }),\n  z.union([\n    z.object({\n      RPC_HTTP_URL: z.string(),\n      RPC_WS_URL: z.string().optional(),\n    }),\n    z.object({\n      RPC_HTTP_URL: z.string().optional(),\n      RPC_WS_URL: z.string(),\n    }),\n  ])\n);\n\nexport function parseEnv<TSchema extends ZodTypeAny | undefined = undefined>(\n  schema?: TSchema\n): z.infer<TSchema extends ZodTypeAny ? ZodIntersection<typeof commonSchema, TSchema> : typeof commonSchema> {\n  const envSchema = schema !== undefined ? z.intersection(commonSchema, schema) : commonSchema;\n  try {\n    return envSchema.parse(process.env);\n  } catch (error) {\n    if (error instanceof ZodError) {\n      const { _errors, ...invalidEnvVars } = error.format();\n      console.error(`\\nMissing or invalid environment variables:\\n\\n  ${Object.keys(invalidEnvVars).join(\"\\n  \")}\\n`);\n      process.exit(1);\n    }\n    throw error;\n  }\n}\n","import createDebug from \"debug\";\n\nexport const debug = createDebug(\"mud:store-indexer\");\n"],"mappings":"AAAA,OAAS,KAAAA,EAAG,YAAAC,MAA6C,MAEzD,IAAMC,EAAeF,EAAE,aACrBA,EAAE,OAAO,CACP,KAAMA,EAAE,OAAO,EAAE,QAAQ,SAAS,EAClC,KAAMA,EAAE,OAAO,OAAO,EAAE,SAAS,EAAE,QAAQ,IAAI,EAC/C,YAAaA,EAAE,OAAO,OAAO,EAAE,YAAY,EAAE,QAAQ,EAAE,EACvD,gBAAiBA,EAAE,OAAO,OAAO,EAAE,SAAS,EAAE,QAAQ,KAAK,EAC3D,iBAAkBA,EAAE,OAAO,OAAO,EAAE,SAAS,EAAE,QAAQ,GAAI,CAC7D,CAAC,EACDA,EAAE,MAAM,CACNA,EAAE,OAAO,CACP,aAAcA,EAAE,OAAO,EACvB,WAAYA,EAAE,OAAO,EAAE,SAAS,CAClC,CAAC,EACDA,EAAE,OAAO,CACP,aAAcA,EAAE,OAAO,EAAE,SAAS,EAClC,WAAYA,EAAE,OAAO,CACvB,CAAC,CACH,CAAC,CACH,EAEO,SAASG,EACdC,EAC2G,CAC3G,IAAMC,EAAYD,IAAW,OAAYJ,EAAE,aAAaE,EAAcE,CAAM,EAAIF,EAChF,GAAI,CACF,OAAOG,EAAU,MAAM,QAAQ,GAAG,CACpC,OAASC,EAAP,CACA,GAAIA,aAAiBL,EAAU,CAC7B,GAAM,CAAE,QAAAM,EAAS,GAAGC,CAAe,EAAIF,EAAM,OAAO,EACpD,QAAQ,MAAM;AAAA;AAAA;AAAA,IAAoD,OAAO,KAAKE,CAAc,EAAE,KAAK;AAAA,GAAM;AAAA,CAAK,EAC9G,QAAQ,KAAK,CAAC,EAEhB,MAAMF,CACR,CACF,CCpCA,OAAOG,MAAiB,QAEjB,IAAMC,EAAQD,EAAY,mBAAmB","names":["z","ZodError","commonSchema","parseEnv","schema","envSchema","error","_errors","invalidEnvVars","createDebug","debug"]}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@latticexyz/store-indexer",
-  "version": "2.0.0-next.1",
+  "version": "2.0.0-next.11",
   "description": "Minimal Typescript indexer for Store",
   "repository": {
     "type": "git",
@@ -13,22 +13,29 @@
     ".": "./dist/index.js"
   },
   "types": "src/index.ts",
+  "bin": {
+    "postgres-indexer": "./dist/bin/postgres-indexer.js",
+    "sqlite-indexer": "./dist/bin/sqlite-indexer.js"
+  },
   "dependencies": {
+    "@fastify/cors": "^8.3.0",
     "@trpc/client": "10.34.0",
     "@trpc/server": "10.34.0",
     "@wagmi/chains": "^0.2.22",
-    "better-sqlite3": "^8.4.0",
-    "cors": "^2.8.5",
+    "better-sqlite3": "^8.6.0",
     "debug": "^4.3.4",
-    "drizzle-orm": "^0.27.0",
+    "dotenv": "^16.0.3",
+    "drizzle-orm": "^0.28.5",
+    "fastify": "^4.21.0",
+    "postgres": "^3.3.5",
     "rxjs": "7.5.5",
     "superjson": "^1.12.4",
-    "viem": "1.3.1",
+    "viem": "1.14.0",
     "zod": "^3.21.4",
-    "@latticexyz/block-logs-stream": "2.0.0-next.1",
-    "@latticexyz/common": "2.0.0-next.1",
-    "@latticexyz/store": "2.0.0-next.1",
-    "@latticexyz/store-sync": "2.0.0-next.1"
+    "@latticexyz/block-logs-stream": "2.0.0-next.11",
+    "@latticexyz/common": "2.0.0-next.11",
+    "@latticexyz/store": "2.0.0-next.11",
+    "@latticexyz/store-sync": "2.0.0-next.11"
   },
   "devDependencies": {
     "@types/better-sqlite3": "^7.6.4",
@@ -49,10 +56,13 @@
     "clean:js": "rimraf dist",
     "dev": "tsup --watch",
     "lint": "eslint .",
-    "start": "tsx bin/sqlite-indexer",
-    "start:local": "SQLITE_FILENAME=anvil.db CHAIN_ID=31337 pnpm start",
-    "start:testnet": "SQLITE_FILENAME=testnet.db CHAIN_ID=4242 START_BLOCK=19037160 pnpm start",
-    "start:testnet2": "SQLITE_FILENAME=testnet2.db CHAIN_ID=4243 pnpm start",
-    "test": "tsc --noEmit --skipLibCheck"
+    "start:postgres": "tsx bin/postgres-indexer",
+    "start:postgres:local": "DEBUG=mud:store-sync:createStoreSync DATABASE_URL=postgres://127.0.0.1/postgres RPC_HTTP_URL=http://127.0.0.1:8545 pnpm start:postgres",
+    "start:postgres:testnet": "DEBUG=mud:store-sync:createStoreSync DATABASE_URL=postgres://127.0.0.1/postgres RPC_HTTP_URL=https://follower.testnet-chain.linfra.xyz pnpm start:postgres",
+    "start:sqlite": "tsx bin/sqlite-indexer",
+    "start:sqlite:local": "DEBUG=mud:store-sync:createStoreSync SQLITE_FILENAME=anvil.db RPC_HTTP_URL=http://127.0.0.1:8545 pnpm start:sqlite",
+    "start:sqlite:testnet": "DEBUG=mud:store-sync:createStoreSync SQLITE_FILENAME=testnet.db RPC_HTTP_URL=https://follower.testnet-chain.linfra.xyz pnpm start:sqlite",
+    "test": "tsc --noEmit --skipLibCheck",
+    "test:ci": "pnpm run test"
   }
 }

package/src/postgres/createQueryAdapter.ts

@@ -0,0 +1,54 @@
+import { eq } from "drizzle-orm";
+import { PgDatabase } from "drizzle-orm/pg-core";
+import { buildTable, buildInternalTables, getTables } from "@latticexyz/store-sync/postgres";
+import { QueryAdapter } from "@latticexyz/store-sync/trpc-indexer";
+import { debug } from "../debug";
+import { getAddress } from "viem";
+
+/**
+ * Creates a query adapter for the tRPC server/client to query data from Postgres.
+ *
+ * @param {PgDatabase<any>} database Postgres database object from Drizzle
+ * @returns {Promise<QueryAdapter>} A set of methods used by tRPC endpoints.
+ */
+export async function createQueryAdapter(database: PgDatabase<any>): Promise<QueryAdapter> {
+  const adapter: QueryAdapter = {
+    async findAll({ chainId, address, tableIds = [] }) {
+      const tables = (await getTables(database))
+        .filter((table) => address == null || getAddress(address) === getAddress(table.address))
+        .filter((table) => !tableIds.length || tableIds.includes(table.tableId));
+
+      const tablesWithRecords = await Promise.all(
+        tables.map(async (table) => {
+          const sqliteTable = buildTable(table);
+          const records = await database.select().from(sqliteTable).where(eq(sqliteTable.__isDeleted, false)).execute();
+          return {
+            ...table,
+            records: records.map((record) => ({
+              key: Object.fromEntries(Object.entries(table.keySchema).map(([name]) => [name, record[name]])),
+              value: Object.fromEntries(Object.entries(table.valueSchema).map(([name]) => [name, record[name]])),
+            })),
+          };
+        })
+      );
+
+      const internalTables = buildInternalTables();
+      const metadata = await database
+        .select()
+        .from(internalTables.chain)
+        .where(eq(internalTables.chain.chainId, chainId))
+        .execute();
+      const { lastUpdatedBlockNumber } = metadata[0] ?? {};
+
+      const result = {
+        blockNumber: lastUpdatedBlockNumber ?? null,
+        tables: tablesWithRecords,
+      };
+
+      debug("findAll", chainId, address, result);
+
+      return result;
+    },
+  };
+  return adapter;
+}

package/src/sqlite/{createStorageAdapter.ts → createQueryAdapter.ts}

@@ -1,22 +1,25 @@
 import { eq } from "drizzle-orm";
 import { BaseSQLiteDatabase } from "drizzle-orm/sqlite-core";
-import { createSqliteTable, chainState, getTables } from "@latticexyz/store-sync/sqlite";
-import { StorageAdapter } from "@latticexyz/store-sync/trpc-indexer";
+import { buildTable, chainState, getTables } from "@latticexyz/store-sync/sqlite";
+import { QueryAdapter } from "@latticexyz/store-sync/trpc-indexer";
 import { debug } from "../debug";
+import { getAddress } from "viem";
 
 /**
  * Creates a storage adapter for the tRPC server/client to query data from SQLite.
  *
  * @param {BaseSQLiteDatabase<"sync", any>} database SQLite database object from Drizzle
- * @returns {Promise<StorageAdapter>} A set of methods used by tRPC endpoints.
+ * @returns {Promise<QueryAdapter>} A set of methods used by tRPC endpoints.
  */
-export async function createStorageAdapter(database: BaseSQLiteDatabase<"sync", any>): Promise<StorageAdapter> {
-  const adapter: StorageAdapter = {
-    async findAll(chainId, address) {
-      const tables = getTables(database).filter((table) => table.address === address);
+export async function createQueryAdapter(database: BaseSQLiteDatabase<"sync", any>): Promise<QueryAdapter> {
+  const adapter: QueryAdapter = {
+    async findAll({ chainId, address, tableIds = [] }) {
+      const tables = getTables(database)
+        .filter((table) => address == null || getAddress(address) === getAddress(table.address))
+        .filter((table) => !tableIds.length || tableIds.includes(table.tableId));
 
       const tablesWithRecords = tables.map((table) => {
-        const sqliteTable = createSqliteTable(table);
+        const sqliteTable = buildTable(table);
         const records = database.select().from(sqliteTable).where(eq(sqliteTable.__isDeleted, false)).all();
         return {
           ...table,

package/src/sqlite/createIndexer.ts

@@ -1,92 +0,0 @@
-import { PublicClient } from "viem";
-import {
-  createBlockStream,
-  isNonPendingBlock,
-  blockRangeToLogs,
-  groupLogsByBlockNumber,
-} from "@latticexyz/block-logs-stream";
-import { concatMap, filter, from, map, mergeMap, tap } from "rxjs";
-import { storeEventsAbi } from "@latticexyz/store";
-import { blockLogsToStorage } from "@latticexyz/store-sync";
-import { sqliteStorage } from "@latticexyz/store-sync/sqlite";
-import { BaseSQLiteDatabase } from "drizzle-orm/sqlite-core";
-import { debug } from "../debug";
-
-type CreateIndexerOptions = {
-  /**
-   * [SQLite database object from Drizzle][0].
-   *
-   * [0]: https://orm.drizzle.team/docs/installation-and-db-connection/sqlite/better-sqlite3
-   */
-  database: BaseSQLiteDatabase<"sync", any>;
-  /**
-   * [viem `PublicClient`][0] used for fetching logs from the RPC.
-   *
-   * [0]: https://viem.sh/docs/clients/public.html
-   */
-  publicClient: PublicClient;
-  /**
-   * Optional block number to start indexing from. Useful for resuming the indexer from a particular point in time or starting after a particular contract deployment.
-   */
-  startBlock?: bigint;
-  /**
-   * Optional maximum block range, if your RPC limits the amount of blocks fetched at a time.
-   */
-  maxBlockRange?: bigint;
-};
-
-/**
- * Creates an indexer to process and store blockchain events.
- *
- * @param {CreateIndexerOptions} options See `CreateIndexerOptions`.
- * @returns A function to unsubscribe from the block stream, effectively stopping the indexer.
- */
-export async function createIndexer({
-  database,
-  publicClient,
-  startBlock = 0n,
-  maxBlockRange,
-}: CreateIndexerOptions): Promise<() => void> {
-  const latestBlock$ = createBlockStream({ publicClient, blockTag: "latest" });
-
-  const latestBlockNumber$ = latestBlock$.pipe(
-    filter(isNonPendingBlock),
-    map((block) => block.number)
-  );
-
-  let latestBlockNumber: bigint | null = null;
-  const blockLogs$ = latestBlockNumber$.pipe(
-    tap((blockNumber) => {
-      latestBlockNumber = blockNumber;
-      debug("latest block number", blockNumber);
-    }),
-    map((blockNumber) => ({ startBlock, endBlock: blockNumber })),
-    blockRangeToLogs({
-      publicClient,
-      events: storeEventsAbi,
-      maxBlockRange,
-    }),
-    tap(({ fromBlock, toBlock, logs }) => {
-      debug("found", logs.length, "logs for block", fromBlock, "-", toBlock);
-    }),
-    mergeMap(({ toBlock, logs }) => from(groupLogsByBlockNumber(logs, toBlock)))
-  );
-
-  let lastBlockNumberProcessed: bigint | null = null;
-  const sub = blockLogs$
-    .pipe(
-      concatMap(blockLogsToStorage(await sqliteStorage({ database, publicClient }))),
-      tap(({ blockNumber, operations }) => {
-        lastBlockNumberProcessed = blockNumber;
-        debug("stored", operations.length, "operations for block", blockNumber);
-        if (latestBlockNumber === lastBlockNumberProcessed) {
-          debug("all caught up");
-        }
-      })
-    )
-    .subscribe();
-
-  return () => {
-    sub.unsubscribe();
-  };
-}