@latticexyz/store-indexer 2.0.0-skystrife-playtest-19728914 → 2.0.0-snapshot-test-32d38619

package/README.md

@@ -0,0 +1,53 @@
+# store-indexer
+
+A minimal Typescript indexer for [MUD Store](https://mud.dev/store) events (built on [store-sync](https://npmjs.com/package/@latticexyz/store-sync))
+
+## Usage
+
+Install and run with:
+
+```sh
+npm install @latticexyz/store-indexer
+
+npm sqlite-indexer
+# or
+npm postgres-indexer
+```
+
+or execute the one of the package bins directly:
+
+```sh
+npx -p @latticexyz/store-indexer sqlite-indexer
+# or
+npx -p @latticexyz/store-indexer postgres-indexer
+```
+
+## Configuration
+
+Each indexer can be configured with environment variables.
+
+### Common environment variables
+
+| Variable           | Description                                                | Default   |
+| ------------------ | ---------------------------------------------------------- | --------- |
+| `HOST`             | Host that the indexer server listens on                    | `0.0.0.0` |
+| `PORT`             | Port that the indexer server listens on                    | `3001`    |
+| `RPC_HTTP_URL`     | HTTP URL for Ethereum RPC to fetch data from               |           |
+| `RPC_WS_URL`       | WebSocket URL for Ethereum RPC to fetch data from          |           |
+| `START_BLOCK`      | Block number to start indexing from                        | `0`       |
+| `MAX_BLOCK_RANGE`  | Maximum number of blocks to fetch from the RPC per request | `1000`    |
+| `POLLING_INTERVAL` | How often to poll for new blocks (in milliseconds)         | `1000`    |
+
+Note that you only need one of `RPC_HTTP_URL` or `RPC_WS_URL`, but we recommend both. The WebSocket URL will be prioritized and fall back to the HTTP URL if there are any connection issues.
+
+### Postgres indexer environment variables
+
+| Variable       | Description             | Default |
+| -------------- | ----------------------- | ------- |
+| `DATABASE_URL` | Postgres connection URL |         |
+
+### SQLite indexer environment variables
+
+| Variable          | Description              | Default      |
+| ----------------- | ------------------------ | ------------ |
+| `SQLITE_FILENAME` | SQLite database filename | `indexer.db` |

package/dist/bin/postgres-indexer.js

@@ -0,0 +1,3 @@
+#!/usr/bin/env node
+import{a as d,b}from"../chunk-X3OEYQLT.js";import"dotenv/config";import{z as T}from"zod";import{DefaultLogger as U,eq as v}from"drizzle-orm";import{createPublicClient as x,fallback as D,webSocket as E,http as I}from"viem";import j from"fastify";import{fastifyTRPCPlugin as Q}from"@trpc/server/adapters/fastify";import{createAppRouter as q}from"@latticexyz/store-sync/trpc-indexer";import{eq as g}from"drizzle-orm";import{buildTable as B,buildInternalTables as O,getTables as N}from"@latticexyz/store-sync/postgres";import{getAddress as h}from"viem";async function A(r){return{async findAll(l,s){let m=O(),S=(await N(r)).filter(o=>s!=null&&h(s)===h(o.address)),_=await Promise.all(S.map(async o=>{let u=B(o),L=await r.select().from(u).where(g(u.__isDeleted,!1)).execute();return{...o,records:L.map(f=>({key:Object.fromEntries(Object.entries(o.keySchema).map(([a])=>[a,f[a]])),value:Object.fromEntries(Object.entries(o.valueSchema).map(([a])=>[a,f[a]]))}))}})),k=await r.select().from(m.chain).where(g(m.chain.chainId,l)).execute(),{lastUpdatedBlockNumber:C}=k[0]??{},p={blockNumber:C??null,tables:_};return d("findAll",l,s,p),p}}}import{isDefined as H}from"@latticexyz/common/utils";import{combineLatest as V,filter as $,first as z}from"rxjs";import{drizzle as W}from"drizzle-orm/postgres-js";import G from"postgres";import{cleanDatabase as K,postgresStorage as M,schemaVersion as R}from"@latticexyz/store-sync/postgres";import{createStoreSync as X}from"@latticexyz/store-sync";var e=b(T.object({DATABASE_URL:T.string()})),F=[e.RPC_WS_URL?E(e.RPC_WS_URL):void 0,e.RPC_HTTP_URL?I(e.RPC_HTTP_URL):void 0].filter(H),i=x({transport:D(F),pollingInterval:e.POLLING_INTERVAL}),J=await i.getChainId(),n=W(G(e.DATABASE_URL),{logger:new U}),{storageAdapter:Y,internalTables:y}=await M({database:n,publicClient:i}),w=e.START_BLOCK;try{let t=(await n.select().from(y.chain).where(v(y.chain.chainId,J)).execute())[0];t!=null&&(t.schemaVersion!=R?(console.log("schema version changed from",t.schemaVersion,"to",R,"cleaning database"),await K(n)):t.lastUpdatedBlockNumber!=null&&(console.log("resuming from block number",t.lastUpdatedBlockNumber+1n),w=t.lastUpdatedBlockNumber+1n))}catch{}var{latestBlockNumber$:Z,storedBlockLogs$:P}=await X({storageAdapter:Y,publicClient:i,startBlock:w,maxBlockRange:e.MAX_BLOCK_RANGE});P.subscribe();V([Z,P]).pipe($(([r,{blockNumber:t}])=>r===t),z()).subscribe(()=>{console.log("all caught up")});var c=j({maxParamLength:5e3});await c.register(import("@fastify/cors"));c.register(Q,{prefix:"/trpc",trpcOptions:{router:q(),createContext:async()=>({queryAdapter:await A(n)})}});await c.listen({host:e.HOST,port:e.PORT});console.log(`indexer server listening on http://${e.HOST}:${e.PORT}`);
+//# sourceMappingURL=postgres-indexer.js.map

package/dist/bin/postgres-indexer.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../bin/postgres-indexer.ts","../../src/postgres/createQueryAdapter.ts"],"sourcesContent":["#!/usr/bin/env node\nimport \"dotenv/config\";\nimport { z } from \"zod\";\nimport { DefaultLogger, eq } from \"drizzle-orm\";\nimport { createPublicClient, fallback, webSocket, http, Transport } from \"viem\";\nimport fastify from \"fastify\";\nimport { fastifyTRPCPlugin } from \"@trpc/server/adapters/fastify\";\nimport { AppRouter, createAppRouter } from \"@latticexyz/store-sync/trpc-indexer\";\nimport { createQueryAdapter } from \"../src/postgres/createQueryAdapter\";\nimport { isDefined } from \"@latticexyz/common/utils\";\nimport { combineLatest, filter, first } from \"rxjs\";\nimport { drizzle } from \"drizzle-orm/postgres-js\";\nimport postgres from \"postgres\";\nimport { cleanDatabase, postgresStorage, schemaVersion } from \"@latticexyz/store-sync/postgres\";\nimport { createStoreSync } from \"@latticexyz/store-sync\";\nimport { parseEnv } from \"./parseEnv\";\n\nconst env = parseEnv(\n  z.object({\n    DATABASE_URL: z.string(),\n  })\n);\n\nconst transports: Transport[] = [\n  // prefer WS when specified\n  env.RPC_WS_URL ? webSocket(env.RPC_WS_URL) : undefined,\n  // otherwise use or fallback to HTTP\n  env.RPC_HTTP_URL ? http(env.RPC_HTTP_URL) : undefined,\n].filter(isDefined);\n\nconst publicClient = createPublicClient({\n  transport: fallback(transports),\n  pollingInterval: env.POLLING_INTERVAL,\n});\n\nconst chainId = await publicClient.getChainId();\nconst database = drizzle(postgres(env.DATABASE_URL), {\n  logger: new DefaultLogger(),\n});\n\nconst { storageAdapter, internalTables } = await postgresStorage({ database, publicClient });\n\nlet startBlock = env.START_BLOCK;\n\n// Resume from latest block stored in DB. This will throw if the DB doesn't exist yet, so we wrap in a try/catch and ignore the error.\ntry {\n  const currentChainStates = await database\n    .select()\n    .from(internalTables.chain)\n    .where(eq(internalTables.chain.chainId, chainId))\n    .execute();\n  // TODO: replace this type workaround with `noUncheckedIndexedAccess: true` when we can fix all the issues related (https://github.com/latticexyz/mud/issues/1212)\n  const currentChainState: (typeof currentChainStates)[number] | undefined = currentChainStates[0];\n\n  if (currentChainState != null) {\n    if (currentChainState.schemaVersion != schemaVersion) {\n      console.log(\n        \"schema version changed from\",\n        currentChainState.schemaVersion,\n        \"to\",\n        schemaVersion,\n        \"cleaning database\"\n      );\n      await cleanDatabase(database);\n    } else if (currentChainState.lastUpdatedBlockNumber != null) {\n      console.log(\"resuming from block number\", currentChainState.lastUpdatedBlockNumber + 1n);\n      startBlock = currentChainState.lastUpdatedBlockNumber + 1n;\n    }\n  }\n} catch (error) {\n  // ignore errors, this is optional\n}\n\nconst { latestBlockNumber$, storedBlockLogs$ } = await createStoreSync({\n  storageAdapter,\n  publicClient,\n  startBlock,\n  maxBlockRange: env.MAX_BLOCK_RANGE,\n});\n\nstoredBlockLogs$.subscribe();\n\ncombineLatest([latestBlockNumber$, storedBlockLogs$])\n  .pipe(\n    filter(\n      ([latestBlockNumber, { blockNumber: lastBlockNumberProcessed }]) => latestBlockNumber === lastBlockNumberProcessed\n    ),\n    first()\n  )\n  .subscribe(() => {\n    console.log(\"all caught up\");\n  });\n\n// @see https://fastify.dev/docs/latest/\nconst server = fastify({\n  maxParamLength: 5000,\n});\n\nawait server.register(import(\"@fastify/cors\"));\n\n// @see https://trpc.io/docs/server/adapters/fastify\nserver.register(fastifyTRPCPlugin<AppRouter>, {\n  prefix: \"/trpc\",\n  trpcOptions: {\n    router: createAppRouter(),\n    createContext: async () => ({\n      queryAdapter: await createQueryAdapter(database),\n    }),\n  },\n});\n\nawait server.listen({ host: env.HOST, port: env.PORT });\nconsole.log(`indexer server listening on http://${env.HOST}:${env.PORT}`);\n","import { eq } from \"drizzle-orm\";\nimport { PgDatabase } from \"drizzle-orm/pg-core\";\nimport { buildTable, buildInternalTables, getTables } from \"@latticexyz/store-sync/postgres\";\nimport { QueryAdapter } from \"@latticexyz/store-sync/trpc-indexer\";\nimport { debug } from \"../debug\";\nimport { getAddress } from \"viem\";\n\n/**\n * Creates a query adapter for the tRPC server/client to query data from Postgres.\n *\n * @param {PgDatabase<any>} database Postgres database object from Drizzle\n * @returns {Promise<QueryAdapter>} A set of methods used by tRPC endpoints.\n */\nexport async function createQueryAdapter(database: PgDatabase<any>): Promise<QueryAdapter> {\n  const adapter: QueryAdapter = {\n    async findAll(chainId, address) {\n      const internalTables = buildInternalTables();\n      const tables = (await getTables(database)).filter(\n        (table) => address != null && getAddress(address) === getAddress(table.address)\n      );\n\n      const tablesWithRecords = await Promise.all(\n        tables.map(async (table) => {\n          const sqliteTable = buildTable(table);\n          const records = await database.select().from(sqliteTable).where(eq(sqliteTable.__isDeleted, false)).execute();\n          return {\n            ...table,\n            records: records.map((record) => ({\n              key: Object.fromEntries(Object.entries(table.keySchema).map(([name]) => [name, record[name]])),\n              value: Object.fromEntries(Object.entries(table.valueSchema).map(([name]) => [name, record[name]])),\n            })),\n          };\n        })\n      );\n\n      const metadata = await database\n        .select()\n        .from(internalTables.chain)\n        .where(eq(internalTables.chain.chainId, chainId))\n        .execute();\n      const { lastUpdatedBlockNumber } = metadata[0] ?? {};\n\n      const result = {\n        blockNumber: lastUpdatedBlockNumber ?? null,\n        tables: tablesWithRecords,\n      };\n\n      debug(\"findAll\", chainId, address, result);\n\n      return result;\n    },\n  };\n  return adapter;\n}\n"],"mappings":";2CACA,MAAO,gBACP,OAAS,KAAAA,MAAS,MAClB,OAAS,iBAAAC,EAAe,MAAAC,MAAU,cAClC,OAAS,sBAAAC,EAAoB,YAAAC,EAAU,aAAAC,EAAW,QAAAC,MAAuB,OACzE,OAAOC,MAAa,UACpB,OAAS,qBAAAC,MAAyB,gCAClC,OAAoB,mBAAAC,MAAuB,sCCP3C,OAAS,MAAAC,MAAU,cAEnB,OAAS,cAAAC,EAAY,uBAAAC,EAAqB,aAAAC,MAAiB,kCAG3D,OAAS,cAAAC,MAAkB,OAQ3B,eAAsBC,EAAmBC,EAAkD,CAuCzF,MAtC8B,CAC5B,MAAM,QAAQC,EAASC,EAAS,CAC9B,IAAMC,EAAiBC,EAAoB,EACrCC,GAAU,MAAMC,EAAUN,CAAQ,GAAG,OACxCO,GAAUL,GAAW,MAAQJ,EAAWI,CAAO,IAAMJ,EAAWS,EAAM,OAAO,CAChF,EAEMC,EAAoB,MAAM,QAAQ,IACtCH,EAAO,IAAI,MAAOE,GAAU,CAC1B,IAAME,EAAcC,EAAWH,CAAK,EAC9BI,EAAU,MAAMX,EAAS,OAAO,EAAE,KAAKS,CAAW,EAAE,MAAMG,EAAGH,EAAY,YAAa,EAAK,CAAC,EAAE,QAAQ,EAC5G,MAAO,CACL,GAAGF,EACH,QAASI,EAAQ,IAAKE,IAAY,CAChC,IAAK,OAAO,YAAY,OAAO,QAAQN,EAAM,SAAS,EAAE,IAAI,CAAC,CAACO,CAAI,IAAM,CAACA,EAAMD,EAAOC,CAAI,CAAC,CAAC,CAAC,EAC7F,MAAO,OAAO,YAAY,OAAO,QAAQP,EAAM,WAAW,EAAE,IAAI,CAAC,CAACO,CAAI,IAAM,CAACA,EAAMD,EAAOC,CAAI,CAAC,CAAC,CAAC,CACnG,EAAE,CACJ,CACF,CAAC,CACH,EAEMC,EAAW,MAAMf,EACpB,OAAO,EACP,KAAKG,EAAe,KAAK,EACzB,MAAMS,EAAGT,EAAe,MAAM,QAASF,CAAO,CAAC,EAC/C,QAAQ,EACL,CAAE,uBAAAe,CAAuB,EAAID,EAAS,CAAC,GAAK,CAAC,EAE7CE,EAAS,CACb,YAAaD,GAA0B,KACvC,OAAQR,CACV,EAEA,OAAAU,EAAM,UAAWjB,EAASC,EAASe,CAAM,EAElCA,CACT,CACF,CAEF,CD5CA,OAAS,aAAAE,MAAiB,2BAC1B,OAAS,iBAAAC,EAAe,UAAAC,EAAQ,SAAAC,MAAa,OAC7C,OAAS,WAAAC,MAAe,0BACxB,OAAOC,MAAc,WACrB,OAAS,iBAAAC,EAAe,mBAAAC,EAAiB,iBAAAC,MAAqB,kCAC9D,OAAS,mBAAAC,MAAuB,yBAGhC,IAAMC,EAAMC,EACVC,EAAE,OAAO,CACP,aAAcA,EAAE,OAAO,CACzB,CAAC,CACH,EAEMC,EAA0B,CAE9BH,EAAI,WAAaI,EAAUJ,EAAI,UAAU,EAAI,OAE7CA,EAAI,aAAeK,EAAKL,EAAI,YAAY,EAAI,MAC9C,EAAE,OAAOM,CAAS,EAEZC,EAAeC,EAAmB,CACtC,UAAWC,EAASN,CAAU,EAC9B,gBAAiBH,EAAI,gBACvB,CAAC,EAEKU,EAAU,MAAMH,EAAa,WAAW,EACxCI,EAAWC,EAAQC,EAASb,EAAI,YAAY,EAAG,CACnD,OAAQ,IAAIc,CACd,CAAC,EAEK,CAAE,eAAAC,EAAgB,eAAAC,CAAe,EAAI,MAAMC,EAAgB,CAAE,SAAAN,EAAU,aAAAJ,CAAa,CAAC,EAEvFW,EAAalB,EAAI,YAGrB,GAAI,CAOF,IAAMmB,GANqB,MAAMR,EAC9B,OAAO,EACP,KAAKK,EAAe,KAAK,EACzB,MAAMI,EAAGJ,EAAe,MAAM,QAASN,CAAO,CAAC,EAC/C,QAAQ,GAEmF,CAAC,EAE3FS,GAAqB,OACnBA,EAAkB,eAAiBE,GACrC,QAAQ,IACN,8BACAF,EAAkB,cAClB,KACAE,EACA,mBACF,EACA,MAAMC,EAAcX,CAAQ,GACnBQ,EAAkB,wBAA0B,OACrD,QAAQ,IAAI,6BAA8BA,EAAkB,uBAAyB,EAAE,EACvFD,EAAaC,EAAkB,uBAAyB,IAG9D,MAAE,CAEF,CAEA,GAAM,CAAE,mBAAAI,EAAoB,iBAAAC,CAAiB,EAAI,MAAMC,EAAgB,CACrE,eAAAV,EACA,aAAAR,EACA,WAAAW,EACA,cAAelB,EAAI,eACrB,CAAC,EAEDwB,EAAiB,UAAU,EAE3BE,EAAc,CAACH,EAAoBC,CAAgB,CAAC,EACjD,KACCG,EACE,CAAC,CAACC,EAAmB,CAAE,YAAaC,CAAyB,CAAC,IAAMD,IAAsBC,CAC5F,EACAC,EAAM,CACR,EACC,UAAU,IAAM,CACf,QAAQ,IAAI,eAAe,CAC7B,CAAC,EAGH,IAAMC,EAASC,EAAQ,CACrB,eAAgB,GAClB,CAAC,EAED,MAAMD,EAAO,SAAS,OAAO,eAAe,CAAC,EAG7CA,EAAO,SAASE,EAA8B,CAC5C,OAAQ,QACR,YAAa,CACX,OAAQC,EAAgB,EACxB,cAAe,UAAa,CAC1B,aAAc,MAAMC,EAAmBxB,CAAQ,CACjD,EACF,CACF,CAAC,EAED,MAAMoB,EAAO,OAAO,CAAE,KAAM/B,EAAI,KAAM,KAAMA,EAAI,IAAK,CAAC,EACtD,QAAQ,IAAI,sCAAsCA,EAAI,QAAQA,EAAI,MAAM","names":["z","DefaultLogger","eq","createPublicClient","fallback","webSocket","http","fastify","fastifyTRPCPlugin","createAppRouter","eq","buildTable","buildInternalTables","getTables","getAddress","createQueryAdapter","database","chainId","address","internalTables","buildInternalTables","tables","getTables","table","tablesWithRecords","sqliteTable","buildTable","records","eq","record","name","metadata","lastUpdatedBlockNumber","result","debug","isDefined","combineLatest","filter","first","drizzle","postgres","cleanDatabase","postgresStorage","schemaVersion","createStoreSync","env","parseEnv","z","transports","webSocket","http","isDefined","publicClient","createPublicClient","fallback","chainId","database","drizzle","postgres","DefaultLogger","storageAdapter","internalTables","postgresStorage","startBlock","currentChainState","eq","schemaVersion","cleanDatabase","latestBlockNumber$","storedBlockLogs$","createStoreSync","combineLatest","filter","latestBlockNumber","lastBlockNumberProcessed","first","server","fastify","fastifyTRPCPlugin","createAppRouter","createQueryAdapter"]}

package/dist/bin/sqlite-indexer.js

@@ -0,0 +1,3 @@
+#!/usr/bin/env node
+import{a as f,b as u}from"../chunk-X3OEYQLT.js";import"dotenv/config";import P from"node:fs";import{z as S}from"zod";import{eq as N}from"drizzle-orm";import{drizzle as B}from"drizzle-orm/better-sqlite3";import I from"better-sqlite3";import{createPublicClient as O,fallback as w,webSocket as Q,http as v}from"viem";import U from"fastify";import{fastifyTRPCPlugin as x}from"@trpc/server/adapters/fastify";import{createAppRouter as j}from"@latticexyz/store-sync/trpc-indexer";import{chainState as T,schemaVersion as g,syncToSqlite as q}from"@latticexyz/store-sync/sqlite";import{eq as d}from"drizzle-orm";import{buildTable as C,chainState as b,getTables as E}from"@latticexyz/store-sync/sqlite";async function h(r){return{async findAll(i,c){let R=E(r).filter(o=>o.address===c).map(o=>{let m=C(o),k=r.select().from(m).where(d(m.__isDeleted,!1)).all();return{...o,records:k.map(p=>({key:Object.fromEntries(Object.entries(o.keySchema).map(([a])=>[a,p[a]])),value:Object.fromEntries(Object.entries(o.valueSchema).map(([a])=>[a,p[a]]))}))}}),A=r.select().from(b).where(d(b.chainId,i)).all(),{lastUpdatedBlockNumber:_}=A[0]??{},l={blockNumber:_??null,tables:R};return f("findAll",i,c,l),l}}}import{isDefined as D}from"@latticexyz/common/utils";import{combineLatest as H,filter as M,first as V}from"rxjs";var e=u(S.object({SQLITE_FILENAME:S.string().default("indexer.db")})),$=[e.RPC_WS_URL?Q(e.RPC_WS_URL):void 0,e.RPC_HTTP_URL?v(e.RPC_HTTP_URL):void 0].filter(D),L=O({transport:w($),pollingInterval:e.POLLING_INTERVAL}),z=await L.getChainId(),n=B(new I(e.SQLITE_FILENAME)),y=e.START_BLOCK;try{let t=n.select().from(T).where(N(T.chainId,z)).all()[0];t!=null&&(t.schemaVersion!=g?(console.log("schema version changed from",t.schemaVersion,"to",g,"recreating database"),P.truncateSync(e.SQLITE_FILENAME)):t.lastUpdatedBlockNumber!=null&&(console.log("resuming from block number",t.lastUpdatedBlockNumber+1n),y=t.lastUpdatedBlockNumber+1n))}catch{}var{latestBlockNumber$:F,storedBlockLogs$:W}=await q({database:n,publicClient:L,startBlock:y,maxBlockRange:e.MAX_BLOCK_RANGE});H([F,W]).pipe(M(([r,{blockNumber:t}])=>r===t),V()).subscribe(()=>{console.log("all caught up")});var s=U({maxParamLength:5e3});await s.register(import("@fastify/cors"));s.register(x,{prefix:"/trpc",trpcOptions:{router:j(),createContext:async()=>({queryAdapter:await h(n)})}});await s.listen({host:e.HOST,port:e.PORT});console.log(`indexer server listening on http://${e.HOST}:${e.PORT}`);
+//# sourceMappingURL=sqlite-indexer.js.map

package/dist/bin/sqlite-indexer.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../bin/sqlite-indexer.ts","../../src/sqlite/createQueryAdapter.ts"],"sourcesContent":["#!/usr/bin/env node\nimport \"dotenv/config\";\nimport fs from \"node:fs\";\nimport { z } from \"zod\";\nimport { eq } from \"drizzle-orm\";\nimport { drizzle } from \"drizzle-orm/better-sqlite3\";\nimport Database from \"better-sqlite3\";\nimport { createPublicClient, fallback, webSocket, http, Transport } from \"viem\";\nimport fastify from \"fastify\";\nimport { fastifyTRPCPlugin } from \"@trpc/server/adapters/fastify\";\nimport { AppRouter, createAppRouter } from \"@latticexyz/store-sync/trpc-indexer\";\nimport { chainState, schemaVersion, syncToSqlite } from \"@latticexyz/store-sync/sqlite\";\nimport { createQueryAdapter } from \"../src/sqlite/createQueryAdapter\";\nimport { isDefined } from \"@latticexyz/common/utils\";\nimport { combineLatest, filter, first } from \"rxjs\";\nimport { parseEnv } from \"./parseEnv\";\n\nconst env = parseEnv(\n  z.object({\n    SQLITE_FILENAME: z.string().default(\"indexer.db\"),\n  })\n);\n\nconst transports: Transport[] = [\n  // prefer WS when specified\n  env.RPC_WS_URL ? webSocket(env.RPC_WS_URL) : undefined,\n  // otherwise use or fallback to HTTP\n  env.RPC_HTTP_URL ? http(env.RPC_HTTP_URL) : undefined,\n].filter(isDefined);\n\nconst publicClient = createPublicClient({\n  transport: fallback(transports),\n  pollingInterval: env.POLLING_INTERVAL,\n});\n\nconst chainId = await publicClient.getChainId();\nconst database = drizzle(new Database(env.SQLITE_FILENAME));\n\nlet startBlock = env.START_BLOCK;\n\n// Resume from latest block stored in DB. This will throw if the DB doesn't exist yet, so we wrap in a try/catch and ignore the error.\ntry {\n  const currentChainStates = database.select().from(chainState).where(eq(chainState.chainId, chainId)).all();\n  // TODO: replace this type workaround with `noUncheckedIndexedAccess: true` when we can fix all the issues related (https://github.com/latticexyz/mud/issues/1212)\n  const currentChainState: (typeof currentChainStates)[number] | undefined = currentChainStates[0];\n\n  if (currentChainState != null) {\n    if (currentChainState.schemaVersion != schemaVersion) {\n      console.log(\n        \"schema version changed from\",\n        currentChainState.schemaVersion,\n        \"to\",\n        schemaVersion,\n        \"recreating database\"\n      );\n      fs.truncateSync(env.SQLITE_FILENAME);\n    } else if (currentChainState.lastUpdatedBlockNumber != null) {\n      console.log(\"resuming from block number\", currentChainState.lastUpdatedBlockNumber + 1n);\n      startBlock = currentChainState.lastUpdatedBlockNumber + 1n;\n    }\n  }\n} catch (error) {\n  // ignore errors, this is optional\n}\n\nconst { latestBlockNumber$, storedBlockLogs$ } = await syncToSqlite({\n  database,\n  publicClient,\n  startBlock,\n  maxBlockRange: env.MAX_BLOCK_RANGE,\n});\n\ncombineLatest([latestBlockNumber$, storedBlockLogs$])\n  .pipe(\n    filter(\n      ([latestBlockNumber, { blockNumber: lastBlockNumberProcessed }]) => latestBlockNumber === lastBlockNumberProcessed\n    ),\n    first()\n  )\n  .subscribe(() => {\n    console.log(\"all caught up\");\n  });\n\n// @see https://fastify.dev/docs/latest/\nconst server = fastify({\n  maxParamLength: 5000,\n});\n\nawait server.register(import(\"@fastify/cors\"));\n\n// @see https://trpc.io/docs/server/adapters/fastify\nserver.register(fastifyTRPCPlugin<AppRouter>, {\n  prefix: \"/trpc\",\n  trpcOptions: {\n    router: createAppRouter(),\n    createContext: async () => ({\n      queryAdapter: await createQueryAdapter(database),\n    }),\n  },\n});\n\nawait server.listen({ host: env.HOST, port: env.PORT });\nconsole.log(`indexer server listening on http://${env.HOST}:${env.PORT}`);\n","import { eq } from \"drizzle-orm\";\nimport { BaseSQLiteDatabase } from \"drizzle-orm/sqlite-core\";\nimport { buildTable, chainState, getTables } from \"@latticexyz/store-sync/sqlite\";\nimport { QueryAdapter } from \"@latticexyz/store-sync/trpc-indexer\";\nimport { debug } from \"../debug\";\n\n/**\n * Creates a storage adapter for the tRPC server/client to query data from SQLite.\n *\n * @param {BaseSQLiteDatabase<\"sync\", any>} database SQLite database object from Drizzle\n * @returns {Promise<QueryAdapter>} A set of methods used by tRPC endpoints.\n */\nexport async function createQueryAdapter(database: BaseSQLiteDatabase<\"sync\", any>): Promise<QueryAdapter> {\n  const adapter: QueryAdapter = {\n    async findAll(chainId, address) {\n      const tables = getTables(database).filter((table) => table.address === address);\n\n      const tablesWithRecords = tables.map((table) => {\n        const sqliteTable = buildTable(table);\n        const records = database.select().from(sqliteTable).where(eq(sqliteTable.__isDeleted, false)).all();\n        return {\n          ...table,\n          records: records.map((record) => ({\n            key: Object.fromEntries(Object.entries(table.keySchema).map(([name]) => [name, record[name]])),\n            value: Object.fromEntries(Object.entries(table.valueSchema).map(([name]) => [name, record[name]])),\n          })),\n        };\n      });\n\n      const metadata = database.select().from(chainState).where(eq(chainState.chainId, chainId)).all();\n      const { lastUpdatedBlockNumber } = metadata[0] ?? {};\n\n      const result = {\n        blockNumber: lastUpdatedBlockNumber ?? null,\n        tables: tablesWithRecords,\n      };\n\n      debug(\"findAll\", chainId, address, result);\n\n      return result;\n    },\n  };\n  return adapter;\n}\n"],"mappings":";gDACA,MAAO,gBACP,OAAOA,MAAQ,UACf,OAAS,KAAAC,MAAS,MAClB,OAAS,MAAAC,MAAU,cACnB,OAAS,WAAAC,MAAe,6BACxB,OAAOC,MAAc,iBACrB,OAAS,sBAAAC,EAAoB,YAAAC,EAAU,aAAAC,EAAW,QAAAC,MAAuB,OACzE,OAAOC,MAAa,UACpB,OAAS,qBAAAC,MAAyB,gCAClC,OAAoB,mBAAAC,MAAuB,sCAC3C,OAAS,cAAAC,EAAY,iBAAAC,EAAe,gBAAAC,MAAoB,gCCXxD,OAAS,MAAAC,MAAU,cAEnB,OAAS,cAAAC,EAAY,cAAAC,EAAY,aAAAC,MAAiB,gCAUlD,eAAsBC,EAAmBC,EAAkE,CA8BzG,MA7B8B,CAC5B,MAAM,QAAQC,EAASC,EAAS,CAG9B,IAAMC,EAFSC,EAAUJ,CAAQ,EAAE,OAAQK,GAAUA,EAAM,UAAYH,CAAO,EAE7C,IAAKG,GAAU,CAC9C,IAAMC,EAAcC,EAAWF,CAAK,EAC9BG,EAAUR,EAAS,OAAO,EAAE,KAAKM,CAAW,EAAE,MAAMG,EAAGH,EAAY,YAAa,EAAK,CAAC,EAAE,IAAI,EAClG,MAAO,CACL,GAAGD,EACH,QAASG,EAAQ,IAAKE,IAAY,CAChC,IAAK,OAAO,YAAY,OAAO,QAAQL,EAAM,SAAS,EAAE,IAAI,CAAC,CAACM,CAAI,IAAM,CAACA,EAAMD,EAAOC,CAAI,CAAC,CAAC,CAAC,EAC7F,MAAO,OAAO,YAAY,OAAO,QAAQN,EAAM,WAAW,EAAE,IAAI,CAAC,CAACM,CAAI,IAAM,CAACA,EAAMD,EAAOC,CAAI,CAAC,CAAC,CAAC,CACnG,EAAE,CACJ,CACF,CAAC,EAEKC,EAAWZ,EAAS,OAAO,EAAE,KAAKa,CAAU,EAAE,MAAMJ,EAAGI,EAAW,QAASZ,CAAO,CAAC,EAAE,IAAI,EACzF,CAAE,uBAAAa,CAAuB,EAAIF,EAAS,CAAC,GAAK,CAAC,EAE7CG,EAAS,CACb,YAAaD,GAA0B,KACvC,OAAQX,CACV,EAEA,OAAAa,EAAM,UAAWf,EAASC,EAASa,CAAM,EAElCA,CACT,CACF,CAEF,CD9BA,OAAS,aAAAE,MAAiB,2BAC1B,OAAS,iBAAAC,EAAe,UAAAC,EAAQ,SAAAC,MAAa,OAG7C,IAAMC,EAAMC,EACVC,EAAE,OAAO,CACP,gBAAiBA,EAAE,OAAO,EAAE,QAAQ,YAAY,CAClD,CAAC,CACH,EAEMC,EAA0B,CAE9BH,EAAI,WAAaI,EAAUJ,EAAI,UAAU,EAAI,OAE7CA,EAAI,aAAeK,EAAKL,EAAI,YAAY,EAAI,MAC9C,EAAE,OAAOM,CAAS,EAEZC,EAAeC,EAAmB,CACtC,UAAWC,EAASN,CAAU,EAC9B,gBAAiBH,EAAI,gBACvB,CAAC,EAEKU,EAAU,MAAMH,EAAa,WAAW,EACxCI,EAAWC,EAAQ,IAAIC,EAASb,EAAI,eAAe,CAAC,EAEtDc,EAAad,EAAI,YAGrB,GAAI,CAGF,IAAMe,EAFqBJ,EAAS,OAAO,EAAE,KAAKK,CAAU,EAAE,MAAMC,EAAGD,EAAW,QAASN,CAAO,CAAC,EAAE,IAAI,EAEX,CAAC,EAE3FK,GAAqB,OACnBA,EAAkB,eAAiBG,GACrC,QAAQ,IACN,8BACAH,EAAkB,cAClB,KACAG,EACA,qBACF,EACAC,EAAG,aAAanB,EAAI,eAAe,GAC1Be,EAAkB,wBAA0B,OACrD,QAAQ,IAAI,6BAA8BA,EAAkB,uBAAyB,EAAE,EACvFD,EAAaC,EAAkB,uBAAyB,IAG9D,MAAE,CAEF,CAEA,GAAM,CAAE,mBAAAK,EAAoB,iBAAAC,CAAiB,EAAI,MAAMC,EAAa,CAClE,SAAAX,EACA,aAAAJ,EACA,WAAAO,EACA,cAAed,EAAI,eACrB,CAAC,EAEDuB,EAAc,CAACH,EAAoBC,CAAgB,CAAC,EACjD,KACCG,EACE,CAAC,CAACC,EAAmB,CAAE,YAAaC,CAAyB,CAAC,IAAMD,IAAsBC,CAC5F,EACAC,EAAM,CACR,EACC,UAAU,IAAM,CACf,QAAQ,IAAI,eAAe,CAC7B,CAAC,EAGH,IAAMC,EAASC,EAAQ,CACrB,eAAgB,GAClB,CAAC,EAED,MAAMD,EAAO,SAAS,OAAO,eAAe,CAAC,EAG7CA,EAAO,SAASE,EAA8B,CAC5C,OAAQ,QACR,YAAa,CACX,OAAQC,EAAgB,EACxB,cAAe,UAAa,CAC1B,aAAc,MAAMC,EAAmBrB,CAAQ,CACjD,EACF,CACF,CAAC,EAED,MAAMiB,EAAO,OAAO,CAAE,KAAM5B,EAAI,KAAM,KAAMA,EAAI,IAAK,CAAC,EACtD,QAAQ,IAAI,sCAAsCA,EAAI,QAAQA,EAAI,MAAM","names":["fs","z","eq","drizzle","Database","createPublicClient","fallback","webSocket","http","fastify","fastifyTRPCPlugin","createAppRouter","chainState","schemaVersion","syncToSqlite","eq","buildTable","chainState","getTables","createQueryAdapter","database","chainId","address","tablesWithRecords","getTables","table","sqliteTable","buildTable","records","eq","record","name","metadata","chainState","lastUpdatedBlockNumber","result","debug","isDefined","combineLatest","filter","first","env","parseEnv","z","transports","webSocket","http","isDefined","publicClient","createPublicClient","fallback","chainId","database","drizzle","Database","startBlock","currentChainState","chainState","eq","schemaVersion","fs","latestBlockNumber$","storedBlockLogs$","syncToSqlite","combineLatest","filter","latestBlockNumber","lastBlockNumberProcessed","first","server","fastify","fastifyTRPCPlugin","createAppRouter","createQueryAdapter"]}

package/dist/chunk-X3OEYQLT.js

@@ -0,0 +1,7 @@
+import{z as e,ZodError as c}from"zod";var t=e.intersection(e.object({HOST:e.string().default("0.0.0.0"),PORT:e.coerce.number().positive().default(3001),START_BLOCK:e.coerce.bigint().nonnegative().default(0n),MAX_BLOCK_RANGE:e.coerce.bigint().positive().default(1000n),POLLING_INTERVAL:e.coerce.number().positive().default(1e3)}),e.union([e.object({RPC_HTTP_URL:e.string(),RPC_WS_URL:e.string().optional()}),e.object({RPC_HTTP_URL:e.string().optional(),RPC_WS_URL:e.string()})]));function f(o){let r=o!==void 0?e.intersection(t,o):t;try{return r.parse(process.env)}catch(n){if(n instanceof c){let{_errors:a,...i}=n.format();console.error(`
+Missing or invalid environment variables:
+
+  ${Object.keys(i).join(`
+  `)}
+`),process.exit(1)}throw n}}import s from"debug";var _=s("mud:store-indexer");export{_ as a,f as b};
+//# sourceMappingURL=chunk-X3OEYQLT.js.map

package/dist/chunk-X3OEYQLT.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../bin/parseEnv.ts","../src/debug.ts"],"sourcesContent":["import { z, ZodError, ZodIntersection, ZodTypeAny } from \"zod\";\n\nconst commonSchema = z.intersection(\n  z.object({\n    HOST: z.string().default(\"0.0.0.0\"),\n    PORT: z.coerce.number().positive().default(3001),\n    START_BLOCK: z.coerce.bigint().nonnegative().default(0n),\n    MAX_BLOCK_RANGE: z.coerce.bigint().positive().default(1000n),\n    POLLING_INTERVAL: z.coerce.number().positive().default(1000),\n  }),\n  z.union([\n    z.object({\n      RPC_HTTP_URL: z.string(),\n      RPC_WS_URL: z.string().optional(),\n    }),\n    z.object({\n      RPC_HTTP_URL: z.string().optional(),\n      RPC_WS_URL: z.string(),\n    }),\n  ])\n);\n\nexport function parseEnv<TSchema extends ZodTypeAny | undefined = undefined>(\n  schema?: TSchema\n): z.infer<TSchema extends ZodTypeAny ? ZodIntersection<typeof commonSchema, TSchema> : typeof commonSchema> {\n  const envSchema = schema !== undefined ? z.intersection(commonSchema, schema) : commonSchema;\n  try {\n    return envSchema.parse(process.env);\n  } catch (error) {\n    if (error instanceof ZodError) {\n      const { _errors, ...invalidEnvVars } = error.format();\n      console.error(`\\nMissing or invalid environment variables:\\n\\n  ${Object.keys(invalidEnvVars).join(\"\\n  \")}\\n`);\n      process.exit(1);\n    }\n    throw error;\n  }\n}\n","import createDebug from \"debug\";\n\nexport const debug = createDebug(\"mud:store-indexer\");\n"],"mappings":"AAAA,OAAS,KAAAA,EAAG,YAAAC,MAA6C,MAEzD,IAAMC,EAAeF,EAAE,aACrBA,EAAE,OAAO,CACP,KAAMA,EAAE,OAAO,EAAE,QAAQ,SAAS,EAClC,KAAMA,EAAE,OAAO,OAAO,EAAE,SAAS,EAAE,QAAQ,IAAI,EAC/C,YAAaA,EAAE,OAAO,OAAO,EAAE,YAAY,EAAE,QAAQ,EAAE,EACvD,gBAAiBA,EAAE,OAAO,OAAO,EAAE,SAAS,EAAE,QAAQ,KAAK,EAC3D,iBAAkBA,EAAE,OAAO,OAAO,EAAE,SAAS,EAAE,QAAQ,GAAI,CAC7D,CAAC,EACDA,EAAE,MAAM,CACNA,EAAE,OAAO,CACP,aAAcA,EAAE,OAAO,EACvB,WAAYA,EAAE,OAAO,EAAE,SAAS,CAClC,CAAC,EACDA,EAAE,OAAO,CACP,aAAcA,EAAE,OAAO,EAAE,SAAS,EAClC,WAAYA,EAAE,OAAO,CACvB,CAAC,CACH,CAAC,CACH,EAEO,SAASG,EACdC,EAC2G,CAC3G,IAAMC,EAAYD,IAAW,OAAYJ,EAAE,aAAaE,EAAcE,CAAM,EAAIF,EAChF,GAAI,CACF,OAAOG,EAAU,MAAM,QAAQ,GAAG,CACpC,OAASC,EAAP,CACA,GAAIA,aAAiBL,EAAU,CAC7B,GAAM,CAAE,QAAAM,EAAS,GAAGC,CAAe,EAAIF,EAAM,OAAO,EACpD,QAAQ,MAAM;AAAA;AAAA;AAAA,IAAoD,OAAO,KAAKE,CAAc,EAAE,KAAK;AAAA,GAAM;AAAA,CAAK,EAC9G,QAAQ,KAAK,CAAC,EAEhB,MAAMF,CACR,CACF,CCpCA,OAAOG,MAAiB,QAEjB,IAAMC,EAAQD,EAAY,mBAAmB","names":["z","ZodError","commonSchema","parseEnv","schema","envSchema","error","_errors","invalidEnvVars","createDebug","debug"]}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@latticexyz/store-indexer",
-  "version": "2.0.0-skystrife-playtest-19728914",
+  "version": "2.0.0-snapshot-test-32d38619",
   "description": "Minimal Typescript indexer for Store",
   "repository": {
     "type": "git",
@@ -13,24 +13,29 @@
     ".": "./dist/index.js"
   },
   "types": "src/index.ts",
+  "bin": {
+    "postgres-indexer": "./dist/bin/postgres-indexer.js",
+    "sqlite-indexer": "./dist/bin/sqlite-indexer.js"
+  },
   "dependencies": {
     "@fastify/cors": "^8.3.0",
     "@trpc/client": "10.34.0",
     "@trpc/server": "10.34.0",
     "@wagmi/chains": "^0.2.22",
-    "better-sqlite3": "^8.4.0",
+    "better-sqlite3": "^8.6.0",
     "debug": "^4.3.4",
-    "drizzle-orm": "^0.27.0",
+    "dotenv": "^16.0.3",
+    "drizzle-orm": "^0.28.5",
     "fastify": "^4.21.0",
     "postgres": "^3.3.5",
     "rxjs": "7.5.5",
     "superjson": "^1.12.4",
     "viem": "1.6.0",
     "zod": "^3.21.4",
-    "@latticexyz/block-logs-stream": "2.0.0-skystrife-playtest-19728914",
-    "@latticexyz/common": "2.0.0-skystrife-playtest-19728914",
-    "@latticexyz/store": "2.0.0-skystrife-playtest-19728914",
-    "@latticexyz/store-sync": "2.0.0-skystrife-playtest-19728914"
+    "@latticexyz/block-logs-stream": "2.0.0-snapshot-test-32d38619",
+    "@latticexyz/common": "2.0.0-snapshot-test-32d38619",
+    "@latticexyz/store": "2.0.0-snapshot-test-32d38619",
+    "@latticexyz/store-sync": "2.0.0-snapshot-test-32d38619"
   },
   "devDependencies": {
     "@types/better-sqlite3": "^7.6.4",

package/src/postgres/createQueryAdapter.ts

@@ -13,11 +13,11 @@ import { getAddress } from "viem";
  */
 export async function createQueryAdapter(database: PgDatabase<any>): Promise<QueryAdapter> {
   const adapter: QueryAdapter = {
-    async findAll({ chainId, address, tableIds }) {
+    async findAll(chainId, address) {
       const internalTables = buildInternalTables();
-      const tables = (await getTables(database))
-        .filter((table) => address == null || getAddress(address) === getAddress(table.address))
-        .filter((table) => tableIds == null || tableIds.includes(table.tableId));
+      const tables = (await getTables(database)).filter(
+        (table) => address != null && getAddress(address) === getAddress(table.address)
+      );
 
       const tablesWithRecords = await Promise.all(
         tables.map(async (table) => {

package/src/sqlite/createQueryAdapter.ts

@@ -1,9 +1,8 @@
-import { eq, inArray, and } from "drizzle-orm";
+import { eq } from "drizzle-orm";
 import { BaseSQLiteDatabase } from "drizzle-orm/sqlite-core";
-import { createSqliteTable, chainState, getTables } from "@latticexyz/store-sync/sqlite";
+import { buildTable, chainState, getTables } from "@latticexyz/store-sync/sqlite";
 import { QueryAdapter } from "@latticexyz/store-sync/trpc-indexer";
 import { debug } from "../debug";
-import { Hex, getAddress } from "viem";
 
 /**
  * Creates a storage adapter for the tRPC server/client to query data from SQLite.
@@ -13,51 +12,12 @@ import { Hex, getAddress } from "viem";
  */
 export async function createQueryAdapter(database: BaseSQLiteDatabase<"sync", any>): Promise<QueryAdapter> {
   const adapter: QueryAdapter = {
-    async findAll({ chainId, address, tableIds, matchId }) {
-      const tables = getTables(database)
-        .filter((table) => address == null || getAddress(address) === getAddress(table.address))
-        .filter((table) => tableIds == null || tableIds.includes(table.tableId))
-        .filter((table) =>
-          // we don't need KeysWithValue tables
-          matchId != null ? table.namespace !== "keyswval" : true
-        );
-
-      const entities = ((): Hex[] => {
-        if (!address || !matchId) return [];
-        try {
-          const Position = createSqliteTable({
-            address,
-            namespace: "",
-            name: "Position",
-            keySchema: { key: "bytes32" },
-            valueSchema: {
-              x: "int32",
-              y: "int32",
-              z: "int32",
-            },
-          });
-          const positions = database.select().from(Position).where(eq(Position.z, matchId)).all();
-          return positions.map((pos) => pos.key);
-        } catch (error: unknown) {
-          return [];
-        }
-      })();
+    async findAll(chainId, address) {
+      const tables = getTables(database).filter((table) => table.address === address);
 
       const tablesWithRecords = tables.map((table) => {
-        const sqliteTable = createSqliteTable(table);
-        const records = database
-          .select()
-          .from(sqliteTable)
-          .where(
-            and(
-              eq(sqliteTable.__isDeleted, false),
-              entities.length &&
-                (table.name === "MoveDifficulty" || table.name === "TerrainType") /* || table.name === "ArmorModifier"*/
-                ? inArray(sqliteTable.__key, entities)
-                : undefined
-            )
-          )
-          .all();
+        const sqliteTable = buildTable(table);
+        const records = database.select().from(sqliteTable).where(eq(sqliteTable.__isDeleted, false)).all();
         return {
           ...table,
           records: records.map((record) => ({
@@ -75,19 +35,7 @@ export async function createQueryAdapter(database: BaseSQLiteDatabase<"sync", an
         tables: tablesWithRecords,
       };
 
-      const count = tablesWithRecords.reduce((sum, table) => sum + table.records.length, 0);
-
-      debug(
-        "findAll",
-        "chainId:",
-        chainId,
-        "address:",
-        address,
-        "tables:",
-        tablesWithRecords.length,
-        "records:",
-        count
-      );
+      debug("findAll", chainId, address, result);
 
       return result;
     },