@latticexyz/store-indexer 2.0.0-next.13 → 2.0.0-next.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -15
- package/dist/bin/postgres-frontend.js +3 -0
- package/dist/bin/postgres-frontend.js.map +1 -0
- package/dist/bin/postgres-indexer.js +1 -1
- package/dist/bin/postgres-indexer.js.map +1 -1
- package/dist/bin/sqlite-indexer.js +1 -1
- package/dist/bin/sqlite-indexer.js.map +1 -1
- package/dist/chunk-2E5MDUA2.js +7 -0
- package/dist/chunk-2E5MDUA2.js.map +1 -0
- package/dist/chunk-LOFAZSVJ.js +2 -0
- package/dist/chunk-LOFAZSVJ.js.map +1 -0
- package/package.json +9 -7
- package/dist/chunk-X3OEYQLT.js +0 -7
- package/dist/chunk-X3OEYQLT.js.map +0 -1
package/README.md
CHANGED
|
@@ -11,7 +11,7 @@ npm install @latticexyz/store-indexer
|
|
|
11
11
|
|
|
12
12
|
npm sqlite-indexer
|
|
13
13
|
# or
|
|
14
|
-
npm postgres-indexer
|
|
14
|
+
npm postgres-indexer & npm postgres-frontend
|
|
15
15
|
```
|
|
16
16
|
|
|
17
17
|
or execute the one of the package bins directly:
|
|
@@ -19,32 +19,40 @@ or execute the one of the package bins directly:
|
|
|
19
19
|
```sh
|
|
20
20
|
npx -p @latticexyz/store-indexer sqlite-indexer
|
|
21
21
|
# or
|
|
22
|
-
npx -p @latticexyz/store-indexer postgres-indexer
|
|
22
|
+
npx -p @latticexyz/store-indexer postgres-indexer & npx -p @latticexyz/store-indexer postgres-frontend
|
|
23
23
|
```
|
|
24
24
|
|
|
25
25
|
## Configuration
|
|
26
26
|
|
|
27
27
|
Each indexer can be configured with environment variables.
|
|
28
28
|
|
|
29
|
-
### Common environment variables
|
|
29
|
+
### Common environment variables for indexer
|
|
30
30
|
|
|
31
|
-
| Variable | Description
|
|
32
|
-
| ------------------ |
|
|
33
|
-
| `
|
|
34
|
-
| `
|
|
35
|
-
| `
|
|
36
|
-
| `
|
|
37
|
-
| `
|
|
38
|
-
| `
|
|
39
|
-
| `POLLING_INTERVAL` | How often to poll for new blocks (in milliseconds) | `1000` |
|
|
31
|
+
| Variable | Description | Default |
|
|
32
|
+
| ------------------ | ------------------------------------------------------------------------------------------------------------- | ------- |
|
|
33
|
+
| `RPC_HTTP_URL` | HTTP URL for Ethereum RPC to fetch data from | |
|
|
34
|
+
| `RPC_WS_URL` | WebSocket URL for Ethereum RPC to fetch data from | |
|
|
35
|
+
| `START_BLOCK` | Block number to start indexing from | `0` |
|
|
36
|
+
| `MAX_BLOCK_RANGE` | Maximum number of blocks to fetch from the RPC per request | `1000` |
|
|
37
|
+
| `POLLING_INTERVAL` | How often to poll for new blocks (in milliseconds) | `1000` |
|
|
38
|
+
| `STORE_ADDRESS` | Optional address of the MUD Store to index. By default, store-indexer will index all MUD Stores on the chain. | |
|
|
40
39
|
|
|
41
40
|
Note that you only need one of `RPC_HTTP_URL` or `RPC_WS_URL`, but we recommend both. The WebSocket URL will be prioritized and fall back to the HTTP URL if there are any connection issues.
|
|
42
41
|
|
|
42
|
+
### Common environment variables for frontend
|
|
43
|
+
|
|
44
|
+
| Variable | Description | Default |
|
|
45
|
+
| -------- | ------------------------------------------------ | --------- |
|
|
46
|
+
| `HOST` | Host that the indexer frontend server listens on | `0.0.0.0` |
|
|
47
|
+
| `PORT` | Port that the indexer frontend server listens on | `3001` |
|
|
48
|
+
|
|
43
49
|
### Postgres indexer environment variables
|
|
44
50
|
|
|
45
|
-
| Variable
|
|
46
|
-
|
|
|
47
|
-
| `DATABASE_URL`
|
|
51
|
+
| Variable | Description | Default |
|
|
52
|
+
| ------------------ | --------------------------------------------------- | ------- |
|
|
53
|
+
| `DATABASE_URL` | Postgres connection URL | |
|
|
54
|
+
| `HEALTHCHECK_HOST` | Host that the indexer healthcheck server listens on | |
|
|
55
|
+
| `HEATHCHECK_PORT` | Port that the indexer healthcheck server listens on | |
|
|
48
56
|
|
|
49
57
|
### SQLite indexer environment variables
|
|
50
58
|
|
|
@@ -0,0 +1,3 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import{a as A}from"../chunk-LOFAZSVJ.js";import{a as g,c as h}from"../chunk-2E5MDUA2.js";import"dotenv/config";import{z as l}from"zod";import _ from"fastify";import{fastifyTRPCPlugin as j}from"@trpc/server/adapters/fastify";import{createAppRouter as q}from"@latticexyz/store-sync/trpc-indexer";import{eq as T}from"drizzle-orm";import{buildTable as x,buildInternalTables as D,getTables as E}from"@latticexyz/store-sync/postgres";import{getAddress as k}from"viem";import{decodeDynamicField as I}from"@latticexyz/protocol-parser";async function w(r){return{async findAll({chainId:p,address:c,filters:m=[]}){let d=Array.from(new Set(m.map(e=>e.tableId))),R=(await E(r)).filter(e=>c==null||k(c)===k(e.address)).filter(e=>!d.length||d.includes(e.tableId)),O=await Promise.all(R.map(async e=>{let y=x(e),b=await r.select().from(y).where(T(y.__isDeleted,!1)).execute(),v=m.length?b.filter(i=>{let t=I("bytes32[]",i.__key);return m.some(o=>o.tableId===e.tableId&&(o.key0==null||o.key0===t[0])&&(o.key1==null||o.key1===t[1]))}):b;return{...e,records:v.map(i=>({key:Object.fromEntries(Object.entries(e.keySchema).map(([t])=>[t,i[t]])),value:Object.fromEntries(Object.entries(e.valueSchema).map(([t])=>[t,i[t]]))}))}})),u=D(),P=await r.select().from(u.chain).where(T(u.chain.chainId,p)).execute(),{lastUpdatedBlockNumber:S}=P[0]??{},f={blockNumber:S??null,tables:O};return A("findAll",p,c,f),f}}}import{drizzle as z}from"drizzle-orm/postgres-js";import Q from"postgres";var a=h(l.intersection(g,l.object({DATABASE_URL:l.string()}))),B=z(Q(a.DATABASE_URL)),n=_({maxParamLength:5e3});await n.register(import("@fastify/cors"));n.get("/healthz",(r,s)=>s.code(200).send());n.get("/readyz",(r,s)=>s.code(200).send());n.register(j,{prefix:"/trpc",trpcOptions:{router:q(),createContext:async()=>({queryAdapter:await w(B)})}});await n.listen({host:a.HOST,port:a.PORT});console.log(`postgres indexer frontend listening on http://${a.HOST}:${a.PORT}`);
|
|
3
|
+
//# sourceMappingURL=postgres-frontend.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../../bin/postgres-frontend.ts","../../src/postgres/createQueryAdapter.ts"],"sourcesContent":["#!/usr/bin/env node\nimport \"dotenv/config\";\nimport { z } from \"zod\";\nimport fastify from \"fastify\";\nimport { fastifyTRPCPlugin } from \"@trpc/server/adapters/fastify\";\nimport { AppRouter, createAppRouter } from \"@latticexyz/store-sync/trpc-indexer\";\nimport { createQueryAdapter } from \"../src/postgres/createQueryAdapter\";\nimport { drizzle } from \"drizzle-orm/postgres-js\";\nimport postgres from \"postgres\";\nimport { frontendEnvSchema, parseEnv } from \"./parseEnv\";\n\nconst env = parseEnv(\n z.intersection(\n frontendEnvSchema,\n z.object({\n DATABASE_URL: z.string(),\n })\n )\n);\n\nconst database = drizzle(postgres(env.DATABASE_URL));\n\n// @see https://fastify.dev/docs/latest/\nconst server = fastify({\n maxParamLength: 5000,\n});\n\nawait server.register(import(\"@fastify/cors\"));\n\n// k8s healthchecks\nserver.get(\"/healthz\", (req, res) => res.code(200).send());\nserver.get(\"/readyz\", (req, res) => res.code(200).send());\n\n// @see https://trpc.io/docs/server/adapters/fastify\nserver.register(fastifyTRPCPlugin<AppRouter>, {\n prefix: \"/trpc\",\n trpcOptions: {\n router: createAppRouter(),\n createContext: async () => ({\n queryAdapter: await createQueryAdapter(database),\n }),\n },\n});\n\nawait server.listen({ host: env.HOST, port: env.PORT });\nconsole.log(`postgres indexer frontend listening on http://${env.HOST}:${env.PORT}`);\n","import { eq } from \"drizzle-orm\";\nimport { PgDatabase } from \"drizzle-orm/pg-core\";\nimport { buildTable, buildInternalTables, getTables } from \"@latticexyz/store-sync/postgres\";\nimport { QueryAdapter } from \"@latticexyz/store-sync/trpc-indexer\";\nimport { debug } from \"../debug\";\nimport { getAddress } from \"viem\";\nimport { decodeDynamicField } from \"@latticexyz/protocol-parser\";\n\n/**\n * Creates a query adapter for the tRPC server/client to query data from Postgres.\n *\n * @param {PgDatabase<any>} database Postgres database object from Drizzle\n * @returns {Promise<QueryAdapter>} A set of methods used by tRPC endpoints.\n */\nexport async function createQueryAdapter(database: PgDatabase<any>): Promise<QueryAdapter> {\n const adapter: QueryAdapter = {\n async findAll({ chainId, address, filters = [] }) {\n // If _any_ filter has a table ID, this will filter down all data to just those tables. Which mean we can't yet mix table filters with key-only filters.\n // TODO: improve this so we can express this in the query (need to be able to query data across tables more easily)\n const tableIds = Array.from(new Set(filters.map((filter) => filter.tableId)));\n const tables = (await getTables(database))\n .filter((table) => address == null || getAddress(address) === getAddress(table.address))\n .filter((table) => !tableIds.length || tableIds.includes(table.tableId));\n\n const tablesWithRecords = await Promise.all(\n tables.map(async (table) => {\n const sqliteTable = buildTable(table);\n const records = await database.select().from(sqliteTable).where(eq(sqliteTable.__isDeleted, false)).execute();\n const filteredRecords = !filters.length\n ? records\n : records.filter((record) => {\n const keyTuple = decodeDynamicField(\"bytes32[]\", record.__key);\n return filters.some(\n (filter) =>\n filter.tableId === table.tableId &&\n (filter.key0 == null || filter.key0 === keyTuple[0]) &&\n (filter.key1 == null || filter.key1 === keyTuple[1])\n );\n });\n return {\n ...table,\n records: filteredRecords.map((record) => ({\n key: Object.fromEntries(Object.entries(table.keySchema).map(([name]) => [name, record[name]])),\n value: Object.fromEntries(Object.entries(table.valueSchema).map(([name]) => [name, record[name]])),\n })),\n };\n })\n );\n\n const internalTables = buildInternalTables();\n const metadata = await database\n .select()\n .from(internalTables.chain)\n .where(eq(internalTables.chain.chainId, chainId))\n .execute();\n const { lastUpdatedBlockNumber } = metadata[0] ?? {};\n\n const result = {\n blockNumber: lastUpdatedBlockNumber ?? null,\n tables: tablesWithRecords,\n };\n\n debug(\"findAll\", chainId, address, result);\n\n return result;\n },\n };\n return adapter;\n}\n"],"mappings":";yFACA,MAAO,gBACP,OAAS,KAAAA,MAAS,MAClB,OAAOC,MAAa,UACpB,OAAS,qBAAAC,MAAyB,gCAClC,OAAoB,mBAAAC,MAAuB,sCCL3C,OAAS,MAAAC,MAAU,cAEnB,OAAS,cAAAC,EAAY,uBAAAC,EAAqB,aAAAC,MAAiB,kCAG3D,OAAS,cAAAC,MAAkB,OAC3B,OAAS,sBAAAC,MAA0B,8BAQnC,eAAsBC,EAAmBC,EAAkD,CAqDzF,MApD8B,CAC5B,MAAM,QAAQ,CAAE,QAAAC,EAAS,QAAAC,EAAS,QAAAC,EAAU,CAAC,CAAE,EAAG,CAGhD,IAAMC,EAAW,MAAM,KAAK,IAAI,IAAID,EAAQ,IAAKE,GAAWA,EAAO,OAAO,CAAC,CAAC,EACtEC,GAAU,MAAMC,EAAUP,CAAQ,GACrC,OAAQQ,GAAUN,GAAW,MAAQL,EAAWK,CAAO,IAAML,EAAWW,EAAM,OAAO,CAAC,EACtF,OAAQA,GAAU,CAACJ,EAAS,QAAUA,EAAS,SAASI,EAAM,OAAO,CAAC,EAEnEC,EAAoB,MAAM,QAAQ,IACtCH,EAAO,IAAI,MAAOE,GAAU,CAC1B,IAAME,EAAcC,EAAWH,CAAK,EAC9BI,EAAU,MAAMZ,EAAS,OAAO,EAAE,KAAKU,CAAW,EAAE,MAAMG,EAAGH,EAAY,YAAa,EAAK,CAAC,EAAE,QAAQ,EACtGI,EAAmBX,EAAQ,OAE7BS,EAAQ,OAAQG,GAAW,CACzB,IAAMC,EAAWlB,EAAmB,YAAaiB,EAAO,KAAK,EAC7D,OAAOZ,EAAQ,KACZE,GACCA,EAAO,UAAYG,EAAM,UACxBH,EAAO,MAAQ,MAAQA,EAAO,OAASW,EAAS,CAAC,KACjDX,EAAO,MAAQ,MAAQA,EAAO,OAASW,EAAS,CAAC,EACtD,CACF,CAAC,EATDJ,EAUJ,MAAO,CACL,GAAGJ,EACH,QAASM,EAAgB,IAAKC,IAAY,CACxC,IAAK,OAAO,YAAY,OAAO,QAAQP,EAAM,SAAS,EAAE,IAAI,CAAC,CAACS,CAAI,IAAM,CAACA,EAAMF,EAAOE,CAAI,CAAC,CAAC,CAAC,EAC7F,MAAO,OAAO,YAAY,OAAO,QAAQT,EAAM,WAAW,EAAE,IAAI,CAAC,CAACS,CAAI,IAAM,CAACA,EAAMF,EAAOE,CAAI,CAAC,CAAC,CAAC,CACnG,EAAE,CACJ,CACF,CAAC,CACH,EAEMC,EAAiBC,EAAoB,EACrCC,EAAW,MAAMpB,EACpB,OAAO,EACP,KAAKkB,EAAe,KAAK,EACzB,MAAML,EAAGK,EAAe,MAAM,QAASjB,CAAO,CAAC,EAC/C,QAAQ,EACL,CAAE,uBAAAoB,CAAuB,EAAID,EAAS,CAAC,GAAK,CAAC,EAE7CE,EAAS,CACb,YAAaD,GAA0B,KACvC,OAAQZ,CACV,EAEA,OAAAc,EAAM,UAAWtB,EAASC,EAASoB,CAAM,EAElCA,CACT,CACF,CAEF,CD7DA,OAAS,WAAAE,MAAe,0BACxB,OAAOC,MAAc,WAGrB,IAAMC,EAAMC,EACVC,EAAE,aACAC,EACAD,EAAE,OAAO,CACP,aAAcA,EAAE,OAAO,CACzB,CAAC,CACH,CACF,EAEME,EAAWC,EAAQC,EAASN,EAAI,YAAY,CAAC,EAG7CO,EAASC,EAAQ,CACrB,eAAgB,GAClB,CAAC,EAED,MAAMD,EAAO,SAAS,OAAO,eAAe,CAAC,EAG7CA,EAAO,IAAI,WAAY,CAACE,EAAKC,IAAQA,EAAI,KAAK,GAAG,EAAE,KAAK,CAAC,EACzDH,EAAO,IAAI,UAAW,CAACE,EAAKC,IAAQA,EAAI,KAAK,GAAG,EAAE,KAAK,CAAC,EAGxDH,EAAO,SAASI,EAA8B,CAC5C,OAAQ,QACR,YAAa,CACX,OAAQC,EAAgB,EACxB,cAAe,UAAa,CAC1B,aAAc,MAAMC,EAAmBT,CAAQ,CACjD,EACF,CACF,CAAC,EAED,MAAMG,EAAO,OAAO,CAAE,KAAMP,EAAI,KAAM,KAAMA,EAAI,IAAK,CAAC,EACtD,QAAQ,IAAI,iDAAiDA,EAAI,QAAQA,EAAI,MAAM","names":["z","fastify","fastifyTRPCPlugin","createAppRouter","eq","buildTable","buildInternalTables","getTables","getAddress","decodeDynamicField","createQueryAdapter","database","chainId","address","filters","tableIds","filter","tables","getTables","table","tablesWithRecords","sqliteTable","buildTable","records","eq","filteredRecords","record","keyTuple","name","internalTables","buildInternalTables","metadata","lastUpdatedBlockNumber","result","debug","drizzle","postgres","env","parseEnv","z","frontendEnvSchema","database","drizzle","postgres","server","fastify","req","res","fastifyTRPCPlugin","createAppRouter","createQueryAdapter"]}
|
|
@@ -1,3 +1,3 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import{
|
|
2
|
+
import{b as c,c as l}from"../chunk-2E5MDUA2.js";import"dotenv/config";import{z as n}from"zod";import{eq as b}from"drizzle-orm";import{createPublicClient as h,fallback as C,webSocket as T,http as g}from"viem";import{isDefined as H}from"@latticexyz/common/utils";import{combineLatest as _,filter as E,first as S}from"rxjs";import{drizzle as L}from"drizzle-orm/postgres-js";import R from"postgres";import{cleanDatabase as A,postgresStorage as k,schemaVersion as m}from"@latticexyz/store-sync/postgres";import{createStoreSync as B}from"@latticexyz/store-sync";var e=l(n.intersection(c,n.object({DATABASE_URL:n.string(),HEALTHCHECK_HOST:n.string().optional(),HEALTHCHECK_PORT:n.coerce.number().optional()}))),P=[e.RPC_WS_URL?T(e.RPC_WS_URL):void 0,e.RPC_HTTP_URL?g(e.RPC_HTTP_URL):void 0].filter(H),s=h({transport:C(P),pollingInterval:e.POLLING_INTERVAL}),O=await s.getChainId(),a=L(R(e.DATABASE_URL)),{storageAdapter:U,internalTables:d}=await k({database:a,publicClient:s}),p=e.START_BLOCK;try{let t=(await a.select().from(d.chain).where(b(d.chain.chainId,O)).execute())[0];t!=null&&(t.schemaVersion!=m?(console.log("schema version changed from",t.schemaVersion,"to",m,"cleaning database"),await A(a)):t.lastUpdatedBlockNumber!=null&&(console.log("resuming from block number",t.lastUpdatedBlockNumber+1n),p=t.lastUpdatedBlockNumber+1n))}catch{}var{latestBlockNumber$:v,storedBlockLogs$:f}=await B({storageAdapter:U,publicClient:s,startBlock:p,maxBlockRange:e.MAX_BLOCK_RANGE,address:e.STORE_ADDRESS});f.subscribe();var u=!1;_([v,f]).pipe(E(([o,{blockNumber:t}])=>o===t),S()).subscribe(()=>{u=!0,console.log("all caught up")});if(e.HEALTHCHECK_HOST!=null||e.HEALTHCHECK_PORT!=null){let{default:o}=await import("fastify"),t=o();t.get("/healthz",(i,r)=>r.code(200).send()),t.get("/readyz",(i,r)=>u?r.code(200).send("ready"):r.code(424).send("backfilling")),t.listen({host:e.HEALTHCHECK_HOST,port:e.HEALTHCHECK_PORT},(i,r)=>{console.log(`postgres indexer healthcheck server listening on ${r}`)})}
|
|
3
3
|
//# sourceMappingURL=postgres-indexer.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../bin/postgres-indexer.ts","../../src/postgres/createQueryAdapter.ts"],"sourcesContent":["#!/usr/bin/env node\nimport \"dotenv/config\";\nimport { z } from \"zod\";\nimport { DefaultLogger, eq } from \"drizzle-orm\";\nimport { createPublicClient, fallback, webSocket, http, Transport } from \"viem\";\nimport fastify from \"fastify\";\nimport { fastifyTRPCPlugin } from \"@trpc/server/adapters/fastify\";\nimport { AppRouter, createAppRouter } from \"@latticexyz/store-sync/trpc-indexer\";\nimport { createQueryAdapter } from \"../src/postgres/createQueryAdapter\";\nimport { isDefined } from \"@latticexyz/common/utils\";\nimport { combineLatest, filter, first } from \"rxjs\";\nimport { drizzle } from \"drizzle-orm/postgres-js\";\nimport postgres from \"postgres\";\nimport { cleanDatabase, postgresStorage, schemaVersion } from \"@latticexyz/store-sync/postgres\";\nimport { createStoreSync } from \"@latticexyz/store-sync\";\nimport { parseEnv } from \"./parseEnv\";\n\nconst env = parseEnv(\n z.object({\n DATABASE_URL: z.string(),\n })\n);\n\nconst transports: Transport[] = [\n // prefer WS when specified\n env.RPC_WS_URL ? webSocket(env.RPC_WS_URL) : undefined,\n // otherwise use or fallback to HTTP\n env.RPC_HTTP_URL ? http(env.RPC_HTTP_URL) : undefined,\n].filter(isDefined);\n\nconst publicClient = createPublicClient({\n transport: fallback(transports),\n pollingInterval: env.POLLING_INTERVAL,\n});\n\nconst chainId = await publicClient.getChainId();\nconst database = drizzle(postgres(env.DATABASE_URL), {\n logger: new DefaultLogger(),\n});\n\nconst { storageAdapter, internalTables } = await postgresStorage({ database, publicClient });\n\nlet startBlock = env.START_BLOCK;\n\n// Resume from latest block stored in DB. This will throw if the DB doesn't exist yet, so we wrap in a try/catch and ignore the error.\ntry {\n const currentChainStates = await database\n .select()\n .from(internalTables.chain)\n .where(eq(internalTables.chain.chainId, chainId))\n .execute();\n // TODO: replace this type workaround with `noUncheckedIndexedAccess: true` when we can fix all the issues related (https://github.com/latticexyz/mud/issues/1212)\n const currentChainState: (typeof currentChainStates)[number] | undefined = currentChainStates[0];\n\n if (currentChainState != null) {\n if (currentChainState.schemaVersion != schemaVersion) {\n console.log(\n \"schema version changed from\",\n currentChainState.schemaVersion,\n \"to\",\n schemaVersion,\n \"cleaning database\"\n );\n await cleanDatabase(database);\n } else if (currentChainState.lastUpdatedBlockNumber != null) {\n console.log(\"resuming from block number\", currentChainState.lastUpdatedBlockNumber + 1n);\n startBlock = currentChainState.lastUpdatedBlockNumber + 1n;\n }\n }\n} catch (error) {\n // ignore errors, this is optional\n}\n\nconst { latestBlockNumber$, storedBlockLogs$ } = await createStoreSync({\n storageAdapter,\n publicClient,\n startBlock,\n maxBlockRange: env.MAX_BLOCK_RANGE,\n});\n\nstoredBlockLogs$.subscribe();\n\nlet isCaughtUp = false;\ncombineLatest([latestBlockNumber$, storedBlockLogs$])\n .pipe(\n filter(\n ([latestBlockNumber, { blockNumber: lastBlockNumberProcessed }]) => latestBlockNumber === lastBlockNumberProcessed\n ),\n first()\n )\n .subscribe(() => {\n isCaughtUp = true;\n console.log(\"all caught up\");\n });\n\n// @see https://fastify.dev/docs/latest/\nconst server = fastify({\n maxParamLength: 5000,\n});\n\nawait server.register(import(\"@fastify/cors\"));\n\n// k8s healthchecks\nserver.get(\"/healthz\", (req, res) => res.code(200).send());\nserver.get(\"/readyz\", (req, res) => (isCaughtUp ? res.code(200).send(\"ready\") : res.code(424).send(\"backfilling\")));\n\n// @see https://trpc.io/docs/server/adapters/fastify\nserver.register(fastifyTRPCPlugin<AppRouter>, {\n prefix: \"/trpc\",\n trpcOptions: {\n router: createAppRouter(),\n createContext: async () => ({\n queryAdapter: await createQueryAdapter(database),\n }),\n },\n});\n\nawait server.listen({ host: env.HOST, port: env.PORT });\nconsole.log(`indexer server listening on http://${env.HOST}:${env.PORT}`);\n","import { eq } from \"drizzle-orm\";\nimport { PgDatabase } from \"drizzle-orm/pg-core\";\nimport { buildTable, buildInternalTables, getTables } from \"@latticexyz/store-sync/postgres\";\nimport { QueryAdapter } from \"@latticexyz/store-sync/trpc-indexer\";\nimport { debug } from \"../debug\";\nimport { getAddress } from \"viem\";\nimport { decodeDynamicField } from \"@latticexyz/protocol-parser\";\n\n/**\n * Creates a query adapter for the tRPC server/client to query data from Postgres.\n *\n * @param {PgDatabase<any>} database Postgres database object from Drizzle\n * @returns {Promise<QueryAdapter>} A set of methods used by tRPC endpoints.\n */\nexport async function createQueryAdapter(database: PgDatabase<any>): Promise<QueryAdapter> {\n const adapter: QueryAdapter = {\n async findAll({ chainId, address, filters = [] }) {\n // If _any_ filter has a table ID, this will filter down all data to just those tables. Which mean we can't yet mix table filters with key-only filters.\n // TODO: improve this so we can express this in the query (need to be able to query data across tables more easily)\n const tableIds = Array.from(new Set(filters.map((filter) => filter.tableId)));\n const tables = (await getTables(database))\n .filter((table) => address == null || getAddress(address) === getAddress(table.address))\n .filter((table) => !tableIds.length || tableIds.includes(table.tableId));\n\n const tablesWithRecords = await Promise.all(\n tables.map(async (table) => {\n const sqliteTable = buildTable(table);\n const records = await database.select().from(sqliteTable).where(eq(sqliteTable.__isDeleted, false)).execute();\n const filteredRecords = !filters.length\n ? records\n : records.filter((record) => {\n const keyTuple = decodeDynamicField(\"bytes32[]\", record.__key);\n return filters.some(\n (filter) =>\n filter.tableId === table.tableId &&\n (filter.key0 == null || filter.key0 === keyTuple[0]) &&\n (filter.key1 == null || filter.key1 === keyTuple[1])\n );\n });\n return {\n ...table,\n records: filteredRecords.map((record) => ({\n key: Object.fromEntries(Object.entries(table.keySchema).map(([name]) => [name, record[name]])),\n value: Object.fromEntries(Object.entries(table.valueSchema).map(([name]) => [name, record[name]])),\n })),\n };\n })\n );\n\n const internalTables = buildInternalTables();\n const metadata = await database\n .select()\n .from(internalTables.chain)\n .where(eq(internalTables.chain.chainId, chainId))\n .execute();\n const { lastUpdatedBlockNumber } = metadata[0] ?? {};\n\n const result = {\n blockNumber: lastUpdatedBlockNumber ?? null,\n tables: tablesWithRecords,\n };\n\n debug(\"findAll\", chainId, address, result);\n\n return result;\n },\n };\n return adapter;\n}\n"],"mappings":";gDACA,MAAO,gBACP,OAAS,KAAAA,MAAS,MAClB,OAAS,iBAAAC,EAAe,MAAAC,MAAU,cAClC,OAAS,sBAAAC,EAAoB,YAAAC,EAAU,aAAAC,EAAW,QAAAC,MAAuB,OACzE,OAAOC,MAAa,UACpB,OAAS,qBAAAC,MAAyB,gCAClC,OAAoB,mBAAAC,MAAuB,sCCP3C,OAAS,MAAAC,MAAU,cAEnB,OAAS,cAAAC,EAAY,uBAAAC,EAAqB,aAAAC,MAAiB,kCAG3D,OAAS,cAAAC,MAAkB,OAC3B,OAAS,sBAAAC,MAA0B,8BAQnC,eAAsBC,EAAmBC,EAAkD,CAqDzF,MApD8B,CAC5B,MAAM,QAAQ,CAAE,QAAAC,EAAS,QAAAC,EAAS,QAAAC,EAAU,CAAC,CAAE,EAAG,CAGhD,IAAMC,EAAW,MAAM,KAAK,IAAI,IAAID,EAAQ,IAAKE,GAAWA,EAAO,OAAO,CAAC,CAAC,EACtEC,GAAU,MAAMC,EAAUP,CAAQ,GACrC,OAAQQ,GAAUN,GAAW,MAAQL,EAAWK,CAAO,IAAML,EAAWW,EAAM,OAAO,CAAC,EACtF,OAAQA,GAAU,CAACJ,EAAS,QAAUA,EAAS,SAASI,EAAM,OAAO,CAAC,EAEnEC,EAAoB,MAAM,QAAQ,IACtCH,EAAO,IAAI,MAAOE,GAAU,CAC1B,IAAME,EAAcC,EAAWH,CAAK,EAC9BI,EAAU,MAAMZ,EAAS,OAAO,EAAE,KAAKU,CAAW,EAAE,MAAMG,EAAGH,EAAY,YAAa,EAAK,CAAC,EAAE,QAAQ,EACtGI,EAAmBX,EAAQ,OAE7BS,EAAQ,OAAQG,GAAW,CACzB,IAAMC,EAAWlB,EAAmB,YAAaiB,EAAO,KAAK,EAC7D,OAAOZ,EAAQ,KACZE,GACCA,EAAO,UAAYG,EAAM,UACxBH,EAAO,MAAQ,MAAQA,EAAO,OAASW,EAAS,CAAC,KACjDX,EAAO,MAAQ,MAAQA,EAAO,OAASW,EAAS,CAAC,EACtD,CACF,CAAC,EATDJ,EAUJ,MAAO,CACL,GAAGJ,EACH,QAASM,EAAgB,IAAKC,IAAY,CACxC,IAAK,OAAO,YAAY,OAAO,QAAQP,EAAM,SAAS,EAAE,IAAI,CAAC,CAACS,CAAI,IAAM,CAACA,EAAMF,EAAOE,CAAI,CAAC,CAAC,CAAC,EAC7F,MAAO,OAAO,YAAY,OAAO,QAAQT,EAAM,WAAW,EAAE,IAAI,CAAC,CAACS,CAAI,IAAM,CAACA,EAAMF,EAAOE,CAAI,CAAC,CAAC,CAAC,CACnG,EAAE,CACJ,CACF,CAAC,CACH,EAEMC,EAAiBC,EAAoB,EACrCC,EAAW,MAAMpB,EACpB,OAAO,EACP,KAAKkB,EAAe,KAAK,EACzB,MAAML,EAAGK,EAAe,MAAM,QAASjB,CAAO,CAAC,EAC/C,QAAQ,EACL,CAAE,uBAAAoB,CAAuB,EAAID,EAAS,CAAC,GAAK,CAAC,EAE7CE,EAAS,CACb,YAAaD,GAA0B,KACvC,OAAQZ,CACV,EAEA,OAAAc,EAAM,UAAWtB,EAASC,EAASoB,CAAM,EAElCA,CACT,CACF,CAEF,CD3DA,OAAS,aAAAE,MAAiB,2BAC1B,OAAS,iBAAAC,EAAe,UAAAC,EAAQ,SAAAC,MAAa,OAC7C,OAAS,WAAAC,MAAe,0BACxB,OAAOC,MAAc,WACrB,OAAS,iBAAAC,EAAe,mBAAAC,GAAiB,iBAAAC,MAAqB,kCAC9D,OAAS,mBAAAC,OAAuB,yBAGhC,IAAMC,EAAMC,EACVC,EAAE,OAAO,CACP,aAAcA,EAAE,OAAO,CACzB,CAAC,CACH,EAEMC,GAA0B,CAE9BH,EAAI,WAAaI,EAAUJ,EAAI,UAAU,EAAI,OAE7CA,EAAI,aAAeK,EAAKL,EAAI,YAAY,EAAI,MAC9C,EAAE,OAAOM,CAAS,EAEZC,EAAeC,EAAmB,CACtC,UAAWC,EAASN,EAAU,EAC9B,gBAAiBH,EAAI,gBACvB,CAAC,EAEKU,GAAU,MAAMH,EAAa,WAAW,EACxCI,EAAWC,EAAQC,EAASb,EAAI,YAAY,EAAG,CACnD,OAAQ,IAAIc,CACd,CAAC,EAEK,CAAE,eAAAC,GAAgB,eAAAC,CAAe,EAAI,MAAMC,GAAgB,CAAE,SAAAN,EAAU,aAAAJ,CAAa,CAAC,EAEvFW,EAAalB,EAAI,YAGrB,GAAI,CAOF,IAAMmB,GANqB,MAAMR,EAC9B,OAAO,EACP,KAAKK,EAAe,KAAK,EACzB,MAAMI,EAAGJ,EAAe,MAAM,QAASN,EAAO,CAAC,EAC/C,QAAQ,GAEmF,CAAC,EAE3FS,GAAqB,OACnBA,EAAkB,eAAiBE,GACrC,QAAQ,IACN,8BACAF,EAAkB,cAClB,KACAE,EACA,mBACF,EACA,MAAMC,EAAcX,CAAQ,GACnBQ,EAAkB,wBAA0B,OACrD,QAAQ,IAAI,6BAA8BA,EAAkB,uBAAyB,EAAE,EACvFD,EAAaC,EAAkB,uBAAyB,IAG9D,MAAE,CAEF,CAEA,GAAM,CAAE,mBAAAI,GAAoB,iBAAAC,CAAiB,EAAI,MAAMC,GAAgB,CACrE,eAAAV,GACA,aAAAR,EACA,WAAAW,EACA,cAAelB,EAAI,eACrB,CAAC,EAEDwB,EAAiB,UAAU,EAE3B,IAAIE,EAAa,GACjBC,EAAc,CAACJ,GAAoBC,CAAgB,CAAC,EACjD,KACCI,EACE,CAAC,CAACC,EAAmB,CAAE,YAAaC,CAAyB,CAAC,IAAMD,IAAsBC,CAC5F,EACAC,EAAM,CACR,EACC,UAAU,IAAM,CACfL,EAAa,GACb,QAAQ,IAAI,eAAe,CAC7B,CAAC,EAGH,IAAMM,EAASC,EAAQ,CACrB,eAAgB,GAClB,CAAC,EAED,MAAMD,EAAO,SAAS,OAAO,eAAe,CAAC,EAG7CA,EAAO,IAAI,WAAY,CAACE,EAAKC,IAAQA,EAAI,KAAK,GAAG,EAAE,KAAK,CAAC,EACzDH,EAAO,IAAI,UAAW,CAACE,EAAKC,IAAST,EAAaS,EAAI,KAAK,GAAG,EAAE,KAAK,OAAO,EAAIA,EAAI,KAAK,GAAG,EAAE,KAAK,aAAa,CAAE,EAGlHH,EAAO,SAASI,EAA8B,CAC5C,OAAQ,QACR,YAAa,CACX,OAAQC,EAAgB,EACxB,cAAe,UAAa,CAC1B,aAAc,MAAMC,EAAmB3B,CAAQ,CACjD,EACF,CACF,CAAC,EAED,MAAMqB,EAAO,OAAO,CAAE,KAAMhC,EAAI,KAAM,KAAMA,EAAI,IAAK,CAAC,EACtD,QAAQ,IAAI,sCAAsCA,EAAI,QAAQA,EAAI,MAAM","names":["z","DefaultLogger","eq","createPublicClient","fallback","webSocket","http","fastify","fastifyTRPCPlugin","createAppRouter","eq","buildTable","buildInternalTables","getTables","getAddress","decodeDynamicField","createQueryAdapter","database","chainId","address","filters","tableIds","filter","tables","getTables","table","tablesWithRecords","sqliteTable","buildTable","records","eq","filteredRecords","record","keyTuple","name","internalTables","buildInternalTables","metadata","lastUpdatedBlockNumber","result","debug","isDefined","combineLatest","filter","first","drizzle","postgres","cleanDatabase","postgresStorage","schemaVersion","createStoreSync","env","parseEnv","z","transports","webSocket","http","isDefined","publicClient","createPublicClient","fallback","chainId","database","drizzle","postgres","DefaultLogger","storageAdapter","internalTables","postgresStorage","startBlock","currentChainState","eq","schemaVersion","cleanDatabase","latestBlockNumber$","storedBlockLogs$","createStoreSync","isCaughtUp","combineLatest","filter","latestBlockNumber","lastBlockNumberProcessed","first","server","fastify","req","res","fastifyTRPCPlugin","createAppRouter","createQueryAdapter"]}
|
|
1
|
+
{"version":3,"sources":["../../bin/postgres-indexer.ts"],"sourcesContent":["#!/usr/bin/env node\nimport \"dotenv/config\";\nimport { z } from \"zod\";\nimport { eq } from \"drizzle-orm\";\nimport { createPublicClient, fallback, webSocket, http, Transport } from \"viem\";\nimport { isDefined } from \"@latticexyz/common/utils\";\nimport { combineLatest, filter, first } from \"rxjs\";\nimport { drizzle } from \"drizzle-orm/postgres-js\";\nimport postgres from \"postgres\";\nimport { cleanDatabase, postgresStorage, schemaVersion } from \"@latticexyz/store-sync/postgres\";\nimport { createStoreSync } from \"@latticexyz/store-sync\";\nimport { indexerEnvSchema, parseEnv } from \"./parseEnv\";\n\nconst env = parseEnv(\n z.intersection(\n indexerEnvSchema,\n z.object({\n DATABASE_URL: z.string(),\n HEALTHCHECK_HOST: z.string().optional(),\n HEALTHCHECK_PORT: z.coerce.number().optional(),\n })\n )\n);\n\nconst transports: Transport[] = [\n // prefer WS when specified\n env.RPC_WS_URL ? webSocket(env.RPC_WS_URL) : undefined,\n // otherwise use or fallback to HTTP\n env.RPC_HTTP_URL ? http(env.RPC_HTTP_URL) : undefined,\n].filter(isDefined);\n\nconst publicClient = createPublicClient({\n transport: fallback(transports),\n pollingInterval: env.POLLING_INTERVAL,\n});\n\nconst chainId = await publicClient.getChainId();\nconst database = drizzle(postgres(env.DATABASE_URL));\n\nconst { storageAdapter, internalTables } = await postgresStorage({ database, publicClient });\n\nlet startBlock = env.START_BLOCK;\n\n// Resume from latest block stored in DB. This will throw if the DB doesn't exist yet, so we wrap in a try/catch and ignore the error.\ntry {\n const currentChainStates = await database\n .select()\n .from(internalTables.chain)\n .where(eq(internalTables.chain.chainId, chainId))\n .execute();\n // TODO: replace this type workaround with `noUncheckedIndexedAccess: true` when we can fix all the issues related (https://github.com/latticexyz/mud/issues/1212)\n const currentChainState: (typeof currentChainStates)[number] | undefined = currentChainStates[0];\n\n if (currentChainState != null) {\n if (currentChainState.schemaVersion != schemaVersion) {\n console.log(\n \"schema version changed from\",\n currentChainState.schemaVersion,\n \"to\",\n schemaVersion,\n \"cleaning database\"\n );\n await cleanDatabase(database);\n } else if (currentChainState.lastUpdatedBlockNumber != null) {\n console.log(\"resuming from block number\", currentChainState.lastUpdatedBlockNumber + 1n);\n startBlock = currentChainState.lastUpdatedBlockNumber + 1n;\n }\n }\n} catch (error) {\n // ignore errors, this is optional\n}\n\nconst { latestBlockNumber$, storedBlockLogs$ } = await createStoreSync({\n storageAdapter,\n publicClient,\n startBlock,\n maxBlockRange: env.MAX_BLOCK_RANGE,\n address: env.STORE_ADDRESS,\n});\n\nstoredBlockLogs$.subscribe();\n\nlet isCaughtUp = false;\ncombineLatest([latestBlockNumber$, storedBlockLogs$])\n .pipe(\n filter(\n ([latestBlockNumber, { blockNumber: lastBlockNumberProcessed }]) => latestBlockNumber === lastBlockNumberProcessed\n ),\n first()\n )\n .subscribe(() => {\n isCaughtUp = true;\n console.log(\"all caught up\");\n });\n\nif (env.HEALTHCHECK_HOST != null || env.HEALTHCHECK_PORT != null) {\n const { default: fastify } = await import(\"fastify\");\n\n const server = fastify();\n\n // k8s healthchecks\n server.get(\"/healthz\", (req, res) => res.code(200).send());\n server.get(\"/readyz\", (req, res) => (isCaughtUp ? res.code(200).send(\"ready\") : res.code(424).send(\"backfilling\")));\n\n server.listen({ host: env.HEALTHCHECK_HOST, port: env.HEALTHCHECK_PORT }, (error, address) => {\n console.log(`postgres indexer healthcheck server listening on ${address}`);\n });\n}\n"],"mappings":";gDACA,MAAO,gBACP,OAAS,KAAAA,MAAS,MAClB,OAAS,MAAAC,MAAU,cACnB,OAAS,sBAAAC,EAAoB,YAAAC,EAAU,aAAAC,EAAW,QAAAC,MAAuB,OACzE,OAAS,aAAAC,MAAiB,2BAC1B,OAAS,iBAAAC,EAAe,UAAAC,EAAQ,SAAAC,MAAa,OAC7C,OAAS,WAAAC,MAAe,0BACxB,OAAOC,MAAc,WACrB,OAAS,iBAAAC,EAAe,mBAAAC,EAAiB,iBAAAC,MAAqB,kCAC9D,OAAS,mBAAAC,MAAuB,yBAGhC,IAAMC,EAAMC,EACVC,EAAE,aACAC,EACAD,EAAE,OAAO,CACP,aAAcA,EAAE,OAAO,EACvB,iBAAkBA,EAAE,OAAO,EAAE,SAAS,EACtC,iBAAkBA,EAAE,OAAO,OAAO,EAAE,SAAS,CAC/C,CAAC,CACH,CACF,EAEME,EAA0B,CAE9BJ,EAAI,WAAaK,EAAUL,EAAI,UAAU,EAAI,OAE7CA,EAAI,aAAeM,EAAKN,EAAI,YAAY,EAAI,MAC9C,EAAE,OAAOO,CAAS,EAEZC,EAAeC,EAAmB,CACtC,UAAWC,EAASN,CAAU,EAC9B,gBAAiBJ,EAAI,gBACvB,CAAC,EAEKW,EAAU,MAAMH,EAAa,WAAW,EACxCI,EAAWC,EAAQC,EAASd,EAAI,YAAY,CAAC,EAE7C,CAAE,eAAAe,EAAgB,eAAAC,CAAe,EAAI,MAAMC,EAAgB,CAAE,SAAAL,EAAU,aAAAJ,CAAa,CAAC,EAEvFU,EAAalB,EAAI,YAGrB,GAAI,CAOF,IAAMmB,GANqB,MAAMP,EAC9B,OAAO,EACP,KAAKI,EAAe,KAAK,EACzB,MAAMI,EAAGJ,EAAe,MAAM,QAASL,CAAO,CAAC,EAC/C,QAAQ,GAEmF,CAAC,EAE3FQ,GAAqB,OACnBA,EAAkB,eAAiBE,GACrC,QAAQ,IACN,8BACAF,EAAkB,cAClB,KACAE,EACA,mBACF,EACA,MAAMC,EAAcV,CAAQ,GACnBO,EAAkB,wBAA0B,OACrD,QAAQ,IAAI,6BAA8BA,EAAkB,uBAAyB,EAAE,EACvFD,EAAaC,EAAkB,uBAAyB,IAG9D,MAAE,CAEF,CAEA,GAAM,CAAE,mBAAAI,EAAoB,iBAAAC,CAAiB,EAAI,MAAMC,EAAgB,CACrE,eAAAV,EACA,aAAAP,EACA,WAAAU,EACA,cAAelB,EAAI,gBACnB,QAASA,EAAI,aACf,CAAC,EAEDwB,EAAiB,UAAU,EAE3B,IAAIE,EAAa,GACjBC,EAAc,CAACJ,EAAoBC,CAAgB,CAAC,EACjD,KACCI,EACE,CAAC,CAACC,EAAmB,CAAE,YAAaC,CAAyB,CAAC,IAAMD,IAAsBC,CAC5F,EACAC,EAAM,CACR,EACC,UAAU,IAAM,CACfL,EAAa,GACb,QAAQ,IAAI,eAAe,CAC7B,CAAC,EAEH,GAAI1B,EAAI,kBAAoB,MAAQA,EAAI,kBAAoB,KAAM,CAChE,GAAM,CAAE,QAASgC,CAAQ,EAAI,KAAM,QAAO,SAAS,EAE7CC,EAASD,EAAQ,EAGvBC,EAAO,IAAI,WAAY,CAACC,EAAKC,IAAQA,EAAI,KAAK,GAAG,EAAE,KAAK,CAAC,EACzDF,EAAO,IAAI,UAAW,CAACC,EAAKC,IAAST,EAAaS,EAAI,KAAK,GAAG,EAAE,KAAK,OAAO,EAAIA,EAAI,KAAK,GAAG,EAAE,KAAK,aAAa,CAAE,EAElHF,EAAO,OAAO,CAAE,KAAMjC,EAAI,iBAAkB,KAAMA,EAAI,gBAAiB,EAAG,CAACoC,EAAOC,IAAY,CAC5F,QAAQ,IAAI,oDAAoDA,GAAS,CAC3E,CAAC","names":["z","eq","createPublicClient","fallback","webSocket","http","isDefined","combineLatest","filter","first","drizzle","postgres","cleanDatabase","postgresStorage","schemaVersion","createStoreSync","env","parseEnv","z","indexerEnvSchema","transports","webSocket","http","isDefined","publicClient","createPublicClient","fallback","chainId","database","drizzle","postgres","storageAdapter","internalTables","postgresStorage","startBlock","currentChainState","eq","schemaVersion","cleanDatabase","latestBlockNumber$","storedBlockLogs$","createStoreSync","isCaughtUp","combineLatest","filter","latestBlockNumber","lastBlockNumberProcessed","first","fastify","server","req","res","error","address"]}
|
|
@@ -1,3 +1,3 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import{a as
|
|
2
|
+
import{a as y}from"../chunk-LOFAZSVJ.js";import{a as g,b as S,c as T}from"../chunk-2E5MDUA2.js";import"dotenv/config";import q from"node:fs";import{z as c}from"zod";import{eq as x}from"drizzle-orm";import{drizzle as D}from"drizzle-orm/better-sqlite3";import j from"better-sqlite3";import{createPublicClient as z,fallback as F,webSocket as H,http as M}from"viem";import V from"fastify";import{fastifyTRPCPlugin as $}from"@trpc/server/adapters/fastify";import{createAppRouter as W}from"@latticexyz/store-sync/trpc-indexer";import{chainState as _,schemaVersion as E,syncToSqlite as G}from"@latticexyz/store-sync/sqlite";import{eq as k}from"drizzle-orm";import{buildTable as v,chainState as R,getTables as Q}from"@latticexyz/store-sync/sqlite";import{getAddress as A}from"viem";import{decodeDynamicField as U}from"@latticexyz/protocol-parser";async function L(o){return{async findAll({chainId:d,address:l,filters:m=[]}){let u=Array.from(new Set(m.map(r=>r.tableId))),N=Q(o).filter(r=>l==null||A(l)===A(r.address)).filter(r=>!u.length||u.includes(r.tableId)).map(r=>{let b=v(r),h=o.select().from(b).where(k(b.__isDeleted,!1)).all(),w=m.length?h.filter(i=>{let n=U("bytes32[]",i.__key);return m.some(a=>a.tableId===r.tableId&&(a.key0==null||a.key0===n[0])&&(a.key1==null||a.key1===n[1]))}):h;return{...r,records:w.map(i=>({key:Object.fromEntries(Object.entries(r.keySchema).map(([n])=>[n,i[n]])),value:Object.fromEntries(Object.entries(r.valueSchema).map(([n])=>[n,i[n]]))}))}}),O=o.select().from(R).where(k(R.chainId,d)).all(),{lastUpdatedBlockNumber:B}=O[0]??{},f={blockNumber:B??null,tables:N};return y("findAll",d,l,f),f}}}import{isDefined as K}from"@latticexyz/common/utils";import{combineLatest as X,filter as J,first as Y}from"rxjs";var t=T(c.intersection(c.intersection(S,g),c.object({SQLITE_FILENAME:c.string().default("indexer.db")}))),Z=[t.RPC_WS_URL?H(t.RPC_WS_URL):void 0,t.RPC_HTTP_URL?M(t.RPC_HTTP_URL):void 0].filter(K),I=z({transport:F(Z),pollingInterval:t.POLLING_INTERVAL}),ee=await I.getChainId(),p=D(new j(t.SQLITE_FILENAME)),C=t.START_BLOCK;try{let e=p.select().from(_).where(x(_.chainId,ee)).all()[0];e!=null&&(e.schemaVersion!=E?(console.log("schema version changed from",e.schemaVersion,"to",E,"recreating database"),q.truncateSync(t.SQLITE_FILENAME)):e.lastUpdatedBlockNumber!=null&&(console.log("resuming from block number",e.lastUpdatedBlockNumber+1n),C=e.lastUpdatedBlockNumber+1n))}catch{}var{latestBlockNumber$:te,storedBlockLogs$:re}=await G({database:p,publicClient:I,startBlock:C,maxBlockRange:t.MAX_BLOCK_RANGE,address:t.STORE_ADDRESS}),P=!1;X([te,re]).pipe(J(([o,{blockNumber:e}])=>o===e),Y()).subscribe(()=>{P=!0,console.log("all caught up")});var s=V({maxParamLength:5e3});await s.register(import("@fastify/cors"));s.get("/healthz",(o,e)=>e.code(200).send());s.get("/readyz",(o,e)=>P?e.code(200).send("ready"):e.code(424).send("backfilling"));s.register($,{prefix:"/trpc",trpcOptions:{router:W(),createContext:async()=>({queryAdapter:await L(p)})}});await s.listen({host:t.HOST,port:t.PORT});console.log(`sqlite indexer frontend listening on http://${t.HOST}:${t.PORT}`);
|
|
3
3
|
//# sourceMappingURL=sqlite-indexer.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../bin/sqlite-indexer.ts","../../src/sqlite/createQueryAdapter.ts"],"sourcesContent":["#!/usr/bin/env node\nimport \"dotenv/config\";\nimport fs from \"node:fs\";\nimport { z } from \"zod\";\nimport { eq } from \"drizzle-orm\";\nimport { drizzle } from \"drizzle-orm/better-sqlite3\";\nimport Database from \"better-sqlite3\";\nimport { createPublicClient, fallback, webSocket, http, Transport } from \"viem\";\nimport fastify from \"fastify\";\nimport { fastifyTRPCPlugin } from \"@trpc/server/adapters/fastify\";\nimport { AppRouter, createAppRouter } from \"@latticexyz/store-sync/trpc-indexer\";\nimport { chainState, schemaVersion, syncToSqlite } from \"@latticexyz/store-sync/sqlite\";\nimport { createQueryAdapter } from \"../src/sqlite/createQueryAdapter\";\nimport { isDefined } from \"@latticexyz/common/utils\";\nimport { combineLatest, filter, first } from \"rxjs\";\nimport { parseEnv } from \"./parseEnv\";\n\nconst env = parseEnv(\n z.object({\n SQLITE_FILENAME: z.string().default(\"indexer.db\"),\n })\n);\n\nconst transports: Transport[] = [\n // prefer WS when specified\n env.RPC_WS_URL ? webSocket(env.RPC_WS_URL) : undefined,\n // otherwise use or fallback to HTTP\n env.RPC_HTTP_URL ? http(env.RPC_HTTP_URL) : undefined,\n].filter(isDefined);\n\nconst publicClient = createPublicClient({\n transport: fallback(transports),\n pollingInterval: env.POLLING_INTERVAL,\n});\n\nconst chainId = await publicClient.getChainId();\nconst database = drizzle(new Database(env.SQLITE_FILENAME));\n\nlet startBlock = env.START_BLOCK;\n\n// Resume from latest block stored in DB. This will throw if the DB doesn't exist yet, so we wrap in a try/catch and ignore the error.\ntry {\n const currentChainStates = database.select().from(chainState).where(eq(chainState.chainId, chainId)).all();\n // TODO: replace this type workaround with `noUncheckedIndexedAccess: true` when we can fix all the issues related (https://github.com/latticexyz/mud/issues/1212)\n const currentChainState: (typeof currentChainStates)[number] | undefined = currentChainStates[0];\n\n if (currentChainState != null) {\n if (currentChainState.schemaVersion != schemaVersion) {\n console.log(\n \"schema version changed from\",\n currentChainState.schemaVersion,\n \"to\",\n schemaVersion,\n \"recreating database\"\n );\n fs.truncateSync(env.SQLITE_FILENAME);\n } else if (currentChainState.lastUpdatedBlockNumber != null) {\n console.log(\"resuming from block number\", currentChainState.lastUpdatedBlockNumber + 1n);\n startBlock = currentChainState.lastUpdatedBlockNumber + 1n;\n }\n }\n} catch (error) {\n // ignore errors, this is optional\n}\n\nconst { latestBlockNumber$, storedBlockLogs$ } = await syncToSqlite({\n database,\n publicClient,\n startBlock,\n maxBlockRange: env.MAX_BLOCK_RANGE,\n});\n\nlet isCaughtUp = false;\ncombineLatest([latestBlockNumber$, storedBlockLogs$])\n .pipe(\n filter(\n ([latestBlockNumber, { blockNumber: lastBlockNumberProcessed }]) => latestBlockNumber === lastBlockNumberProcessed\n ),\n first()\n )\n .subscribe(() => {\n isCaughtUp = true;\n console.log(\"all caught up\");\n });\n\n// @see https://fastify.dev/docs/latest/\nconst server = fastify({\n maxParamLength: 5000,\n});\n\nawait server.register(import(\"@fastify/cors\"));\n\n// k8s healthchecks\nserver.get(\"/healthz\", (req, res) => res.code(200).send());\nserver.get(\"/readyz\", (req, res) => (isCaughtUp ? res.code(200).send(\"ready\") : res.code(424).send(\"backfilling\")));\n\n// @see https://trpc.io/docs/server/adapters/fastify\nserver.register(fastifyTRPCPlugin<AppRouter>, {\n prefix: \"/trpc\",\n trpcOptions: {\n router: createAppRouter(),\n createContext: async () => ({\n queryAdapter: await createQueryAdapter(database),\n }),\n },\n});\n\nawait server.listen({ host: env.HOST, port: env.PORT });\nconsole.log(`indexer server listening on http://${env.HOST}:${env.PORT}`);\n","import { eq } from \"drizzle-orm\";\nimport { BaseSQLiteDatabase } from \"drizzle-orm/sqlite-core\";\nimport { buildTable, chainState, getTables } from \"@latticexyz/store-sync/sqlite\";\nimport { QueryAdapter } from \"@latticexyz/store-sync/trpc-indexer\";\nimport { debug } from \"../debug\";\nimport { getAddress } from \"viem\";\nimport { decodeDynamicField } from \"@latticexyz/protocol-parser\";\n\n/**\n * Creates a storage adapter for the tRPC server/client to query data from SQLite.\n *\n * @param {BaseSQLiteDatabase<\"sync\", any>} database SQLite database object from Drizzle\n * @returns {Promise<QueryAdapter>} A set of methods used by tRPC endpoints.\n */\nexport async function createQueryAdapter(database: BaseSQLiteDatabase<\"sync\", any>): Promise<QueryAdapter> {\n const adapter: QueryAdapter = {\n async findAll({ chainId, address, filters = [] }) {\n // If _any_ filter has a table ID, this will filter down all data to just those tables. Which mean we can't yet mix table filters with key-only filters.\n // TODO: improve this so we can express this in the query (need to be able to query data across tables more easily)\n const tableIds = Array.from(new Set(filters.map((filter) => filter.tableId)));\n const tables = getTables(database)\n .filter((table) => address == null || getAddress(address) === getAddress(table.address))\n .filter((table) => !tableIds.length || tableIds.includes(table.tableId));\n\n const tablesWithRecords = tables.map((table) => {\n const sqliteTable = buildTable(table);\n const records = database.select().from(sqliteTable).where(eq(sqliteTable.__isDeleted, false)).all();\n const filteredRecords = !filters.length\n ? records\n : records.filter((record) => {\n const keyTuple = decodeDynamicField(\"bytes32[]\", record.__key);\n return filters.some(\n (filter) =>\n filter.tableId === table.tableId &&\n (filter.key0 == null || filter.key0 === keyTuple[0]) &&\n (filter.key1 == null || filter.key1 === keyTuple[1])\n );\n });\n return {\n ...table,\n records: filteredRecords.map((record) => ({\n key: Object.fromEntries(Object.entries(table.keySchema).map(([name]) => [name, record[name]])),\n value: Object.fromEntries(Object.entries(table.valueSchema).map(([name]) => [name, record[name]])),\n })),\n };\n });\n\n const metadata = database.select().from(chainState).where(eq(chainState.chainId, chainId)).all();\n const { lastUpdatedBlockNumber } = metadata[0] ?? {};\n\n const result = {\n blockNumber: lastUpdatedBlockNumber ?? null,\n tables: tablesWithRecords,\n };\n\n debug(\"findAll\", chainId, address, result);\n\n return result;\n },\n };\n return adapter;\n}\n"],"mappings":";gDACA,MAAO,gBACP,OAAOA,MAAQ,UACf,OAAS,KAAAC,MAAS,MAClB,OAAS,MAAAC,MAAU,cACnB,OAAS,WAAAC,MAAe,6BACxB,OAAOC,MAAc,iBACrB,OAAS,sBAAAC,EAAoB,YAAAC,EAAU,aAAAC,EAAW,QAAAC,MAAuB,OACzE,OAAOC,MAAa,UACpB,OAAS,qBAAAC,MAAyB,gCAClC,OAAoB,mBAAAC,MAAuB,sCAC3C,OAAS,cAAAC,EAAY,iBAAAC,EAAe,gBAAAC,MAAoB,gCCXxD,OAAS,MAAAC,MAAU,cAEnB,OAAS,cAAAC,EAAY,cAAAC,EAAY,aAAAC,MAAiB,gCAGlD,OAAS,cAAAC,MAAkB,OAC3B,OAAS,sBAAAC,MAA0B,8BAQnC,eAAsBC,EAAmBC,EAAkE,CA8CzG,MA7C8B,CAC5B,MAAM,QAAQ,CAAE,QAAAC,EAAS,QAAAC,EAAS,QAAAC,EAAU,CAAC,CAAE,EAAG,CAGhD,IAAMC,EAAW,MAAM,KAAK,IAAI,IAAID,EAAQ,IAAKE,GAAWA,EAAO,OAAO,CAAC,CAAC,EAKtEC,EAJSC,EAAUP,CAAQ,EAC9B,OAAQQ,GAAUN,GAAW,MAAQL,EAAWK,CAAO,IAAML,EAAWW,EAAM,OAAO,CAAC,EACtF,OAAQA,GAAU,CAACJ,EAAS,QAAUA,EAAS,SAASI,EAAM,OAAO,CAAC,EAExC,IAAKA,GAAU,CAC9C,IAAMC,EAAcC,EAAWF,CAAK,EAC9BG,EAAUX,EAAS,OAAO,EAAE,KAAKS,CAAW,EAAE,MAAMG,EAAGH,EAAY,YAAa,EAAK,CAAC,EAAE,IAAI,EAC5FI,EAAmBV,EAAQ,OAE7BQ,EAAQ,OAAQG,GAAW,CACzB,IAAMC,EAAWjB,EAAmB,YAAagB,EAAO,KAAK,EAC7D,OAAOX,EAAQ,KACZE,GACCA,EAAO,UAAYG,EAAM,UACxBH,EAAO,MAAQ,MAAQA,EAAO,OAASU,EAAS,CAAC,KACjDV,EAAO,MAAQ,MAAQA,EAAO,OAASU,EAAS,CAAC,EACtD,CACF,CAAC,EATDJ,EAUJ,MAAO,CACL,GAAGH,EACH,QAASK,EAAgB,IAAKC,IAAY,CACxC,IAAK,OAAO,YAAY,OAAO,QAAQN,EAAM,SAAS,EAAE,IAAI,CAAC,CAACQ,CAAI,IAAM,CAACA,EAAMF,EAAOE,CAAI,CAAC,CAAC,CAAC,EAC7F,MAAO,OAAO,YAAY,OAAO,QAAQR,EAAM,WAAW,EAAE,IAAI,CAAC,CAACQ,CAAI,IAAM,CAACA,EAAMF,EAAOE,CAAI,CAAC,CAAC,CAAC,CACnG,EAAE,CACJ,CACF,CAAC,EAEKC,EAAWjB,EAAS,OAAO,EAAE,KAAKkB,CAAU,EAAE,MAAMN,EAAGM,EAAW,QAASjB,CAAO,CAAC,EAAE,IAAI,EACzF,CAAE,uBAAAkB,CAAuB,EAAIF,EAAS,CAAC,GAAK,CAAC,EAE7CG,EAAS,CACb,YAAaD,GAA0B,KACvC,OAAQb,CACV,EAEA,OAAAe,EAAM,UAAWpB,EAASC,EAASkB,CAAM,EAElCA,CACT,CACF,CAEF,CDhDA,OAAS,aAAAE,MAAiB,2BAC1B,OAAS,iBAAAC,EAAe,UAAAC,EAAQ,SAAAC,MAAa,OAG7C,IAAMC,EAAMC,EACVC,EAAE,OAAO,CACP,gBAAiBA,EAAE,OAAO,EAAE,QAAQ,YAAY,CAClD,CAAC,CACH,EAEMC,EAA0B,CAE9BH,EAAI,WAAaI,EAAUJ,EAAI,UAAU,EAAI,OAE7CA,EAAI,aAAeK,EAAKL,EAAI,YAAY,EAAI,MAC9C,EAAE,OAAOM,CAAS,EAEZC,EAAeC,EAAmB,CACtC,UAAWC,EAASN,CAAU,EAC9B,gBAAiBH,EAAI,gBACvB,CAAC,EAEKU,EAAU,MAAMH,EAAa,WAAW,EACxCI,EAAWC,EAAQ,IAAIC,EAASb,EAAI,eAAe,CAAC,EAEtDc,EAAad,EAAI,YAGrB,GAAI,CAGF,IAAMe,EAFqBJ,EAAS,OAAO,EAAE,KAAKK,CAAU,EAAE,MAAMC,EAAGD,EAAW,QAASN,CAAO,CAAC,EAAE,IAAI,EAEX,CAAC,EAE3FK,GAAqB,OACnBA,EAAkB,eAAiBG,GACrC,QAAQ,IACN,8BACAH,EAAkB,cAClB,KACAG,EACA,qBACF,EACAC,EAAG,aAAanB,EAAI,eAAe,GAC1Be,EAAkB,wBAA0B,OACrD,QAAQ,IAAI,6BAA8BA,EAAkB,uBAAyB,EAAE,EACvFD,EAAaC,EAAkB,uBAAyB,IAG9D,MAAE,CAEF,CAEA,GAAM,CAAE,mBAAAK,EAAoB,iBAAAC,EAAiB,EAAI,MAAMC,EAAa,CAClE,SAAAX,EACA,aAAAJ,EACA,WAAAO,EACA,cAAed,EAAI,eACrB,CAAC,EAEGuB,EAAa,GACjBC,EAAc,CAACJ,EAAoBC,EAAgB,CAAC,EACjD,KACCI,EACE,CAAC,CAACC,EAAmB,CAAE,YAAaC,CAAyB,CAAC,IAAMD,IAAsBC,CAC5F,EACAC,EAAM,CACR,EACC,UAAU,IAAM,CACfL,EAAa,GACb,QAAQ,IAAI,eAAe,CAC7B,CAAC,EAGH,IAAMM,EAASC,EAAQ,CACrB,eAAgB,GAClB,CAAC,EAED,MAAMD,EAAO,SAAS,OAAO,eAAe,CAAC,EAG7CA,EAAO,IAAI,WAAY,CAACE,EAAKC,IAAQA,EAAI,KAAK,GAAG,EAAE,KAAK,CAAC,EACzDH,EAAO,IAAI,UAAW,CAACE,EAAKC,IAAST,EAAaS,EAAI,KAAK,GAAG,EAAE,KAAK,OAAO,EAAIA,EAAI,KAAK,GAAG,EAAE,KAAK,aAAa,CAAE,EAGlHH,EAAO,SAASI,EAA8B,CAC5C,OAAQ,QACR,YAAa,CACX,OAAQC,EAAgB,EACxB,cAAe,UAAa,CAC1B,aAAc,MAAMC,EAAmBxB,CAAQ,CACjD,EACF,CACF,CAAC,EAED,MAAMkB,EAAO,OAAO,CAAE,KAAM7B,EAAI,KAAM,KAAMA,EAAI,IAAK,CAAC,EACtD,QAAQ,IAAI,sCAAsCA,EAAI,QAAQA,EAAI,MAAM","names":["fs","z","eq","drizzle","Database","createPublicClient","fallback","webSocket","http","fastify","fastifyTRPCPlugin","createAppRouter","chainState","schemaVersion","syncToSqlite","eq","buildTable","chainState","getTables","getAddress","decodeDynamicField","createQueryAdapter","database","chainId","address","filters","tableIds","filter","tablesWithRecords","getTables","table","sqliteTable","buildTable","records","eq","filteredRecords","record","keyTuple","name","metadata","chainState","lastUpdatedBlockNumber","result","debug","isDefined","combineLatest","filter","first","env","parseEnv","z","transports","webSocket","http","isDefined","publicClient","createPublicClient","fallback","chainId","database","drizzle","Database","startBlock","currentChainState","chainState","eq","schemaVersion","fs","latestBlockNumber$","storedBlockLogs$","syncToSqlite","isCaughtUp","combineLatest","filter","latestBlockNumber","lastBlockNumberProcessed","first","server","fastify","req","res","fastifyTRPCPlugin","createAppRouter","createQueryAdapter"]}
|
|
1
|
+
{"version":3,"sources":["../../bin/sqlite-indexer.ts","../../src/sqlite/createQueryAdapter.ts"],"sourcesContent":["#!/usr/bin/env node\nimport \"dotenv/config\";\nimport fs from \"node:fs\";\nimport { z } from \"zod\";\nimport { eq } from \"drizzle-orm\";\nimport { drizzle } from \"drizzle-orm/better-sqlite3\";\nimport Database from \"better-sqlite3\";\nimport { createPublicClient, fallback, webSocket, http, Transport } from \"viem\";\nimport fastify from \"fastify\";\nimport { fastifyTRPCPlugin } from \"@trpc/server/adapters/fastify\";\nimport { AppRouter, createAppRouter } from \"@latticexyz/store-sync/trpc-indexer\";\nimport { chainState, schemaVersion, syncToSqlite } from \"@latticexyz/store-sync/sqlite\";\nimport { createQueryAdapter } from \"../src/sqlite/createQueryAdapter\";\nimport { isDefined } from \"@latticexyz/common/utils\";\nimport { combineLatest, filter, first } from \"rxjs\";\nimport { frontendEnvSchema, indexerEnvSchema, parseEnv } from \"./parseEnv\";\n\nconst env = parseEnv(\n z.intersection(\n z.intersection(indexerEnvSchema, frontendEnvSchema),\n z.object({\n SQLITE_FILENAME: z.string().default(\"indexer.db\"),\n })\n )\n);\n\nconst transports: Transport[] = [\n // prefer WS when specified\n env.RPC_WS_URL ? webSocket(env.RPC_WS_URL) : undefined,\n // otherwise use or fallback to HTTP\n env.RPC_HTTP_URL ? http(env.RPC_HTTP_URL) : undefined,\n].filter(isDefined);\n\nconst publicClient = createPublicClient({\n transport: fallback(transports),\n pollingInterval: env.POLLING_INTERVAL,\n});\n\nconst chainId = await publicClient.getChainId();\nconst database = drizzle(new Database(env.SQLITE_FILENAME));\n\nlet startBlock = env.START_BLOCK;\n\n// Resume from latest block stored in DB. This will throw if the DB doesn't exist yet, so we wrap in a try/catch and ignore the error.\ntry {\n const currentChainStates = database.select().from(chainState).where(eq(chainState.chainId, chainId)).all();\n // TODO: replace this type workaround with `noUncheckedIndexedAccess: true` when we can fix all the issues related (https://github.com/latticexyz/mud/issues/1212)\n const currentChainState: (typeof currentChainStates)[number] | undefined = currentChainStates[0];\n\n if (currentChainState != null) {\n if (currentChainState.schemaVersion != schemaVersion) {\n console.log(\n \"schema version changed from\",\n currentChainState.schemaVersion,\n \"to\",\n schemaVersion,\n \"recreating database\"\n );\n fs.truncateSync(env.SQLITE_FILENAME);\n } else if (currentChainState.lastUpdatedBlockNumber != null) {\n console.log(\"resuming from block number\", currentChainState.lastUpdatedBlockNumber + 1n);\n startBlock = currentChainState.lastUpdatedBlockNumber + 1n;\n }\n }\n} catch (error) {\n // ignore errors, this is optional\n}\n\nconst { latestBlockNumber$, storedBlockLogs$ } = await syncToSqlite({\n database,\n publicClient,\n startBlock,\n maxBlockRange: env.MAX_BLOCK_RANGE,\n address: env.STORE_ADDRESS,\n});\n\nlet isCaughtUp = false;\ncombineLatest([latestBlockNumber$, storedBlockLogs$])\n .pipe(\n filter(\n ([latestBlockNumber, { blockNumber: lastBlockNumberProcessed }]) => latestBlockNumber === lastBlockNumberProcessed\n ),\n first()\n )\n .subscribe(() => {\n isCaughtUp = true;\n console.log(\"all caught up\");\n });\n\n// @see https://fastify.dev/docs/latest/\nconst server = fastify({\n maxParamLength: 5000,\n});\n\nawait server.register(import(\"@fastify/cors\"));\n\n// k8s healthchecks\nserver.get(\"/healthz\", (req, res) => res.code(200).send());\nserver.get(\"/readyz\", (req, res) => (isCaughtUp ? res.code(200).send(\"ready\") : res.code(424).send(\"backfilling\")));\n\n// @see https://trpc.io/docs/server/adapters/fastify\nserver.register(fastifyTRPCPlugin<AppRouter>, {\n prefix: \"/trpc\",\n trpcOptions: {\n router: createAppRouter(),\n createContext: async () => ({\n queryAdapter: await createQueryAdapter(database),\n }),\n },\n});\n\nawait server.listen({ host: env.HOST, port: env.PORT });\nconsole.log(`sqlite indexer frontend listening on http://${env.HOST}:${env.PORT}`);\n","import { eq } from \"drizzle-orm\";\nimport { BaseSQLiteDatabase } from \"drizzle-orm/sqlite-core\";\nimport { buildTable, chainState, getTables } from \"@latticexyz/store-sync/sqlite\";\nimport { QueryAdapter } from \"@latticexyz/store-sync/trpc-indexer\";\nimport { debug } from \"../debug\";\nimport { getAddress } from \"viem\";\nimport { decodeDynamicField } from \"@latticexyz/protocol-parser\";\n\n/**\n * Creates a storage adapter for the tRPC server/client to query data from SQLite.\n *\n * @param {BaseSQLiteDatabase<\"sync\", any>} database SQLite database object from Drizzle\n * @returns {Promise<QueryAdapter>} A set of methods used by tRPC endpoints.\n */\nexport async function createQueryAdapter(database: BaseSQLiteDatabase<\"sync\", any>): Promise<QueryAdapter> {\n const adapter: QueryAdapter = {\n async findAll({ chainId, address, filters = [] }) {\n // If _any_ filter has a table ID, this will filter down all data to just those tables. Which mean we can't yet mix table filters with key-only filters.\n // TODO: improve this so we can express this in the query (need to be able to query data across tables more easily)\n const tableIds = Array.from(new Set(filters.map((filter) => filter.tableId)));\n const tables = getTables(database)\n .filter((table) => address == null || getAddress(address) === getAddress(table.address))\n .filter((table) => !tableIds.length || tableIds.includes(table.tableId));\n\n const tablesWithRecords = tables.map((table) => {\n const sqliteTable = buildTable(table);\n const records = database.select().from(sqliteTable).where(eq(sqliteTable.__isDeleted, false)).all();\n const filteredRecords = !filters.length\n ? records\n : records.filter((record) => {\n const keyTuple = decodeDynamicField(\"bytes32[]\", record.__key);\n return filters.some(\n (filter) =>\n filter.tableId === table.tableId &&\n (filter.key0 == null || filter.key0 === keyTuple[0]) &&\n (filter.key1 == null || filter.key1 === keyTuple[1])\n );\n });\n return {\n ...table,\n records: filteredRecords.map((record) => ({\n key: Object.fromEntries(Object.entries(table.keySchema).map(([name]) => [name, record[name]])),\n value: Object.fromEntries(Object.entries(table.valueSchema).map(([name]) => [name, record[name]])),\n })),\n };\n });\n\n const metadata = database.select().from(chainState).where(eq(chainState.chainId, chainId)).all();\n const { lastUpdatedBlockNumber } = metadata[0] ?? {};\n\n const result = {\n blockNumber: lastUpdatedBlockNumber ?? null,\n tables: tablesWithRecords,\n };\n\n debug(\"findAll\", chainId, address, result);\n\n return result;\n },\n };\n return adapter;\n}\n"],"mappings":";gGACA,MAAO,gBACP,OAAOA,MAAQ,UACf,OAAS,KAAAC,MAAS,MAClB,OAAS,MAAAC,MAAU,cACnB,OAAS,WAAAC,MAAe,6BACxB,OAAOC,MAAc,iBACrB,OAAS,sBAAAC,EAAoB,YAAAC,EAAU,aAAAC,EAAW,QAAAC,MAAuB,OACzE,OAAOC,MAAa,UACpB,OAAS,qBAAAC,MAAyB,gCAClC,OAAoB,mBAAAC,MAAuB,sCAC3C,OAAS,cAAAC,EAAY,iBAAAC,EAAe,gBAAAC,MAAoB,gCCXxD,OAAS,MAAAC,MAAU,cAEnB,OAAS,cAAAC,EAAY,cAAAC,EAAY,aAAAC,MAAiB,gCAGlD,OAAS,cAAAC,MAAkB,OAC3B,OAAS,sBAAAC,MAA0B,8BAQnC,eAAsBC,EAAmBC,EAAkE,CA8CzG,MA7C8B,CAC5B,MAAM,QAAQ,CAAE,QAAAC,EAAS,QAAAC,EAAS,QAAAC,EAAU,CAAC,CAAE,EAAG,CAGhD,IAAMC,EAAW,MAAM,KAAK,IAAI,IAAID,EAAQ,IAAKE,GAAWA,EAAO,OAAO,CAAC,CAAC,EAKtEC,EAJSC,EAAUP,CAAQ,EAC9B,OAAQQ,GAAUN,GAAW,MAAQL,EAAWK,CAAO,IAAML,EAAWW,EAAM,OAAO,CAAC,EACtF,OAAQA,GAAU,CAACJ,EAAS,QAAUA,EAAS,SAASI,EAAM,OAAO,CAAC,EAExC,IAAKA,GAAU,CAC9C,IAAMC,EAAcC,EAAWF,CAAK,EAC9BG,EAAUX,EAAS,OAAO,EAAE,KAAKS,CAAW,EAAE,MAAMG,EAAGH,EAAY,YAAa,EAAK,CAAC,EAAE,IAAI,EAC5FI,EAAmBV,EAAQ,OAE7BQ,EAAQ,OAAQG,GAAW,CACzB,IAAMC,EAAWjB,EAAmB,YAAagB,EAAO,KAAK,EAC7D,OAAOX,EAAQ,KACZE,GACCA,EAAO,UAAYG,EAAM,UACxBH,EAAO,MAAQ,MAAQA,EAAO,OAASU,EAAS,CAAC,KACjDV,EAAO,MAAQ,MAAQA,EAAO,OAASU,EAAS,CAAC,EACtD,CACF,CAAC,EATDJ,EAUJ,MAAO,CACL,GAAGH,EACH,QAASK,EAAgB,IAAKC,IAAY,CACxC,IAAK,OAAO,YAAY,OAAO,QAAQN,EAAM,SAAS,EAAE,IAAI,CAAC,CAACQ,CAAI,IAAM,CAACA,EAAMF,EAAOE,CAAI,CAAC,CAAC,CAAC,EAC7F,MAAO,OAAO,YAAY,OAAO,QAAQR,EAAM,WAAW,EAAE,IAAI,CAAC,CAACQ,CAAI,IAAM,CAACA,EAAMF,EAAOE,CAAI,CAAC,CAAC,CAAC,CACnG,EAAE,CACJ,CACF,CAAC,EAEKC,EAAWjB,EAAS,OAAO,EAAE,KAAKkB,CAAU,EAAE,MAAMN,EAAGM,EAAW,QAASjB,CAAO,CAAC,EAAE,IAAI,EACzF,CAAE,uBAAAkB,CAAuB,EAAIF,EAAS,CAAC,GAAK,CAAC,EAE7CG,EAAS,CACb,YAAaD,GAA0B,KACvC,OAAQb,CACV,EAEA,OAAAe,EAAM,UAAWpB,EAASC,EAASkB,CAAM,EAElCA,CACT,CACF,CAEF,CDhDA,OAAS,aAAAE,MAAiB,2BAC1B,OAAS,iBAAAC,EAAe,UAAAC,EAAQ,SAAAC,MAAa,OAG7C,IAAMC,EAAMC,EACVC,EAAE,aACAA,EAAE,aAAaC,EAAkBC,CAAiB,EAClDF,EAAE,OAAO,CACP,gBAAiBA,EAAE,OAAO,EAAE,QAAQ,YAAY,CAClD,CAAC,CACH,CACF,EAEMG,EAA0B,CAE9BL,EAAI,WAAaM,EAAUN,EAAI,UAAU,EAAI,OAE7CA,EAAI,aAAeO,EAAKP,EAAI,YAAY,EAAI,MAC9C,EAAE,OAAOQ,CAAS,EAEZC,EAAeC,EAAmB,CACtC,UAAWC,EAASN,CAAU,EAC9B,gBAAiBL,EAAI,gBACvB,CAAC,EAEKY,GAAU,MAAMH,EAAa,WAAW,EACxCI,EAAWC,EAAQ,IAAIC,EAASf,EAAI,eAAe,CAAC,EAEtDgB,EAAahB,EAAI,YAGrB,GAAI,CAGF,IAAMiB,EAFqBJ,EAAS,OAAO,EAAE,KAAKK,CAAU,EAAE,MAAMC,EAAGD,EAAW,QAASN,EAAO,CAAC,EAAE,IAAI,EAEX,CAAC,EAE3FK,GAAqB,OACnBA,EAAkB,eAAiBG,GACrC,QAAQ,IACN,8BACAH,EAAkB,cAClB,KACAG,EACA,qBACF,EACAC,EAAG,aAAarB,EAAI,eAAe,GAC1BiB,EAAkB,wBAA0B,OACrD,QAAQ,IAAI,6BAA8BA,EAAkB,uBAAyB,EAAE,EACvFD,EAAaC,EAAkB,uBAAyB,IAG9D,MAAE,CAEF,CAEA,GAAM,CAAE,mBAAAK,GAAoB,iBAAAC,EAAiB,EAAI,MAAMC,EAAa,CAClE,SAAAX,EACA,aAAAJ,EACA,WAAAO,EACA,cAAehB,EAAI,gBACnB,QAASA,EAAI,aACf,CAAC,EAEGyB,EAAa,GACjBC,EAAc,CAACJ,GAAoBC,EAAgB,CAAC,EACjD,KACCI,EACE,CAAC,CAACC,EAAmB,CAAE,YAAaC,CAAyB,CAAC,IAAMD,IAAsBC,CAC5F,EACAC,EAAM,CACR,EACC,UAAU,IAAM,CACfL,EAAa,GACb,QAAQ,IAAI,eAAe,CAC7B,CAAC,EAGH,IAAMM,EAASC,EAAQ,CACrB,eAAgB,GAClB,CAAC,EAED,MAAMD,EAAO,SAAS,OAAO,eAAe,CAAC,EAG7CA,EAAO,IAAI,WAAY,CAACE,EAAKC,IAAQA,EAAI,KAAK,GAAG,EAAE,KAAK,CAAC,EACzDH,EAAO,IAAI,UAAW,CAACE,EAAKC,IAAST,EAAaS,EAAI,KAAK,GAAG,EAAE,KAAK,OAAO,EAAIA,EAAI,KAAK,GAAG,EAAE,KAAK,aAAa,CAAE,EAGlHH,EAAO,SAASI,EAA8B,CAC5C,OAAQ,QACR,YAAa,CACX,OAAQC,EAAgB,EACxB,cAAe,UAAa,CAC1B,aAAc,MAAMC,EAAmBxB,CAAQ,CACjD,EACF,CACF,CAAC,EAED,MAAMkB,EAAO,OAAO,CAAE,KAAM/B,EAAI,KAAM,KAAMA,EAAI,IAAK,CAAC,EACtD,QAAQ,IAAI,+CAA+CA,EAAI,QAAQA,EAAI,MAAM","names":["fs","z","eq","drizzle","Database","createPublicClient","fallback","webSocket","http","fastify","fastifyTRPCPlugin","createAppRouter","chainState","schemaVersion","syncToSqlite","eq","buildTable","chainState","getTables","getAddress","decodeDynamicField","createQueryAdapter","database","chainId","address","filters","tableIds","filter","tablesWithRecords","getTables","table","sqliteTable","buildTable","records","eq","filteredRecords","record","keyTuple","name","metadata","chainState","lastUpdatedBlockNumber","result","debug","isDefined","combineLatest","filter","first","env","parseEnv","z","indexerEnvSchema","frontendEnvSchema","transports","webSocket","http","isDefined","publicClient","createPublicClient","fallback","chainId","database","drizzle","Database","startBlock","currentChainState","chainState","eq","schemaVersion","fs","latestBlockNumber$","storedBlockLogs$","syncToSqlite","isCaughtUp","combineLatest","filter","latestBlockNumber","lastBlockNumberProcessed","first","server","fastify","req","res","fastifyTRPCPlugin","createAppRouter","createQueryAdapter"]}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import{isHex as t}from"viem";import{z as e,ZodError as i}from"zod";var T=e.object({HOST:e.string().default("0.0.0.0"),PORT:e.coerce.number().positive().default(3001)}),f=e.intersection(e.object({START_BLOCK:e.coerce.bigint().nonnegative().default(0n),MAX_BLOCK_RANGE:e.coerce.bigint().positive().default(1000n),POLLING_INTERVAL:e.coerce.number().positive().default(1e3),STORE_ADDRESS:e.string().refine(t).optional()}),e.union([e.object({RPC_HTTP_URL:e.string(),RPC_WS_URL:e.string().optional()}),e.object({RPC_HTTP_URL:e.string().optional(),RPC_WS_URL:e.string()})]));function R(o){try{return o.parse(process.env)}catch(n){if(n instanceof i){let{_errors:c,...r}=n.format();console.error(`
|
|
2
|
+
Missing or invalid environment variables:
|
|
3
|
+
|
|
4
|
+
${Object.keys(r).join(`
|
|
5
|
+
`)}
|
|
6
|
+
`),process.exit(1)}throw n}}export{T as a,f as b,R as c};
|
|
7
|
+
//# sourceMappingURL=chunk-2E5MDUA2.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../bin/parseEnv.ts"],"sourcesContent":["import { isHex } from \"viem\";\nimport { z, ZodError, ZodTypeAny } from \"zod\";\n\nexport const frontendEnvSchema = z.object({\n HOST: z.string().default(\"0.0.0.0\"),\n PORT: z.coerce.number().positive().default(3001),\n});\n\nexport const indexerEnvSchema = z.intersection(\n z.object({\n START_BLOCK: z.coerce.bigint().nonnegative().default(0n),\n MAX_BLOCK_RANGE: z.coerce.bigint().positive().default(1000n),\n POLLING_INTERVAL: z.coerce.number().positive().default(1000),\n STORE_ADDRESS: z.string().refine(isHex).optional(),\n }),\n z.union([\n z.object({\n RPC_HTTP_URL: z.string(),\n RPC_WS_URL: z.string().optional(),\n }),\n z.object({\n RPC_HTTP_URL: z.string().optional(),\n RPC_WS_URL: z.string(),\n }),\n ])\n);\n\nexport function parseEnv<TSchema extends ZodTypeAny>(envSchema: TSchema): z.infer<TSchema> {\n try {\n return envSchema.parse(process.env);\n } catch (error) {\n if (error instanceof ZodError) {\n const { _errors, ...invalidEnvVars } = error.format();\n console.error(`\\nMissing or invalid environment variables:\\n\\n ${Object.keys(invalidEnvVars).join(\"\\n \")}\\n`);\n process.exit(1);\n }\n throw error;\n }\n}\n"],"mappings":"AAAA,OAAS,SAAAA,MAAa,OACtB,OAAS,KAAAC,EAAG,YAAAC,MAA4B,MAEjC,IAAMC,EAAoBF,EAAE,OAAO,CACxC,KAAMA,EAAE,OAAO,EAAE,QAAQ,SAAS,EAClC,KAAMA,EAAE,OAAO,OAAO,EAAE,SAAS,EAAE,QAAQ,IAAI,CACjD,CAAC,EAEYG,EAAmBH,EAAE,aAChCA,EAAE,OAAO,CACP,YAAaA,EAAE,OAAO,OAAO,EAAE,YAAY,EAAE,QAAQ,EAAE,EACvD,gBAAiBA,EAAE,OAAO,OAAO,EAAE,SAAS,EAAE,QAAQ,KAAK,EAC3D,iBAAkBA,EAAE,OAAO,OAAO,EAAE,SAAS,EAAE,QAAQ,GAAI,EAC3D,cAAeA,EAAE,OAAO,EAAE,OAAOD,CAAK,EAAE,SAAS,CACnD,CAAC,EACDC,EAAE,MAAM,CACNA,EAAE,OAAO,CACP,aAAcA,EAAE,OAAO,EACvB,WAAYA,EAAE,OAAO,EAAE,SAAS,CAClC,CAAC,EACDA,EAAE,OAAO,CACP,aAAcA,EAAE,OAAO,EAAE,SAAS,EAClC,WAAYA,EAAE,OAAO,CACvB,CAAC,CACH,CAAC,CACH,EAEO,SAASI,EAAqCC,EAAsC,CACzF,GAAI,CACF,OAAOA,EAAU,MAAM,QAAQ,GAAG,CACpC,OAASC,EAAP,CACA,GAAIA,aAAiBL,EAAU,CAC7B,GAAM,CAAE,QAAAM,EAAS,GAAGC,CAAe,EAAIF,EAAM,OAAO,EACpD,QAAQ,MAAM;AAAA;AAAA;AAAA,IAAoD,OAAO,KAAKE,CAAc,EAAE,KAAK;AAAA,GAAM;AAAA,CAAK,EAC9G,QAAQ,KAAK,CAAC,EAEhB,MAAMF,CACR,CACF","names":["isHex","z","ZodError","frontendEnvSchema","indexerEnvSchema","parseEnv","envSchema","error","_errors","invalidEnvVars"]}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/debug.ts"],"sourcesContent":["import createDebug from \"debug\";\n\nexport const debug = createDebug(\"mud:store-indexer\");\n"],"mappings":"AAAA,OAAOA,MAAiB,QAEjB,IAAMC,EAAQD,EAAY,mBAAmB","names":["createDebug","debug"]}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@latticexyz/store-indexer",
|
|
3
|
-
"version": "2.0.0-next.
|
|
3
|
+
"version": "2.0.0-next.14",
|
|
4
4
|
"description": "Minimal Typescript indexer for Store",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -14,6 +14,7 @@
|
|
|
14
14
|
},
|
|
15
15
|
"types": "src/index.ts",
|
|
16
16
|
"bin": {
|
|
17
|
+
"postgres-frontend": "./dist/bin/postgres-frontend.js",
|
|
17
18
|
"postgres-indexer": "./dist/bin/postgres-indexer.js",
|
|
18
19
|
"sqlite-indexer": "./dist/bin/sqlite-indexer.js"
|
|
19
20
|
},
|
|
@@ -32,16 +33,17 @@
|
|
|
32
33
|
"superjson": "^1.12.4",
|
|
33
34
|
"viem": "1.14.0",
|
|
34
35
|
"zod": "^3.21.4",
|
|
35
|
-
"@latticexyz/block-logs-stream": "2.0.0-next.
|
|
36
|
-
"@latticexyz/common": "2.0.0-next.
|
|
37
|
-
"@latticexyz/protocol-parser": "2.0.0-next.
|
|
38
|
-
"@latticexyz/store": "2.0.0-next.
|
|
39
|
-
"@latticexyz/store-sync": "2.0.0-next.
|
|
36
|
+
"@latticexyz/block-logs-stream": "2.0.0-next.14",
|
|
37
|
+
"@latticexyz/common": "2.0.0-next.14",
|
|
38
|
+
"@latticexyz/protocol-parser": "2.0.0-next.14",
|
|
39
|
+
"@latticexyz/store": "2.0.0-next.14",
|
|
40
|
+
"@latticexyz/store-sync": "2.0.0-next.14"
|
|
40
41
|
},
|
|
41
42
|
"devDependencies": {
|
|
42
43
|
"@types/better-sqlite3": "^7.6.4",
|
|
43
44
|
"@types/cors": "^2.8.13",
|
|
44
45
|
"@types/debug": "^4.1.7",
|
|
46
|
+
"concurrently": "^8.2.2",
|
|
45
47
|
"tsup": "^6.7.0",
|
|
46
48
|
"tsx": "^3.12.6",
|
|
47
49
|
"vitest": "0.31.4"
|
|
@@ -57,7 +59,7 @@
|
|
|
57
59
|
"clean:js": "rimraf dist",
|
|
58
60
|
"dev": "tsup --watch",
|
|
59
61
|
"lint": "eslint .",
|
|
60
|
-
"start:postgres": "tsx bin/postgres-indexer",
|
|
62
|
+
"start:postgres": "concurrently -n indexer,frontend -c cyan,magenta 'tsx bin/postgres-indexer' 'tsx bin/postgres-frontend'",
|
|
61
63
|
"start:postgres:local": "DEBUG=mud:store-sync:createStoreSync DATABASE_URL=postgres://127.0.0.1/postgres RPC_HTTP_URL=http://127.0.0.1:8545 pnpm start:postgres",
|
|
62
64
|
"start:postgres:testnet": "DEBUG=mud:store-sync:createStoreSync DATABASE_URL=postgres://127.0.0.1/postgres RPC_HTTP_URL=https://follower.testnet-chain.linfra.xyz pnpm start:postgres",
|
|
63
65
|
"start:sqlite": "tsx bin/sqlite-indexer",
|
package/dist/chunk-X3OEYQLT.js
DELETED
|
@@ -1,7 +0,0 @@
|
|
|
1
|
-
import{z as e,ZodError as c}from"zod";var t=e.intersection(e.object({HOST:e.string().default("0.0.0.0"),PORT:e.coerce.number().positive().default(3001),START_BLOCK:e.coerce.bigint().nonnegative().default(0n),MAX_BLOCK_RANGE:e.coerce.bigint().positive().default(1000n),POLLING_INTERVAL:e.coerce.number().positive().default(1e3)}),e.union([e.object({RPC_HTTP_URL:e.string(),RPC_WS_URL:e.string().optional()}),e.object({RPC_HTTP_URL:e.string().optional(),RPC_WS_URL:e.string()})]));function f(o){let r=o!==void 0?e.intersection(t,o):t;try{return r.parse(process.env)}catch(n){if(n instanceof c){let{_errors:a,...i}=n.format();console.error(`
|
|
2
|
-
Missing or invalid environment variables:
|
|
3
|
-
|
|
4
|
-
${Object.keys(i).join(`
|
|
5
|
-
`)}
|
|
6
|
-
`),process.exit(1)}throw n}}import s from"debug";var _=s("mud:store-indexer");export{_ as a,f as b};
|
|
7
|
-
//# sourceMappingURL=chunk-X3OEYQLT.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"sources":["../bin/parseEnv.ts","../src/debug.ts"],"sourcesContent":["import { z, ZodError, ZodIntersection, ZodTypeAny } from \"zod\";\n\nconst commonSchema = z.intersection(\n z.object({\n HOST: z.string().default(\"0.0.0.0\"),\n PORT: z.coerce.number().positive().default(3001),\n START_BLOCK: z.coerce.bigint().nonnegative().default(0n),\n MAX_BLOCK_RANGE: z.coerce.bigint().positive().default(1000n),\n POLLING_INTERVAL: z.coerce.number().positive().default(1000),\n }),\n z.union([\n z.object({\n RPC_HTTP_URL: z.string(),\n RPC_WS_URL: z.string().optional(),\n }),\n z.object({\n RPC_HTTP_URL: z.string().optional(),\n RPC_WS_URL: z.string(),\n }),\n ])\n);\n\nexport function parseEnv<TSchema extends ZodTypeAny | undefined = undefined>(\n schema?: TSchema\n): z.infer<TSchema extends ZodTypeAny ? ZodIntersection<typeof commonSchema, TSchema> : typeof commonSchema> {\n const envSchema = schema !== undefined ? z.intersection(commonSchema, schema) : commonSchema;\n try {\n return envSchema.parse(process.env);\n } catch (error) {\n if (error instanceof ZodError) {\n const { _errors, ...invalidEnvVars } = error.format();\n console.error(`\\nMissing or invalid environment variables:\\n\\n ${Object.keys(invalidEnvVars).join(\"\\n \")}\\n`);\n process.exit(1);\n }\n throw error;\n }\n}\n","import createDebug from \"debug\";\n\nexport const debug = createDebug(\"mud:store-indexer\");\n"],"mappings":"AAAA,OAAS,KAAAA,EAAG,YAAAC,MAA6C,MAEzD,IAAMC,EAAeF,EAAE,aACrBA,EAAE,OAAO,CACP,KAAMA,EAAE,OAAO,EAAE,QAAQ,SAAS,EAClC,KAAMA,EAAE,OAAO,OAAO,EAAE,SAAS,EAAE,QAAQ,IAAI,EAC/C,YAAaA,EAAE,OAAO,OAAO,EAAE,YAAY,EAAE,QAAQ,EAAE,EACvD,gBAAiBA,EAAE,OAAO,OAAO,EAAE,SAAS,EAAE,QAAQ,KAAK,EAC3D,iBAAkBA,EAAE,OAAO,OAAO,EAAE,SAAS,EAAE,QAAQ,GAAI,CAC7D,CAAC,EACDA,EAAE,MAAM,CACNA,EAAE,OAAO,CACP,aAAcA,EAAE,OAAO,EACvB,WAAYA,EAAE,OAAO,EAAE,SAAS,CAClC,CAAC,EACDA,EAAE,OAAO,CACP,aAAcA,EAAE,OAAO,EAAE,SAAS,EAClC,WAAYA,EAAE,OAAO,CACvB,CAAC,CACH,CAAC,CACH,EAEO,SAASG,EACdC,EAC2G,CAC3G,IAAMC,EAAYD,IAAW,OAAYJ,EAAE,aAAaE,EAAcE,CAAM,EAAIF,EAChF,GAAI,CACF,OAAOG,EAAU,MAAM,QAAQ,GAAG,CACpC,OAASC,EAAP,CACA,GAAIA,aAAiBL,EAAU,CAC7B,GAAM,CAAE,QAAAM,EAAS,GAAGC,CAAe,EAAIF,EAAM,OAAO,EACpD,QAAQ,MAAM;AAAA;AAAA;AAAA,IAAoD,OAAO,KAAKE,CAAc,EAAE,KAAK;AAAA,GAAM;AAAA,CAAK,EAC9G,QAAQ,KAAK,CAAC,EAEhB,MAAMF,CACR,CACF,CCpCA,OAAOG,MAAiB,QAEjB,IAAMC,EAAQD,EAAY,mBAAmB","names":["z","ZodError","commonSchema","parseEnv","schema","envSchema","error","_errors","invalidEnvVars","createDebug","debug"]}
|