@latticexyz/store-indexer 2.0.0-main-082a8590 → 2.0.0-main-c314badd
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -1,3 +1,3 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import{a as
|
|
2
|
+
import{a as A,c as k}from"../chunk-2E5MDUA2.js";import"dotenv/config";import{z as y}from"zod";import U from"koa";import E from"@koa/cors";import O from"@koa/router";import{createKoaMiddleware as W}from"trpc-koa-adapter";import{createAppRouter as _}from"@latticexyz/store-sync/trpc-indexer";import{getAddress as x}from"viem";import{isTableRegistrationLog as N,logToTable as P,storeTables as $}from"@latticexyz/store-sync";import{decodeKey as z,decodeValueArgs as H}from"@latticexyz/protocol-parser";import L from"debug";var S=L("mud:store-indexer");import{tables as a}from"@latticexyz/store-sync/postgres";import{and as R,asc as I,eq as s,or as w}from"drizzle-orm";import{decodeDynamicField as B}from"@latticexyz/protocol-parser";import{bigIntMax as v}from"@latticexyz/common/utils";async function f(t,{chainId:T,address:o,filters:n=[]}){let u=n.length?n.map(e=>R(o!=null?s(a.recordsTable.address,o):void 0,s(a.recordsTable.tableId,e.tableId),e.key0!=null?s(a.recordsTable.key0,e.key0):void 0,e.key1!=null?s(a.recordsTable.key1,e.key1):void 0)):o!=null?[s(a.recordsTable.address,o)]:[],m=(await t.select().from(a.configTable).where(s(a.configTable.chainId,T)).limit(1).execute().then(e=>e.find(()=>!0)))?.lastUpdatedBlockNumber??0n,p=await t.select().from(a.recordsTable).where(w(...u)).orderBy(I(a.recordsTable.lastUpdatedBlockNumber)),g=p.reduce((e,b)=>v(e,b.lastUpdatedBlockNumber??0n),m),r=p.filter(e=>!e.isDeleted).map(e=>({address:e.address,eventName:"Store_SetRecord",args:{tableId:e.tableId,keyTuple:B("bytes32[]",e.keyBytes),staticData:e.staticData??"0x",encodedLengths:e.encodedLengths??"0x",dynamicData:e.dynamicData??"0x"}}));return{blockNumber:g,logs:r}}import{groupBy as Q}from"@latticexyz/common/utils";async function D(t){return{async getLogs(o){return f(t,o)},async findAll(o){let n=o.filters??[],{blockNumber:u,logs:c}=await f(t,{...o,filters:n.length>0?[...n,{tableId:$.Tables.tableId}]:[]}),m=c.filter(N).map(P),p=Q(c,r=>`${x(r.address)}:${r.args.tableId}`),g=m.map(r=>{let b=(p.get(`${x(r.address)}:${r.tableId}`)??[]).map(h=>({key:z(r.keySchema,h.args.keyTuple),value:H(r.valueSchema,h.args)}));return{...r,records:b}});return S("findAll: decoded %d logs across %d tables",c.length,m.length),{blockNumber:u,tables:g}}}}import{drizzle as F}from"drizzle-orm/postgres-js";import K from"postgres";var d=k(y.intersection(A,y.object({DATABASE_URL:y.string()}))),M=F(K(d.DATABASE_URL)),l=new U;l.use(E());var i=new O;i.get("/",t=>{t.body="emit HelloWorld();"});i.get("/healthz",t=>{t.status=200});i.get("/readyz",t=>{t.status=200});l.use(i.routes());l.use(i.allowedMethods());l.use(W({prefix:"/trpc",router:_(),createContext:async()=>({queryAdapter:await D(M)})}));l.listen({host:d.HOST,port:d.PORT});console.log(`postgres indexer frontend listening on http://${d.HOST}:${d.PORT}`);
|
|
3
3
|
//# sourceMappingURL=postgres-frontend.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../bin/postgres-frontend.ts","../../src/postgres/createQueryAdapter.ts","../../src/debug.ts","../../src/postgres/getLogs.ts"],"sourcesContent":["#!/usr/bin/env node\nimport \"dotenv/config\";\nimport { z } from \"zod\";\nimport fastify from \"fastify\";\nimport { fastifyTRPCPlugin } from \"@trpc/server/adapters/fastify\";\nimport { AppRouter, createAppRouter } from \"@latticexyz/store-sync/trpc-indexer\";\nimport { createQueryAdapter } from \"../src/postgres/createQueryAdapter\";\nimport { drizzle } from \"drizzle-orm/postgres-js\";\nimport postgres from \"postgres\";\nimport { frontendEnvSchema, parseEnv } from \"./parseEnv\";\n\nconst env = parseEnv(\n z.intersection(\n frontendEnvSchema,\n z.object({\n DATABASE_URL: z.string(),\n })\n )\n);\n\nconst database = drizzle(postgres(env.DATABASE_URL));\n\n// @see https://fastify.dev/docs/latest/\nconst server = fastify({\n maxParamLength: 5000,\n logger: true,\n});\n\nawait server.register(import(\"@fastify/compress\"));\nawait server.register(import(\"@fastify/cors\"));\n\n// k8s healthchecks\nserver.get(\"/healthz\", (req, res) => res.code(200).send());\nserver.get(\"/readyz\", (req, res) => res.code(200).send());\n\n// @see https://trpc.io/docs/server/adapters/fastify\nserver.register(fastifyTRPCPlugin<AppRouter>, {\n prefix: \"/trpc\",\n trpcOptions: {\n router: createAppRouter(),\n createContext: async () => ({\n queryAdapter: await createQueryAdapter(database),\n }),\n },\n});\n\nawait server.listen({ host: env.HOST, port: env.PORT });\nconsole.log(`postgres indexer frontend listening on http://${env.HOST}:${env.PORT}`);\n","import { getAddress } from \"viem\";\nimport { PgDatabase } from \"drizzle-orm/pg-core\";\nimport { TableWithRecords, isTableRegistrationLog, logToTable, storeTables } from \"@latticexyz/store-sync\";\nimport { decodeKey, decodeValueArgs } from \"@latticexyz/protocol-parser\";\nimport { QueryAdapter } from \"@latticexyz/store-sync/trpc-indexer\";\nimport { debug } from \"../debug\";\nimport { getLogs } from \"./getLogs\";\nimport { groupBy } from \"@latticexyz/common/utils\";\n\n/**\n * Creates a query adapter for the tRPC server/client to query data from Postgres.\n *\n * @param {PgDatabase<any>} database Postgres database object from Drizzle\n * @returns {Promise<QueryAdapter>} A set of methods used by tRPC endpoints.\n */\nexport async function createQueryAdapter(database: PgDatabase<any>): Promise<QueryAdapter> {\n const adapter: QueryAdapter = {\n async getLogs(opts) {\n return getLogs(database, opts);\n },\n async findAll(opts) {\n const filters = opts.filters ?? [];\n const { blockNumber, logs } = await getLogs(database, {\n ...opts,\n // make sure we're always retrieving `store.Tables` table, so we can decode table values\n filters: filters.length > 0 ? [...filters, { tableId: storeTables.Tables.tableId }] : [],\n });\n\n const tables = logs.filter(isTableRegistrationLog).map(logToTable);\n\n const logsByTable = groupBy(logs, (log) => `${getAddress(log.address)}:${log.args.tableId}`);\n\n const tablesWithRecords: TableWithRecords[] = tables.map((table) => {\n const tableLogs = logsByTable.get(`${getAddress(table.address)}:${table.tableId}`) ?? [];\n const records = tableLogs.map((log) => ({\n key: decodeKey(table.keySchema, log.args.keyTuple),\n value: decodeValueArgs(table.valueSchema, log.args),\n }));\n\n return {\n ...table,\n records,\n };\n });\n\n debug(\"findAll: decoded %d logs across %d tables\", logs.length, tables.length);\n\n return {\n blockNumber,\n tables: tablesWithRecords,\n };\n },\n };\n return adapter;\n}\n","import createDebug from \"debug\";\n\nexport const debug = createDebug(\"mud:store-indexer\");\n","import { PgDatabase } from \"drizzle-orm/pg-core\";\nimport { Hex } from \"viem\";\nimport { StorageAdapterLog, SyncFilter } from \"@latticexyz/store-sync\";\nimport { tables } from \"@latticexyz/store-sync/postgres\";\nimport { and, asc, eq, or } from \"drizzle-orm\";\nimport { decodeDynamicField } from \"@latticexyz/protocol-parser\";\nimport { bigIntMax } from \"@latticexyz/common/utils\";\n\nexport async function getLogs(\n database: PgDatabase<any>,\n {\n chainId,\n address,\n filters = [],\n }: {\n readonly chainId: number;\n readonly address?: Hex;\n readonly filters?: readonly SyncFilter[];\n }\n): Promise<{ blockNumber: bigint; logs: (StorageAdapterLog & { eventName: \"Store_SetRecord\" })[] }> {\n const conditions = filters.length\n ? filters.map((filter) =>\n and(\n address != null ? eq(tables.recordsTable.address, address) : undefined,\n eq(tables.recordsTable.tableId, filter.tableId),\n filter.key0 != null ? eq(tables.recordsTable.key0, filter.key0) : undefined,\n filter.key1 != null ? eq(tables.recordsTable.key1, filter.key1) : undefined\n )\n )\n : address != null\n ? [eq(tables.recordsTable.address, address)]\n : [];\n\n // Query for the block number that the indexer (i.e. chain) is at, in case the\n // indexer is further along in the chain than a given store/table's last updated\n // block number. We'll then take the highest block number between the indexer's\n // chain state and all the records in the query (in case the records updated\n // between these queries). Using just the highest block number from the queries\n // could potentially signal to the client an older-than-necessary block number,\n // for stores/tables that haven't seen recent activity.\n // TODO: move the block number query into the records query for atomicity so we don't have to merge them here\n const chainState = await database\n .select()\n .from(tables.configTable)\n .where(eq(tables.configTable.chainId, chainId))\n .limit(1)\n .execute()\n // Get the first record in a way that returns a possible `undefined`\n // TODO: move this to `.findFirst` after upgrading drizzle or `rows[0]` after enabling `noUncheckedIndexedAccess: true`\n .then((rows) => rows.find(() => true));\n const indexerBlockNumber = chainState?.lastUpdatedBlockNumber ?? 0n;\n\n const records = await database\n .select()\n .from(tables.recordsTable)\n .where(or(...conditions))\n .orderBy(\n asc(tables.recordsTable.lastUpdatedBlockNumber)\n // TODO: add logIndex (https://github.com/latticexyz/mud/issues/1979)\n );\n\n const blockNumber = records.reduce(\n (max, record) => bigIntMax(max, record.lastUpdatedBlockNumber ?? 0n),\n indexerBlockNumber\n );\n\n const logs = records\n // TODO: add this to the query, assuming we can optimize with an index\n .filter((record) => !record.isDeleted)\n .map(\n (record) =>\n ({\n address: record.address,\n eventName: \"Store_SetRecord\",\n args: {\n tableId: record.tableId,\n keyTuple: decodeDynamicField(\"bytes32[]\", record.keyBytes),\n staticData: record.staticData ?? \"0x\",\n encodedLengths: record.encodedLengths ?? \"0x\",\n dynamicData: record.dynamicData ?? \"0x\",\n },\n } as const)\n );\n\n return { blockNumber, logs };\n}\n"],"mappings":";gDACA,MAAO,gBACP,OAAS,KAAAA,MAAS,MAClB,OAAOC,MAAa,UACpB,OAAS,qBAAAC,MAAyB,gCAClC,OAAoB,mBAAAC,MAAuB,sCCL3C,OAAS,cAAAC,MAAkB,OAE3B,OAA2B,0BAAAC,EAAwB,cAAAC,EAAY,eAAAC,MAAmB,yBAClF,OAAS,aAAAC,EAAW,mBAAAC,MAAuB,8BCH3C,OAAOC,MAAiB,QAEjB,IAAMC,EAAQD,EAAY,mBAAmB,ECCpD,OAAS,UAAAE,MAAc,kCACvB,OAAS,OAAAC,EAAK,OAAAC,EAAK,MAAAC,EAAI,MAAAC,MAAU,cACjC,OAAS,sBAAAC,MAA0B,8BACnC,OAAS,aAAAC,MAAiB,2BAE1B,eAAsBC,EACpBC,EACA,CACE,QAAAC,EACA,QAAAC,EACA,QAAAC,EAAU,CAAC,CACb,EAKkG,CAClG,IAAMC,EAAaD,EAAQ,OACvBA,EAAQ,IAAKE,GACXZ,EACES,GAAW,KAAOP,EAAGH,EAAO,aAAa,QAASU,CAAO,EAAI,OAC7DP,EAAGH,EAAO,aAAa,QAASa,EAAO,OAAO,EAC9CA,EAAO,MAAQ,KAAOV,EAAGH,EAAO,aAAa,KAAMa,EAAO,IAAI,EAAI,OAClEA,EAAO,MAAQ,KAAOV,EAAGH,EAAO,aAAa,KAAMa,EAAO,IAAI,EAAI,MACpE,CACF,EACAH,GAAW,KACX,CAACP,EAAGH,EAAO,aAAa,QAASU,CAAO,CAAC,EACzC,CAAC,EAmBCI,GATa,MAAMN,EACtB,OAAO,EACP,KAAKR,EAAO,WAAW,EACvB,MAAMG,EAAGH,EAAO,YAAY,QAASS,CAAO,CAAC,EAC7C,MAAM,CAAC,EACP,QAAQ,EAGR,KAAMM,GAASA,EAAK,KAAK,IAAM,EAAI,CAAC,IACA,wBAA0B,GAE3DC,EAAU,MAAMR,EACnB,OAAO,EACP,KAAKR,EAAO,YAAY,EACxB,MAAMI,EAAG,GAAGQ,CAAU,CAAC,EACvB,QACCV,EAAIF,EAAO,aAAa,sBAAsB,CAEhD,EAEIiB,EAAcD,EAAQ,OAC1B,CAACE,EAAKC,IAAWb,EAAUY,EAAKC,EAAO,wBAA0B,EAAE,EACnEL,CACF,EAEMM,EAAOJ,EAEV,OAAQG,GAAW,CAACA,EAAO,SAAS,EACpC,IACEA,IACE,CACC,QAASA,EAAO,QAChB,UAAW,kBACX,KAAM,CACJ,QAASA,EAAO,QAChB,SAAUd,EAAmB,YAAac,EAAO,QAAQ,EACzD,WAAYA,EAAO,YAAc,KACjC,eAAgBA,EAAO,gBAAkB,KACzC,YAAaA,EAAO,aAAe,IACrC,CACF,EACJ,EAEF,MAAO,CAAE,YAAAF,EAAa,KAAAG,CAAK,CAC7B,CF9EA,OAAS,WAAAC,MAAe,2BAQxB,eAAsBC,EAAmBC,EAAkD,CAsCzF,MArC8B,CAC5B,MAAM,QAAQC,EAAM,CAClB,OAAOC,EAAQF,EAAUC,CAAI,CAC/B,EACA,MAAM,QAAQA,EAAM,CAClB,IAAME,EAAUF,EAAK,SAAW,CAAC,EAC3B,CAAE,YAAAG,EAAa,KAAAC,CAAK,EAAI,MAAMH,EAAQF,EAAU,CACpD,GAAGC,EAEH,QAASE,EAAQ,OAAS,EAAI,CAAC,GAAGA,EAAS,CAAE,QAASG,EAAY,OAAO,OAAQ,CAAC,EAAI,CAAC,CACzF,CAAC,EAEKC,EAASF,EAAK,OAAOG,CAAsB,EAAE,IAAIC,CAAU,EAE3DC,EAAcZ,EAAQO,EAAOM,GAAQ,GAAGC,EAAWD,EAAI,OAAO,KAAKA,EAAI,KAAK,SAAS,EAErFE,EAAwCN,EAAO,IAAKO,GAAU,CAElE,IAAMC,GADYL,EAAY,IAAI,GAAGE,EAAWE,EAAM,OAAO,KAAKA,EAAM,SAAS,GAAK,CAAC,GAC7D,IAAKH,IAAS,CACtC,IAAKK,EAAUF,EAAM,UAAWH,EAAI,KAAK,QAAQ,EACjD,MAAOM,EAAgBH,EAAM,YAAaH,EAAI,IAAI,CACpD,EAAE,EAEF,MAAO,CACL,GAAGG,EACH,QAAAC,CACF,CACF,CAAC,EAED,OAAAG,EAAM,4CAA6Cb,EAAK,OAAQE,EAAO,MAAM,EAEtE,CACL,YAAAH,EACA,OAAQS,CACV,CACF,CACF,CAEF,CD/CA,OAAS,WAAAM,MAAe,0BACxB,OAAOC,MAAc,WAGrB,IAAMC,EAAMC,EACVC,EAAE,aACAC,EACAD,EAAE,OAAO,CACP,aAAcA,EAAE,OAAO,CACzB,CAAC,CACH,CACF,EAEME,EAAWC,EAAQC,EAASN,EAAI,YAAY,CAAC,EAG7CO,EAASC,EAAQ,CACrB,eAAgB,IAChB,OAAQ,EACV,CAAC,EAED,MAAMD,EAAO,SAAS,OAAO,mBAAmB,CAAC,EACjD,MAAMA,EAAO,SAAS,OAAO,eAAe,CAAC,EAG7CA,EAAO,IAAI,WAAY,CAACE,EAAKC,IAAQA,EAAI,KAAK,GAAG,EAAE,KAAK,CAAC,EACzDH,EAAO,IAAI,UAAW,CAACE,EAAKC,IAAQA,EAAI,KAAK,GAAG,EAAE,KAAK,CAAC,EAGxDH,EAAO,SAASI,EAA8B,CAC5C,OAAQ,QACR,YAAa,CACX,OAAQC,EAAgB,EACxB,cAAe,UAAa,CAC1B,aAAc,MAAMC,EAAmBT,CAAQ,CACjD,EACF,CACF,CAAC,EAED,MAAMG,EAAO,OAAO,CAAE,KAAMP,EAAI,KAAM,KAAMA,EAAI,IAAK,CAAC,EACtD,QAAQ,IAAI,iDAAiDA,EAAI,QAAQA,EAAI,MAAM","names":["z","fastify","fastifyTRPCPlugin","createAppRouter","getAddress","isTableRegistrationLog","logToTable","storeTables","decodeKey","decodeValueArgs","createDebug","debug","tables","and","asc","eq","or","decodeDynamicField","bigIntMax","getLogs","database","chainId","address","filters","conditions","filter","indexerBlockNumber","rows","records","blockNumber","max","record","logs","groupBy","createQueryAdapter","database","opts","getLogs","filters","blockNumber","logs","storeTables","tables","isTableRegistrationLog","logToTable","logsByTable","log","getAddress","tablesWithRecords","table","records","decodeKey","decodeValueArgs","debug","drizzle","postgres","env","parseEnv","z","frontendEnvSchema","database","drizzle","postgres","server","fastify","req","res","fastifyTRPCPlugin","createAppRouter","createQueryAdapter"]}
|
|
1
|
+
{"version":3,"sources":["../../bin/postgres-frontend.ts","../../src/postgres/createQueryAdapter.ts","../../src/debug.ts","../../src/postgres/getLogs.ts"],"sourcesContent":["#!/usr/bin/env node\nimport \"dotenv/config\";\nimport { z } from \"zod\";\nimport Koa from \"koa\";\nimport cors from \"@koa/cors\";\nimport Router from \"@koa/router\";\nimport { createKoaMiddleware } from \"trpc-koa-adapter\";\nimport { createAppRouter } from \"@latticexyz/store-sync/trpc-indexer\";\nimport { createQueryAdapter } from \"../src/postgres/createQueryAdapter\";\nimport { drizzle } from \"drizzle-orm/postgres-js\";\nimport postgres from \"postgres\";\nimport { frontendEnvSchema, parseEnv } from \"./parseEnv\";\n\nconst env = parseEnv(\n z.intersection(\n frontendEnvSchema,\n z.object({\n DATABASE_URL: z.string(),\n })\n )\n);\n\nconst database = drizzle(postgres(env.DATABASE_URL));\n\nconst server = new Koa();\nserver.use(cors());\n\nconst router = new Router();\n\nrouter.get(\"/\", (ctx) => {\n ctx.body = \"emit HelloWorld();\";\n});\n\n// k8s healthchecks\nrouter.get(\"/healthz\", (ctx) => {\n ctx.status = 200;\n});\nrouter.get(\"/readyz\", (ctx) => {\n ctx.status = 200;\n});\n\nserver.use(router.routes());\nserver.use(router.allowedMethods());\n\nserver.use(\n createKoaMiddleware({\n prefix: \"/trpc\",\n router: createAppRouter(),\n createContext: async () => ({\n queryAdapter: await createQueryAdapter(database),\n }),\n })\n);\n\nserver.listen({ host: env.HOST, port: env.PORT });\nconsole.log(`postgres indexer frontend listening on http://${env.HOST}:${env.PORT}`);\n","import { getAddress } from \"viem\";\nimport { PgDatabase } from \"drizzle-orm/pg-core\";\nimport { TableWithRecords, isTableRegistrationLog, logToTable, storeTables } from \"@latticexyz/store-sync\";\nimport { decodeKey, decodeValueArgs } from \"@latticexyz/protocol-parser\";\nimport { QueryAdapter } from \"@latticexyz/store-sync/trpc-indexer\";\nimport { debug } from \"../debug\";\nimport { getLogs } from \"./getLogs\";\nimport { groupBy } from \"@latticexyz/common/utils\";\n\n/**\n * Creates a query adapter for the tRPC server/client to query data from Postgres.\n *\n * @param {PgDatabase<any>} database Postgres database object from Drizzle\n * @returns {Promise<QueryAdapter>} A set of methods used by tRPC endpoints.\n */\nexport async function createQueryAdapter(database: PgDatabase<any>): Promise<QueryAdapter> {\n const adapter: QueryAdapter = {\n async getLogs(opts) {\n return getLogs(database, opts);\n },\n async findAll(opts) {\n const filters = opts.filters ?? [];\n const { blockNumber, logs } = await getLogs(database, {\n ...opts,\n // make sure we're always retrieving `store.Tables` table, so we can decode table values\n filters: filters.length > 0 ? [...filters, { tableId: storeTables.Tables.tableId }] : [],\n });\n\n const tables = logs.filter(isTableRegistrationLog).map(logToTable);\n\n const logsByTable = groupBy(logs, (log) => `${getAddress(log.address)}:${log.args.tableId}`);\n\n const tablesWithRecords: TableWithRecords[] = tables.map((table) => {\n const tableLogs = logsByTable.get(`${getAddress(table.address)}:${table.tableId}`) ?? [];\n const records = tableLogs.map((log) => ({\n key: decodeKey(table.keySchema, log.args.keyTuple),\n value: decodeValueArgs(table.valueSchema, log.args),\n }));\n\n return {\n ...table,\n records,\n };\n });\n\n debug(\"findAll: decoded %d logs across %d tables\", logs.length, tables.length);\n\n return {\n blockNumber,\n tables: tablesWithRecords,\n };\n },\n };\n return adapter;\n}\n","import createDebug from \"debug\";\n\nexport const debug = createDebug(\"mud:store-indexer\");\n","import { PgDatabase } from \"drizzle-orm/pg-core\";\nimport { Hex } from \"viem\";\nimport { StorageAdapterLog, SyncFilter } from \"@latticexyz/store-sync\";\nimport { tables } from \"@latticexyz/store-sync/postgres\";\nimport { and, asc, eq, or } from \"drizzle-orm\";\nimport { decodeDynamicField } from \"@latticexyz/protocol-parser\";\nimport { bigIntMax } from \"@latticexyz/common/utils\";\n\nexport async function getLogs(\n database: PgDatabase<any>,\n {\n chainId,\n address,\n filters = [],\n }: {\n readonly chainId: number;\n readonly address?: Hex;\n readonly filters?: readonly SyncFilter[];\n }\n): Promise<{ blockNumber: bigint; logs: (StorageAdapterLog & { eventName: \"Store_SetRecord\" })[] }> {\n const conditions = filters.length\n ? filters.map((filter) =>\n and(\n address != null ? eq(tables.recordsTable.address, address) : undefined,\n eq(tables.recordsTable.tableId, filter.tableId),\n filter.key0 != null ? eq(tables.recordsTable.key0, filter.key0) : undefined,\n filter.key1 != null ? eq(tables.recordsTable.key1, filter.key1) : undefined\n )\n )\n : address != null\n ? [eq(tables.recordsTable.address, address)]\n : [];\n\n // Query for the block number that the indexer (i.e. chain) is at, in case the\n // indexer is further along in the chain than a given store/table's last updated\n // block number. We'll then take the highest block number between the indexer's\n // chain state and all the records in the query (in case the records updated\n // between these queries). Using just the highest block number from the queries\n // could potentially signal to the client an older-than-necessary block number,\n // for stores/tables that haven't seen recent activity.\n // TODO: move the block number query into the records query for atomicity so we don't have to merge them here\n const chainState = await database\n .select()\n .from(tables.configTable)\n .where(eq(tables.configTable.chainId, chainId))\n .limit(1)\n .execute()\n // Get the first record in a way that returns a possible `undefined`\n // TODO: move this to `.findFirst` after upgrading drizzle or `rows[0]` after enabling `noUncheckedIndexedAccess: true`\n .then((rows) => rows.find(() => true));\n const indexerBlockNumber = chainState?.lastUpdatedBlockNumber ?? 0n;\n\n const records = await database\n .select()\n .from(tables.recordsTable)\n .where(or(...conditions))\n .orderBy(\n asc(tables.recordsTable.lastUpdatedBlockNumber)\n // TODO: add logIndex (https://github.com/latticexyz/mud/issues/1979)\n );\n\n const blockNumber = records.reduce(\n (max, record) => bigIntMax(max, record.lastUpdatedBlockNumber ?? 0n),\n indexerBlockNumber\n );\n\n const logs = records\n // TODO: add this to the query, assuming we can optimize with an index\n .filter((record) => !record.isDeleted)\n .map(\n (record) =>\n ({\n address: record.address,\n eventName: \"Store_SetRecord\",\n args: {\n tableId: record.tableId,\n keyTuple: decodeDynamicField(\"bytes32[]\", record.keyBytes),\n staticData: record.staticData ?? \"0x\",\n encodedLengths: record.encodedLengths ?? \"0x\",\n dynamicData: record.dynamicData ?? \"0x\",\n },\n } as const)\n );\n\n return { blockNumber, logs };\n}\n"],"mappings":";gDACA,MAAO,gBACP,OAAS,KAAAA,MAAS,MAClB,OAAOC,MAAS,MAChB,OAAOC,MAAU,YACjB,OAAOC,MAAY,cACnB,OAAS,uBAAAC,MAA2B,mBACpC,OAAS,mBAAAC,MAAuB,sCCPhC,OAAS,cAAAC,MAAkB,OAE3B,OAA2B,0BAAAC,EAAwB,cAAAC,EAAY,eAAAC,MAAmB,yBAClF,OAAS,aAAAC,EAAW,mBAAAC,MAAuB,8BCH3C,OAAOC,MAAiB,QAEjB,IAAMC,EAAQD,EAAY,mBAAmB,ECCpD,OAAS,UAAAE,MAAc,kCACvB,OAAS,OAAAC,EAAK,OAAAC,EAAK,MAAAC,EAAI,MAAAC,MAAU,cACjC,OAAS,sBAAAC,MAA0B,8BACnC,OAAS,aAAAC,MAAiB,2BAE1B,eAAsBC,EACpBC,EACA,CACE,QAAAC,EACA,QAAAC,EACA,QAAAC,EAAU,CAAC,CACb,EAKkG,CAClG,IAAMC,EAAaD,EAAQ,OACvBA,EAAQ,IAAKE,GACXZ,EACES,GAAW,KAAOP,EAAGH,EAAO,aAAa,QAASU,CAAO,EAAI,OAC7DP,EAAGH,EAAO,aAAa,QAASa,EAAO,OAAO,EAC9CA,EAAO,MAAQ,KAAOV,EAAGH,EAAO,aAAa,KAAMa,EAAO,IAAI,EAAI,OAClEA,EAAO,MAAQ,KAAOV,EAAGH,EAAO,aAAa,KAAMa,EAAO,IAAI,EAAI,MACpE,CACF,EACAH,GAAW,KACX,CAACP,EAAGH,EAAO,aAAa,QAASU,CAAO,CAAC,EACzC,CAAC,EAmBCI,GATa,MAAMN,EACtB,OAAO,EACP,KAAKR,EAAO,WAAW,EACvB,MAAMG,EAAGH,EAAO,YAAY,QAASS,CAAO,CAAC,EAC7C,MAAM,CAAC,EACP,QAAQ,EAGR,KAAMM,GAASA,EAAK,KAAK,IAAM,EAAI,CAAC,IACA,wBAA0B,GAE3DC,EAAU,MAAMR,EACnB,OAAO,EACP,KAAKR,EAAO,YAAY,EACxB,MAAMI,EAAG,GAAGQ,CAAU,CAAC,EACvB,QACCV,EAAIF,EAAO,aAAa,sBAAsB,CAEhD,EAEIiB,EAAcD,EAAQ,OAC1B,CAACE,EAAKC,IAAWb,EAAUY,EAAKC,EAAO,wBAA0B,EAAE,EACnEL,CACF,EAEMM,EAAOJ,EAEV,OAAQG,GAAW,CAACA,EAAO,SAAS,EACpC,IACEA,IACE,CACC,QAASA,EAAO,QAChB,UAAW,kBACX,KAAM,CACJ,QAASA,EAAO,QAChB,SAAUd,EAAmB,YAAac,EAAO,QAAQ,EACzD,WAAYA,EAAO,YAAc,KACjC,eAAgBA,EAAO,gBAAkB,KACzC,YAAaA,EAAO,aAAe,IACrC,CACF,EACJ,EAEF,MAAO,CAAE,YAAAF,EAAa,KAAAG,CAAK,CAC7B,CF9EA,OAAS,WAAAC,MAAe,2BAQxB,eAAsBC,EAAmBC,EAAkD,CAsCzF,MArC8B,CAC5B,MAAM,QAAQC,EAAM,CAClB,OAAOC,EAAQF,EAAUC,CAAI,CAC/B,EACA,MAAM,QAAQA,EAAM,CAClB,IAAME,EAAUF,EAAK,SAAW,CAAC,EAC3B,CAAE,YAAAG,EAAa,KAAAC,CAAK,EAAI,MAAMH,EAAQF,EAAU,CACpD,GAAGC,EAEH,QAASE,EAAQ,OAAS,EAAI,CAAC,GAAGA,EAAS,CAAE,QAASG,EAAY,OAAO,OAAQ,CAAC,EAAI,CAAC,CACzF,CAAC,EAEKC,EAASF,EAAK,OAAOG,CAAsB,EAAE,IAAIC,CAAU,EAE3DC,EAAcZ,EAAQO,EAAOM,GAAQ,GAAGC,EAAWD,EAAI,OAAO,KAAKA,EAAI,KAAK,SAAS,EAErFE,EAAwCN,EAAO,IAAKO,GAAU,CAElE,IAAMC,GADYL,EAAY,IAAI,GAAGE,EAAWE,EAAM,OAAO,KAAKA,EAAM,SAAS,GAAK,CAAC,GAC7D,IAAKH,IAAS,CACtC,IAAKK,EAAUF,EAAM,UAAWH,EAAI,KAAK,QAAQ,EACjD,MAAOM,EAAgBH,EAAM,YAAaH,EAAI,IAAI,CACpD,EAAE,EAEF,MAAO,CACL,GAAGG,EACH,QAAAC,CACF,CACF,CAAC,EAED,OAAAG,EAAM,4CAA6Cb,EAAK,OAAQE,EAAO,MAAM,EAEtE,CACL,YAAAH,EACA,OAAQS,CACV,CACF,CACF,CAEF,CD7CA,OAAS,WAAAM,MAAe,0BACxB,OAAOC,MAAc,WAGrB,IAAMC,EAAMC,EACVC,EAAE,aACAC,EACAD,EAAE,OAAO,CACP,aAAcA,EAAE,OAAO,CACzB,CAAC,CACH,CACF,EAEME,EAAWC,EAAQC,EAASN,EAAI,YAAY,CAAC,EAE7CO,EAAS,IAAIC,EACnBD,EAAO,IAAIE,EAAK,CAAC,EAEjB,IAAMC,EAAS,IAAIC,EAEnBD,EAAO,IAAI,IAAME,GAAQ,CACvBA,EAAI,KAAO,oBACb,CAAC,EAGDF,EAAO,IAAI,WAAaE,GAAQ,CAC9BA,EAAI,OAAS,GACf,CAAC,EACDF,EAAO,IAAI,UAAYE,GAAQ,CAC7BA,EAAI,OAAS,GACf,CAAC,EAEDL,EAAO,IAAIG,EAAO,OAAO,CAAC,EAC1BH,EAAO,IAAIG,EAAO,eAAe,CAAC,EAElCH,EAAO,IACLM,EAAoB,CAClB,OAAQ,QACR,OAAQC,EAAgB,EACxB,cAAe,UAAa,CAC1B,aAAc,MAAMC,EAAmBX,CAAQ,CACjD,EACF,CAAC,CACH,EAEAG,EAAO,OAAO,CAAE,KAAMP,EAAI,KAAM,KAAMA,EAAI,IAAK,CAAC,EAChD,QAAQ,IAAI,iDAAiDA,EAAI,QAAQA,EAAI,MAAM","names":["z","Koa","cors","Router","createKoaMiddleware","createAppRouter","getAddress","isTableRegistrationLog","logToTable","storeTables","decodeKey","decodeValueArgs","createDebug","debug","tables","and","asc","eq","or","decodeDynamicField","bigIntMax","getLogs","database","chainId","address","filters","conditions","filter","indexerBlockNumber","rows","records","blockNumber","max","record","logs","groupBy","createQueryAdapter","database","opts","getLogs","filters","blockNumber","logs","storeTables","tables","isTableRegistrationLog","logToTable","logsByTable","log","getAddress","tablesWithRecords","table","records","decodeKey","decodeValueArgs","debug","drizzle","postgres","env","parseEnv","z","frontendEnvSchema","database","drizzle","postgres","server","Koa","cors","router","Router","ctx","createKoaMiddleware","createAppRouter","createQueryAdapter"]}
|
|
@@ -1,3 +1,3 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import{b as
|
|
2
|
+
import{b as d,c as m}from"../chunk-2E5MDUA2.js";import"dotenv/config";import{z as s}from"zod";import{eq as T}from"drizzle-orm";import{createPublicClient as C,fallback as g,webSocket as E,http as _}from"viem";import{isDefined as h}from"@latticexyz/common/utils";import{combineLatest as A,filter as L,first as R}from"rxjs";import{drizzle as S}from"drizzle-orm/postgres-js";import w from"postgres";import{cleanDatabase as k,createStorageAdapter as O,shouldCleanDatabase as P}from"@latticexyz/store-sync/postgres";import{createStoreSync as K}from"@latticexyz/store-sync";var t=m(s.intersection(d,s.object({DATABASE_URL:s.string(),HEALTHCHECK_HOST:s.string().optional(),HEALTHCHECK_PORT:s.coerce.number().optional()}))),B=[t.RPC_WS_URL?E(t.RPC_WS_URL):void 0,t.RPC_HTTP_URL?_(t.RPC_HTTP_URL):void 0].filter(h),c=C({transport:g(B),pollingInterval:t.POLLING_INTERVAL}),p=await c.getChainId(),i=S(w(t.DATABASE_URL));await P(i,p)&&(console.log("outdated database detected, clearing data to start fresh"),k(i));var{storageAdapter:U,tables:u}=await O({database:i,publicClient:c}),l=t.START_BLOCK;try{let o=await i.select().from(u.configTable).where(T(u.configTable.chainId,p)).limit(1).execute().then(r=>r.find(()=>!0));o?.lastUpdatedBlockNumber!=null&&(l=o.lastUpdatedBlockNumber+1n,console.log("resuming from block number",l))}catch{}var{latestBlockNumber$:v,storedBlockLogs$:f}=await K({storageAdapter:U,publicClient:c,startBlock:l,maxBlockRange:t.MAX_BLOCK_RANGE,address:t.STORE_ADDRESS});f.subscribe();var H=!1;A([v,f]).pipe(L(([o,{blockNumber:r}])=>o===r),R()).subscribe(()=>{H=!0,console.log("all caught up")});if(t.HEALTHCHECK_HOST!=null||t.HEALTHCHECK_PORT!=null){let{default:o}=await import("koa"),{default:r}=await import("@koa/cors"),{default:b}=await import("@koa/router"),n=new o;n.use(r());let a=new b;a.get("/",e=>{e.body="emit HelloWorld();"}),a.get("/healthz",e=>{e.status=200}),a.get("/readyz",e=>{H?(e.status=200,e.body="ready"):(e.status=424,e.body="backfilling")}),n.use(a.routes()),n.use(a.allowedMethods()),n.listen({host:t.HEALTHCHECK_HOST,port:t.HEALTHCHECK_PORT}),console.log(`postgres indexer healthcheck server listening on http://${t.HEALTHCHECK_HOST}:${t.HEALTHCHECK_PORT}`)}
|
|
3
3
|
//# sourceMappingURL=postgres-indexer.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../bin/postgres-indexer.ts"],"sourcesContent":["#!/usr/bin/env node\nimport \"dotenv/config\";\nimport { z } from \"zod\";\nimport { eq } from \"drizzle-orm\";\nimport { createPublicClient, fallback, webSocket, http, Transport } from \"viem\";\nimport { isDefined } from \"@latticexyz/common/utils\";\nimport { combineLatest, filter, first } from \"rxjs\";\nimport { drizzle } from \"drizzle-orm/postgres-js\";\nimport postgres from \"postgres\";\nimport { cleanDatabase, createStorageAdapter, shouldCleanDatabase } from \"@latticexyz/store-sync/postgres\";\nimport { createStoreSync } from \"@latticexyz/store-sync\";\nimport { indexerEnvSchema, parseEnv } from \"./parseEnv\";\n\nconst env = parseEnv(\n z.intersection(\n indexerEnvSchema,\n z.object({\n DATABASE_URL: z.string(),\n HEALTHCHECK_HOST: z.string().optional(),\n HEALTHCHECK_PORT: z.coerce.number().optional(),\n })\n )\n);\n\nconst transports: Transport[] = [\n // prefer WS when specified\n env.RPC_WS_URL ? webSocket(env.RPC_WS_URL) : undefined,\n // otherwise use or fallback to HTTP\n env.RPC_HTTP_URL ? http(env.RPC_HTTP_URL) : undefined,\n].filter(isDefined);\n\nconst publicClient = createPublicClient({\n transport: fallback(transports),\n pollingInterval: env.POLLING_INTERVAL,\n});\n\nconst chainId = await publicClient.getChainId();\nconst database = drizzle(postgres(env.DATABASE_URL));\n\nif (await shouldCleanDatabase(database, chainId)) {\n console.log(\"outdated database detected, clearing data to start fresh\");\n cleanDatabase(database);\n}\n\nconst { storageAdapter, tables } = await createStorageAdapter({ database, publicClient });\n\nlet startBlock = env.START_BLOCK;\n\n// Resume from latest block stored in DB. This will throw if the DB doesn't exist yet, so we wrap in a try/catch and ignore the error.\n// TODO: query if the DB exists instead of try/catch\ntry {\n const chainState = await database\n .select()\n .from(tables.configTable)\n .where(eq(tables.configTable.chainId, chainId))\n .limit(1)\n .execute()\n // Get the first record in a way that returns a possible `undefined`\n // TODO: move this to `.findFirst` after upgrading drizzle or `rows[0]` after enabling `noUncheckedIndexedAccess: true`\n .then((rows) => rows.find(() => true));\n\n if (chainState?.lastUpdatedBlockNumber != null) {\n startBlock = chainState.lastUpdatedBlockNumber + 1n;\n console.log(\"resuming from block number\", startBlock);\n }\n} catch (error) {\n // ignore errors for now\n}\n\nconst { latestBlockNumber$, storedBlockLogs$ } = await createStoreSync({\n storageAdapter,\n publicClient,\n startBlock,\n maxBlockRange: env.MAX_BLOCK_RANGE,\n address: env.STORE_ADDRESS,\n});\n\nstoredBlockLogs$.subscribe();\n\nlet isCaughtUp = false;\ncombineLatest([latestBlockNumber$, storedBlockLogs$])\n .pipe(\n filter(\n ([latestBlockNumber, { blockNumber: lastBlockNumberProcessed }]) => latestBlockNumber === lastBlockNumberProcessed\n ),\n first()\n )\n .subscribe(() => {\n isCaughtUp = true;\n console.log(\"all caught up\");\n });\n\nif (env.HEALTHCHECK_HOST != null || env.HEALTHCHECK_PORT != null) {\n const { default:
|
|
1
|
+
{"version":3,"sources":["../../bin/postgres-indexer.ts"],"sourcesContent":["#!/usr/bin/env node\nimport \"dotenv/config\";\nimport { z } from \"zod\";\nimport { eq } from \"drizzle-orm\";\nimport { createPublicClient, fallback, webSocket, http, Transport } from \"viem\";\nimport { isDefined } from \"@latticexyz/common/utils\";\nimport { combineLatest, filter, first } from \"rxjs\";\nimport { drizzle } from \"drizzle-orm/postgres-js\";\nimport postgres from \"postgres\";\nimport { cleanDatabase, createStorageAdapter, shouldCleanDatabase } from \"@latticexyz/store-sync/postgres\";\nimport { createStoreSync } from \"@latticexyz/store-sync\";\nimport { indexerEnvSchema, parseEnv } from \"./parseEnv\";\n\nconst env = parseEnv(\n z.intersection(\n indexerEnvSchema,\n z.object({\n DATABASE_URL: z.string(),\n HEALTHCHECK_HOST: z.string().optional(),\n HEALTHCHECK_PORT: z.coerce.number().optional(),\n })\n )\n);\n\nconst transports: Transport[] = [\n // prefer WS when specified\n env.RPC_WS_URL ? webSocket(env.RPC_WS_URL) : undefined,\n // otherwise use or fallback to HTTP\n env.RPC_HTTP_URL ? http(env.RPC_HTTP_URL) : undefined,\n].filter(isDefined);\n\nconst publicClient = createPublicClient({\n transport: fallback(transports),\n pollingInterval: env.POLLING_INTERVAL,\n});\n\nconst chainId = await publicClient.getChainId();\nconst database = drizzle(postgres(env.DATABASE_URL));\n\nif (await shouldCleanDatabase(database, chainId)) {\n console.log(\"outdated database detected, clearing data to start fresh\");\n cleanDatabase(database);\n}\n\nconst { storageAdapter, tables } = await createStorageAdapter({ database, publicClient });\n\nlet startBlock = env.START_BLOCK;\n\n// Resume from latest block stored in DB. This will throw if the DB doesn't exist yet, so we wrap in a try/catch and ignore the error.\n// TODO: query if the DB exists instead of try/catch\ntry {\n const chainState = await database\n .select()\n .from(tables.configTable)\n .where(eq(tables.configTable.chainId, chainId))\n .limit(1)\n .execute()\n // Get the first record in a way that returns a possible `undefined`\n // TODO: move this to `.findFirst` after upgrading drizzle or `rows[0]` after enabling `noUncheckedIndexedAccess: true`\n .then((rows) => rows.find(() => true));\n\n if (chainState?.lastUpdatedBlockNumber != null) {\n startBlock = chainState.lastUpdatedBlockNumber + 1n;\n console.log(\"resuming from block number\", startBlock);\n }\n} catch (error) {\n // ignore errors for now\n}\n\nconst { latestBlockNumber$, storedBlockLogs$ } = await createStoreSync({\n storageAdapter,\n publicClient,\n startBlock,\n maxBlockRange: env.MAX_BLOCK_RANGE,\n address: env.STORE_ADDRESS,\n});\n\nstoredBlockLogs$.subscribe();\n\nlet isCaughtUp = false;\ncombineLatest([latestBlockNumber$, storedBlockLogs$])\n .pipe(\n filter(\n ([latestBlockNumber, { blockNumber: lastBlockNumberProcessed }]) => latestBlockNumber === lastBlockNumberProcessed\n ),\n first()\n )\n .subscribe(() => {\n isCaughtUp = true;\n console.log(\"all caught up\");\n });\n\nif (env.HEALTHCHECK_HOST != null || env.HEALTHCHECK_PORT != null) {\n const { default: Koa } = await import(\"koa\");\n const { default: cors } = await import(\"@koa/cors\");\n const { default: Router } = await import(\"@koa/router\");\n\n const server = new Koa();\n server.use(cors());\n\n const router = new Router();\n\n router.get(\"/\", (ctx) => {\n ctx.body = \"emit HelloWorld();\";\n });\n\n // k8s healthchecks\n router.get(\"/healthz\", (ctx) => {\n ctx.status = 200;\n });\n router.get(\"/readyz\", (ctx) => {\n if (isCaughtUp) {\n ctx.status = 200;\n ctx.body = \"ready\";\n } else {\n ctx.status = 424;\n ctx.body = \"backfilling\";\n }\n });\n\n server.use(router.routes());\n server.use(router.allowedMethods());\n\n server.listen({ host: env.HEALTHCHECK_HOST, port: env.HEALTHCHECK_PORT });\n console.log(\n `postgres indexer healthcheck server listening on http://${env.HEALTHCHECK_HOST}:${env.HEALTHCHECK_PORT}`\n );\n}\n"],"mappings":";gDACA,MAAO,gBACP,OAAS,KAAAA,MAAS,MAClB,OAAS,MAAAC,MAAU,cACnB,OAAS,sBAAAC,EAAoB,YAAAC,EAAU,aAAAC,EAAW,QAAAC,MAAuB,OACzE,OAAS,aAAAC,MAAiB,2BAC1B,OAAS,iBAAAC,EAAe,UAAAC,EAAQ,SAAAC,MAAa,OAC7C,OAAS,WAAAC,MAAe,0BACxB,OAAOC,MAAc,WACrB,OAAS,iBAAAC,EAAe,wBAAAC,EAAsB,uBAAAC,MAA2B,kCACzE,OAAS,mBAAAC,MAAuB,yBAGhC,IAAMC,EAAMC,EACVC,EAAE,aACAC,EACAD,EAAE,OAAO,CACP,aAAcA,EAAE,OAAO,EACvB,iBAAkBA,EAAE,OAAO,EAAE,SAAS,EACtC,iBAAkBA,EAAE,OAAO,OAAO,EAAE,SAAS,CAC/C,CAAC,CACH,CACF,EAEME,EAA0B,CAE9BJ,EAAI,WAAaK,EAAUL,EAAI,UAAU,EAAI,OAE7CA,EAAI,aAAeM,EAAKN,EAAI,YAAY,EAAI,MAC9C,EAAE,OAAOO,CAAS,EAEZC,EAAeC,EAAmB,CACtC,UAAWC,EAASN,CAAU,EAC9B,gBAAiBJ,EAAI,gBACvB,CAAC,EAEKW,EAAU,MAAMH,EAAa,WAAW,EACxCI,EAAWC,EAAQC,EAASd,EAAI,YAAY,CAAC,EAE/C,MAAMe,EAAoBH,EAAUD,CAAO,IAC7C,QAAQ,IAAI,0DAA0D,EACtEK,EAAcJ,CAAQ,GAGxB,GAAM,CAAE,eAAAK,EAAgB,OAAAC,CAAO,EAAI,MAAMC,EAAqB,CAAE,SAAAP,EAAU,aAAAJ,CAAa,CAAC,EAEpFY,EAAapB,EAAI,YAIrB,GAAI,CACF,IAAMqB,EAAa,MAAMT,EACtB,OAAO,EACP,KAAKM,EAAO,WAAW,EACvB,MAAMI,EAAGJ,EAAO,YAAY,QAASP,CAAO,CAAC,EAC7C,MAAM,CAAC,EACP,QAAQ,EAGR,KAAMY,GAASA,EAAK,KAAK,IAAM,EAAI,CAAC,EAEnCF,GAAY,wBAA0B,OACxCD,EAAaC,EAAW,uBAAyB,GACjD,QAAQ,IAAI,6BAA8BD,CAAU,EAExD,MAAE,CAEF,CAEA,GAAM,CAAE,mBAAAI,EAAoB,iBAAAC,CAAiB,EAAI,MAAMC,EAAgB,CACrE,eAAAT,EACA,aAAAT,EACA,WAAAY,EACA,cAAepB,EAAI,gBACnB,QAASA,EAAI,aACf,CAAC,EAEDyB,EAAiB,UAAU,EAE3B,IAAIE,EAAa,GACjBC,EAAc,CAACJ,EAAoBC,CAAgB,CAAC,EACjD,KACCI,EACE,CAAC,CAACC,EAAmB,CAAE,YAAaC,CAAyB,CAAC,IAAMD,IAAsBC,CAC5F,EACAC,EAAM,CACR,EACC,UAAU,IAAM,CACfL,EAAa,GACb,QAAQ,IAAI,eAAe,CAC7B,CAAC,EAEH,GAAI3B,EAAI,kBAAoB,MAAQA,EAAI,kBAAoB,KAAM,CAChE,GAAM,CAAE,QAASiC,CAAI,EAAI,KAAM,QAAO,KAAK,EACrC,CAAE,QAASC,CAAK,EAAI,KAAM,QAAO,WAAW,EAC5C,CAAE,QAASC,CAAO,EAAI,KAAM,QAAO,aAAa,EAEhDC,EAAS,IAAIH,EACnBG,EAAO,IAAIF,EAAK,CAAC,EAEjB,IAAMG,EAAS,IAAIF,EAEnBE,EAAO,IAAI,IAAMC,GAAQ,CACvBA,EAAI,KAAO,oBACb,CAAC,EAGDD,EAAO,IAAI,WAAaC,GAAQ,CAC9BA,EAAI,OAAS,GACf,CAAC,EACDD,EAAO,IAAI,UAAYC,GAAQ,CACzBX,GACFW,EAAI,OAAS,IACbA,EAAI,KAAO,UAEXA,EAAI,OAAS,IACbA,EAAI,KAAO,cAEf,CAAC,EAEDF,EAAO,IAAIC,EAAO,OAAO,CAAC,EAC1BD,EAAO,IAAIC,EAAO,eAAe,CAAC,EAElCD,EAAO,OAAO,CAAE,KAAMpC,EAAI,iBAAkB,KAAMA,EAAI,gBAAiB,CAAC,EACxE,QAAQ,IACN,2DAA2DA,EAAI,oBAAoBA,EAAI,kBACzF","names":["z","eq","createPublicClient","fallback","webSocket","http","isDefined","combineLatest","filter","first","drizzle","postgres","cleanDatabase","createStorageAdapter","shouldCleanDatabase","createStoreSync","env","parseEnv","z","indexerEnvSchema","transports","webSocket","http","isDefined","publicClient","createPublicClient","fallback","chainId","database","drizzle","postgres","shouldCleanDatabase","cleanDatabase","storageAdapter","tables","createStorageAdapter","startBlock","chainState","eq","rows","latestBlockNumber$","storedBlockLogs$","createStoreSync","isCaughtUp","combineLatest","filter","latestBlockNumber","lastBlockNumberProcessed","first","Koa","cors","Router","server","router","ctx"]}
|
|
@@ -1,3 +1,3 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import{a as
|
|
2
|
+
import{a as S,b as g,c as T}from"../chunk-2E5MDUA2.js";import"dotenv/config";import U from"node:fs";import{z as u}from"zod";import{eq as W}from"drizzle-orm";import{drizzle as x}from"drizzle-orm/better-sqlite3";import H from"better-sqlite3";import{createPublicClient as q,fallback as F,webSocket as M,http as j}from"viem";import z from"koa";import K from"@koa/cors";import V from"@koa/router";import{createKoaMiddleware as $}from"trpc-koa-adapter";import{createAppRouter as G}from"@latticexyz/store-sync/trpc-indexer";import{chainState as I,schemaVersion as A,syncToSqlite as X}from"@latticexyz/store-sync/sqlite";import{asc as O,eq as k}from"drizzle-orm";import{buildTable as Q,chainState as R,getTables as P}from"@latticexyz/store-sync/sqlite";import{getAddress as L}from"viem";import{decodeDynamicField as v}from"@latticexyz/protocol-parser";function f(e,{chainId:r,address:n,filters:s=[]}){let p=e.select().from(R).where(k(R.chainId,r)).limit(1).all().find(()=>!0),m=Array.from(new Set(s.map(o=>o.tableId))),C=P(e).filter(o=>n==null||L(n)===L(o.address)).filter(o=>!m.length||m.includes(o.tableId)).map(o=>{let b=Q(o),h=e.select().from(b).where(k(b.__isDeleted,!1)).orderBy(O(b.__lastUpdatedBlockNumber)).all(),w=s.length?h.filter(d=>{let a=v("bytes32[]",d.__key);return s.some(l=>l.tableId===o.tableId&&(l.key0==null||l.key0===a[0])&&(l.key1==null||l.key1===a[1]))}):h;return{...o,records:w.map(d=>({key:Object.fromEntries(Object.entries(o.keySchema).map(([a])=>[a,d[a]])),value:Object.fromEntries(Object.entries(o.valueSchema).map(([a])=>[a,d[a]]))}))}});return{blockNumber:p?.lastUpdatedBlockNumber??null,tables:C}}import{tablesWithRecordsToLogs as D}from"@latticexyz/store-sync";async function _(e){return{async getLogs(n){let{blockNumber:s,tables:p}=f(e,n),m=D(p);return{blockNumber:s??0n,logs:m}},async findAll(n){return f(e,n)}}}import{isDefined as J}from"@latticexyz/common/utils";import{combineLatest as Y,filter as Z,first as ee}from"rxjs";var t=T(u.intersection(u.intersection(g,S),u.object({SQLITE_FILENAME:u.string().default("indexer.db")}))),te=[t.RPC_WS_URL?M(t.RPC_WS_URL):void 0,t.RPC_HTTP_URL?j(t.RPC_HTTP_URL):void 0].filter(J),E=q({transport:F(te),pollingInterval:t.POLLING_INTERVAL}),re=await E.getChainId(),y=x(new H(t.SQLITE_FILENAME)),N=t.START_BLOCK;try{let r=y.select().from(I).where(W(I.chainId,re)).all()[0];r!=null&&(r.schemaVersion!=A?(console.log("schema version changed from",r.schemaVersion,"to",A,"recreating database"),U.truncateSync(t.SQLITE_FILENAME)):r.lastUpdatedBlockNumber!=null&&(console.log("resuming from block number",r.lastUpdatedBlockNumber+1n),N=r.lastUpdatedBlockNumber+1n))}catch{}var{latestBlockNumber$:oe,storedBlockLogs$:ae}=await X({database:y,publicClient:E,startBlock:N,maxBlockRange:t.MAX_BLOCK_RANGE,address:t.STORE_ADDRESS}),B=!1;Y([oe,ae]).pipe(Z(([e,{blockNumber:r}])=>e===r),ee()).subscribe(()=>{B=!0,console.log("all caught up")});var i=new z;i.use(K());var c=new V;c.get("/",e=>{e.body="emit HelloWorld();"});c.get("/healthz",e=>{e.status=200});c.get("/readyz",e=>{B?(e.status=200,e.body="ready"):(e.status=424,e.body="backfilling")});i.use(c.routes());i.use(c.allowedMethods());i.use($({prefix:"/trpc",router:G(),createContext:async()=>({queryAdapter:await _(y)})}));i.listen({host:t.HOST,port:t.PORT});console.log(`sqlite indexer frontend listening on http://${t.HOST}:${t.PORT}`);
|
|
3
3
|
//# sourceMappingURL=sqlite-indexer.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../bin/sqlite-indexer.ts","../../src/sqlite/getTablesWithRecords.ts","../../src/sqlite/createQueryAdapter.ts"],"sourcesContent":["#!/usr/bin/env node\nimport \"dotenv/config\";\nimport fs from \"node:fs\";\nimport { z } from \"zod\";\nimport { eq } from \"drizzle-orm\";\nimport { drizzle } from \"drizzle-orm/better-sqlite3\";\nimport Database from \"better-sqlite3\";\nimport { createPublicClient, fallback, webSocket, http, Transport } from \"viem\";\nimport fastify from \"fastify\";\nimport { fastifyTRPCPlugin } from \"@trpc/server/adapters/fastify\";\nimport { AppRouter, createAppRouter } from \"@latticexyz/store-sync/trpc-indexer\";\nimport { chainState, schemaVersion, syncToSqlite } from \"@latticexyz/store-sync/sqlite\";\nimport { createQueryAdapter } from \"../src/sqlite/createQueryAdapter\";\nimport { isDefined } from \"@latticexyz/common/utils\";\nimport { combineLatest, filter, first } from \"rxjs\";\nimport { frontendEnvSchema, indexerEnvSchema, parseEnv } from \"./parseEnv\";\n\nconst env = parseEnv(\n z.intersection(\n z.intersection(indexerEnvSchema, frontendEnvSchema),\n z.object({\n SQLITE_FILENAME: z.string().default(\"indexer.db\"),\n })\n )\n);\n\nconst transports: Transport[] = [\n // prefer WS when specified\n env.RPC_WS_URL ? webSocket(env.RPC_WS_URL) : undefined,\n // otherwise use or fallback to HTTP\n env.RPC_HTTP_URL ? http(env.RPC_HTTP_URL) : undefined,\n].filter(isDefined);\n\nconst publicClient = createPublicClient({\n transport: fallback(transports),\n pollingInterval: env.POLLING_INTERVAL,\n});\n\nconst chainId = await publicClient.getChainId();\nconst database = drizzle(new Database(env.SQLITE_FILENAME));\n\nlet startBlock = env.START_BLOCK;\n\n// Resume from latest block stored in DB. This will throw if the DB doesn't exist yet, so we wrap in a try/catch and ignore the error.\ntry {\n const currentChainStates = database.select().from(chainState).where(eq(chainState.chainId, chainId)).all();\n // TODO: replace this type workaround with `noUncheckedIndexedAccess: true` when we can fix all the issues related (https://github.com/latticexyz/mud/issues/1212)\n const currentChainState: (typeof currentChainStates)[number] | undefined = currentChainStates[0];\n\n if (currentChainState != null) {\n if (currentChainState.schemaVersion != schemaVersion) {\n console.log(\n \"schema version changed from\",\n currentChainState.schemaVersion,\n \"to\",\n schemaVersion,\n \"recreating database\"\n );\n fs.truncateSync(env.SQLITE_FILENAME);\n } else if (currentChainState.lastUpdatedBlockNumber != null) {\n console.log(\"resuming from block number\", currentChainState.lastUpdatedBlockNumber + 1n);\n startBlock = currentChainState.lastUpdatedBlockNumber + 1n;\n }\n }\n} catch (error) {\n // ignore errors, this is optional\n}\n\nconst { latestBlockNumber$, storedBlockLogs$ } = await syncToSqlite({\n database,\n publicClient,\n startBlock,\n maxBlockRange: env.MAX_BLOCK_RANGE,\n address: env.STORE_ADDRESS,\n});\n\nlet isCaughtUp = false;\ncombineLatest([latestBlockNumber$, storedBlockLogs$])\n .pipe(\n filter(\n ([latestBlockNumber, { blockNumber: lastBlockNumberProcessed }]) => latestBlockNumber === lastBlockNumberProcessed\n ),\n first()\n )\n .subscribe(() => {\n isCaughtUp = true;\n console.log(\"all caught up\");\n });\n\n// @see https://fastify.dev/docs/latest/\nconst server = fastify({\n maxParamLength: 5000,\n});\n\nawait server.register(import(\"@fastify/compress\"));\nawait server.register(import(\"@fastify/cors\"));\n\n// k8s healthchecks\nserver.get(\"/healthz\", (req, res) => res.code(200).send());\nserver.get(\"/readyz\", (req, res) => (isCaughtUp ? res.code(200).send(\"ready\") : res.code(424).send(\"backfilling\")));\n\n// @see https://trpc.io/docs/server/adapters/fastify\nserver.register(fastifyTRPCPlugin<AppRouter>, {\n prefix: \"/trpc\",\n trpcOptions: {\n router: createAppRouter(),\n createContext: async () => ({\n queryAdapter: await createQueryAdapter(database),\n }),\n },\n});\n\nawait server.listen({ host: env.HOST, port: env.PORT });\nconsole.log(`sqlite indexer frontend listening on http://${env.HOST}:${env.PORT}`);\n","import { asc, eq } from \"drizzle-orm\";\nimport { BaseSQLiteDatabase } from \"drizzle-orm/sqlite-core\";\nimport { buildTable, chainState, getTables } from \"@latticexyz/store-sync/sqlite\";\nimport { Hex, getAddress } from \"viem\";\nimport { decodeDynamicField } from \"@latticexyz/protocol-parser\";\nimport { SyncFilter, TableWithRecords } from \"@latticexyz/store-sync\";\n\n// TODO: refactor sqlite and replace this with getLogs to match postgres (https://github.com/latticexyz/mud/issues/1970)\n\n/**\n * @deprecated\n * */\nexport function getTablesWithRecords(\n database: BaseSQLiteDatabase<\"sync\", any>,\n {\n chainId,\n address,\n filters = [],\n }: {\n readonly chainId: number;\n readonly address?: Hex;\n readonly filters?: readonly SyncFilter[];\n }\n): { blockNumber: bigint | null; tables: readonly TableWithRecords[] } {\n const metadata = database\n .select()\n .from(chainState)\n .where(eq(chainState.chainId, chainId))\n .limit(1)\n .all()\n .find(() => true);\n\n // If _any_ filter has a table ID, this will filter down all data to just those tables. Which mean we can't yet mix table filters with key-only filters.\n // TODO: improve this so we can express this in the query (need to be able to query data across tables more easily)\n const tableIds = Array.from(new Set(filters.map((filter) => filter.tableId)));\n const tables = getTables(database)\n .filter((table) => address == null || getAddress(address) === getAddress(table.address))\n .filter((table) => !tableIds.length || tableIds.includes(table.tableId));\n\n const tablesWithRecords = tables.map((table) => {\n const sqliteTable = buildTable(table);\n const records = database\n .select()\n .from(sqliteTable)\n .where(eq(sqliteTable.__isDeleted, false))\n .orderBy(\n asc(sqliteTable.__lastUpdatedBlockNumber)\n // TODO: add logIndex (https://github.com/latticexyz/mud/issues/1979)\n )\n .all();\n const filteredRecords = !filters.length\n ? records\n : records.filter((record) => {\n const keyTuple = decodeDynamicField(\"bytes32[]\", record.__key);\n return filters.some(\n (filter) =>\n filter.tableId === table.tableId &&\n (filter.key0 == null || filter.key0 === keyTuple[0]) &&\n (filter.key1 == null || filter.key1 === keyTuple[1])\n );\n });\n return {\n ...table,\n records: filteredRecords.map((record) => ({\n key: Object.fromEntries(Object.entries(table.keySchema).map(([name]) => [name, record[name]])),\n value: Object.fromEntries(Object.entries(table.valueSchema).map(([name]) => [name, record[name]])),\n })),\n };\n });\n\n return {\n blockNumber: metadata?.lastUpdatedBlockNumber ?? null,\n tables: tablesWithRecords,\n };\n}\n","import { BaseSQLiteDatabase } from \"drizzle-orm/sqlite-core\";\nimport { QueryAdapter } from \"@latticexyz/store-sync/trpc-indexer\";\nimport { getTablesWithRecords } from \"./getTablesWithRecords\";\nimport { tablesWithRecordsToLogs } from \"@latticexyz/store-sync\";\n\n/**\n * Creates a storage adapter for the tRPC server/client to query data from SQLite.\n *\n * @param {BaseSQLiteDatabase<\"sync\", any>} database SQLite database object from Drizzle\n * @returns {Promise<QueryAdapter>} A set of methods used by tRPC endpoints.\n */\nexport async function createQueryAdapter(database: BaseSQLiteDatabase<\"sync\", any>): Promise<QueryAdapter> {\n const adapter: QueryAdapter = {\n async getLogs(opts) {\n const { blockNumber, tables } = getTablesWithRecords(database, opts);\n const logs = tablesWithRecordsToLogs(tables);\n return { blockNumber: blockNumber ?? 0n, logs };\n },\n async findAll(opts) {\n return getTablesWithRecords(database, opts);\n },\n };\n return adapter;\n}\n"],"mappings":";uDACA,MAAO,gBACP,OAAOA,MAAQ,UACf,OAAS,KAAAC,MAAS,MAClB,OAAS,MAAAC,MAAU,cACnB,OAAS,WAAAC,MAAe,6BACxB,OAAOC,MAAc,iBACrB,OAAS,sBAAAC,EAAoB,YAAAC,EAAU,aAAAC,EAAW,QAAAC,MAAuB,OACzE,OAAOC,MAAa,UACpB,OAAS,qBAAAC,MAAyB,gCAClC,OAAoB,mBAAAC,MAAuB,sCAC3C,OAAS,cAAAC,EAAY,iBAAAC,EAAe,gBAAAC,MAAoB,gCCXxD,OAAS,OAAAC,EAAK,MAAAC,MAAU,cAExB,OAAS,cAAAC,EAAY,cAAAC,EAAY,aAAAC,MAAiB,gCAClD,OAAc,cAAAC,MAAkB,OAChC,OAAS,sBAAAC,MAA0B,8BAQ5B,SAASC,EACdC,EACA,CACE,QAAAC,EACA,QAAAC,EACA,QAAAC,EAAU,CAAC,CACb,EAKqE,CACrE,IAAMC,EAAWJ,EACd,OAAO,EACP,KAAKL,CAAU,EACf,MAAMF,EAAGE,EAAW,QAASM,CAAO,CAAC,EACrC,MAAM,CAAC,EACP,IAAI,EACJ,KAAK,IAAM,EAAI,EAIZI,EAAW,MAAM,KAAK,IAAI,IAAIF,EAAQ,IAAKG,GAAWA,EAAO,OAAO,CAAC,CAAC,EAKtEC,EAJSX,EAAUI,CAAQ,EAC9B,OAAQQ,GAAUN,GAAW,MAAQL,EAAWK,CAAO,IAAML,EAAWW,EAAM,OAAO,CAAC,EACtF,OAAQA,GAAU,CAACH,EAAS,QAAUA,EAAS,SAASG,EAAM,OAAO,CAAC,EAExC,IAAKA,GAAU,CAC9C,IAAMC,EAAcf,EAAWc,CAAK,EAC9BE,EAAUV,EACb,OAAO,EACP,KAAKS,CAAW,EAChB,MAAMhB,EAAGgB,EAAY,YAAa,EAAK,CAAC,EACxC,QACCjB,EAAIiB,EAAY,wBAAwB,CAE1C,EACC,IAAI,EACDE,EAAmBR,EAAQ,OAE7BO,EAAQ,OAAQE,GAAW,CACzB,IAAMC,EAAWf,EAAmB,YAAac,EAAO,KAAK,EAC7D,OAAOT,EAAQ,KACZG,GACCA,EAAO,UAAYE,EAAM,UACxBF,EAAO,MAAQ,MAAQA,EAAO,OAASO,EAAS,CAAC,KACjDP,EAAO,MAAQ,MAAQA,EAAO,OAASO,EAAS,CAAC,EACtD,CACF,CAAC,EATDH,EAUJ,MAAO,CACL,GAAGF,EACH,QAASG,EAAgB,IAAKC,IAAY,CACxC,IAAK,OAAO,YAAY,OAAO,QAAQJ,EAAM,SAAS,EAAE,IAAI,CAAC,CAACM,CAAI,IAAM,CAACA,EAAMF,EAAOE,CAAI,CAAC,CAAC,CAAC,EAC7F,MAAO,OAAO,YAAY,OAAO,QAAQN,EAAM,WAAW,EAAE,IAAI,CAAC,CAACM,CAAI,IAAM,CAACA,EAAMF,EAAOE,CAAI,CAAC,CAAC,CAAC,CACnG,EAAE,CACJ,CACF,CAAC,EAED,MAAO,CACL,YAAaV,GAAU,wBAA0B,KACjD,OAAQG,CACV,CACF,CCvEA,OAAS,2BAAAQ,MAA+B,yBAQxC,eAAsBC,EAAmBC,EAAkE,CAWzG,MAV8B,CAC5B,MAAM,QAAQC,EAAM,CAClB,GAAM,CAAE,YAAAC,EAAa,OAAAC,CAAO,EAAIC,EAAqBJ,EAAUC,CAAI,EAC7DI,EAAOP,EAAwBK,CAAM,EAC3C,MAAO,CAAE,YAAaD,GAAe,GAAI,KAAAG,CAAK,CAChD,EACA,MAAM,QAAQJ,EAAM,CAClB,OAAOG,EAAqBJ,EAAUC,CAAI,CAC5C,CACF,CAEF,CFVA,OAAS,aAAAK,MAAiB,2BAC1B,OAAS,iBAAAC,EAAe,UAAAC,EAAQ,SAAAC,MAAa,OAG7C,IAAMC,EAAMC,EACVC,EAAE,aACAA,EAAE,aAAaC,EAAkBC,CAAiB,EAClDF,EAAE,OAAO,CACP,gBAAiBA,EAAE,OAAO,EAAE,QAAQ,YAAY,CAClD,CAAC,CACH,CACF,EAEMG,EAA0B,CAE9BL,EAAI,WAAaM,EAAUN,EAAI,UAAU,EAAI,OAE7CA,EAAI,aAAeO,EAAKP,EAAI,YAAY,EAAI,MAC9C,EAAE,OAAOQ,CAAS,EAEZC,EAAeC,EAAmB,CACtC,UAAWC,EAASN,CAAU,EAC9B,gBAAiBL,EAAI,gBACvB,CAAC,EAEKY,EAAU,MAAMH,EAAa,WAAW,EACxCI,EAAWC,EAAQ,IAAIC,EAASf,EAAI,eAAe,CAAC,EAEtDgB,EAAahB,EAAI,YAGrB,GAAI,CAGF,IAAMiB,EAFqBJ,EAAS,OAAO,EAAE,KAAKK,CAAU,EAAE,MAAMC,EAAGD,EAAW,QAASN,CAAO,CAAC,EAAE,IAAI,EAEX,CAAC,EAE3FK,GAAqB,OACnBA,EAAkB,eAAiBG,GACrC,QAAQ,IACN,8BACAH,EAAkB,cAClB,KACAG,EACA,qBACF,EACAC,EAAG,aAAarB,EAAI,eAAe,GAC1BiB,EAAkB,wBAA0B,OACrD,QAAQ,IAAI,6BAA8BA,EAAkB,uBAAyB,EAAE,EACvFD,EAAaC,EAAkB,uBAAyB,IAG9D,MAAE,CAEF,CAEA,GAAM,CAAE,mBAAAK,GAAoB,iBAAAC,EAAiB,EAAI,MAAMC,EAAa,CAClE,SAAAX,EACA,aAAAJ,EACA,WAAAO,EACA,cAAehB,EAAI,gBACnB,QAASA,EAAI,aACf,CAAC,EAEGyB,EAAa,GACjBC,EAAc,CAACJ,GAAoBC,EAAgB,CAAC,EACjD,KACCI,EACE,CAAC,CAACC,EAAmB,CAAE,YAAaC,CAAyB,CAAC,IAAMD,IAAsBC,CAC5F,EACAC,EAAM,CACR,EACC,UAAU,IAAM,CACfL,EAAa,GACb,QAAQ,IAAI,eAAe,CAC7B,CAAC,EAGH,IAAMM,EAASC,EAAQ,CACrB,eAAgB,GAClB,CAAC,EAED,MAAMD,EAAO,SAAS,OAAO,mBAAmB,CAAC,EACjD,MAAMA,EAAO,SAAS,OAAO,eAAe,CAAC,EAG7CA,EAAO,IAAI,WAAY,CAACE,EAAKC,IAAQA,EAAI,KAAK,GAAG,EAAE,KAAK,CAAC,EACzDH,EAAO,IAAI,UAAW,CAACE,EAAKC,IAAST,EAAaS,EAAI,KAAK,GAAG,EAAE,KAAK,OAAO,EAAIA,EAAI,KAAK,GAAG,EAAE,KAAK,aAAa,CAAE,EAGlHH,EAAO,SAASI,EAA8B,CAC5C,OAAQ,QACR,YAAa,CACX,OAAQC,EAAgB,EACxB,cAAe,UAAa,CAC1B,aAAc,MAAMC,EAAmBxB,CAAQ,CACjD,EACF,CACF,CAAC,EAED,MAAMkB,EAAO,OAAO,CAAE,KAAM/B,EAAI,KAAM,KAAMA,EAAI,IAAK,CAAC,EACtD,QAAQ,IAAI,+CAA+CA,EAAI,QAAQA,EAAI,MAAM","names":["fs","z","eq","drizzle","Database","createPublicClient","fallback","webSocket","http","fastify","fastifyTRPCPlugin","createAppRouter","chainState","schemaVersion","syncToSqlite","asc","eq","buildTable","chainState","getTables","getAddress","decodeDynamicField","getTablesWithRecords","database","chainId","address","filters","metadata","tableIds","filter","tablesWithRecords","table","sqliteTable","records","filteredRecords","record","keyTuple","name","tablesWithRecordsToLogs","createQueryAdapter","database","opts","blockNumber","tables","getTablesWithRecords","logs","isDefined","combineLatest","filter","first","env","parseEnv","z","indexerEnvSchema","frontendEnvSchema","transports","webSocket","http","isDefined","publicClient","createPublicClient","fallback","chainId","database","drizzle","Database","startBlock","currentChainState","chainState","eq","schemaVersion","fs","latestBlockNumber$","storedBlockLogs$","syncToSqlite","isCaughtUp","combineLatest","filter","latestBlockNumber","lastBlockNumberProcessed","first","server","fastify","req","res","fastifyTRPCPlugin","createAppRouter","createQueryAdapter"]}
|
|
1
|
+
{"version":3,"sources":["../../bin/sqlite-indexer.ts","../../src/sqlite/getTablesWithRecords.ts","../../src/sqlite/createQueryAdapter.ts"],"sourcesContent":["#!/usr/bin/env node\nimport \"dotenv/config\";\nimport fs from \"node:fs\";\nimport { z } from \"zod\";\nimport { eq } from \"drizzle-orm\";\nimport { drizzle } from \"drizzle-orm/better-sqlite3\";\nimport Database from \"better-sqlite3\";\nimport { createPublicClient, fallback, webSocket, http, Transport } from \"viem\";\nimport Koa from \"koa\";\nimport cors from \"@koa/cors\";\nimport Router from \"@koa/router\";\nimport { createKoaMiddleware } from \"trpc-koa-adapter\";\nimport { createAppRouter } from \"@latticexyz/store-sync/trpc-indexer\";\nimport { chainState, schemaVersion, syncToSqlite } from \"@latticexyz/store-sync/sqlite\";\nimport { createQueryAdapter } from \"../src/sqlite/createQueryAdapter\";\nimport { isDefined } from \"@latticexyz/common/utils\";\nimport { combineLatest, filter, first } from \"rxjs\";\nimport { frontendEnvSchema, indexerEnvSchema, parseEnv } from \"./parseEnv\";\n\nconst env = parseEnv(\n z.intersection(\n z.intersection(indexerEnvSchema, frontendEnvSchema),\n z.object({\n SQLITE_FILENAME: z.string().default(\"indexer.db\"),\n })\n )\n);\n\nconst transports: Transport[] = [\n // prefer WS when specified\n env.RPC_WS_URL ? webSocket(env.RPC_WS_URL) : undefined,\n // otherwise use or fallback to HTTP\n env.RPC_HTTP_URL ? http(env.RPC_HTTP_URL) : undefined,\n].filter(isDefined);\n\nconst publicClient = createPublicClient({\n transport: fallback(transports),\n pollingInterval: env.POLLING_INTERVAL,\n});\n\nconst chainId = await publicClient.getChainId();\nconst database = drizzle(new Database(env.SQLITE_FILENAME));\n\nlet startBlock = env.START_BLOCK;\n\n// Resume from latest block stored in DB. This will throw if the DB doesn't exist yet, so we wrap in a try/catch and ignore the error.\ntry {\n const currentChainStates = database.select().from(chainState).where(eq(chainState.chainId, chainId)).all();\n // TODO: replace this type workaround with `noUncheckedIndexedAccess: true` when we can fix all the issues related (https://github.com/latticexyz/mud/issues/1212)\n const currentChainState: (typeof currentChainStates)[number] | undefined = currentChainStates[0];\n\n if (currentChainState != null) {\n if (currentChainState.schemaVersion != schemaVersion) {\n console.log(\n \"schema version changed from\",\n currentChainState.schemaVersion,\n \"to\",\n schemaVersion,\n \"recreating database\"\n );\n fs.truncateSync(env.SQLITE_FILENAME);\n } else if (currentChainState.lastUpdatedBlockNumber != null) {\n console.log(\"resuming from block number\", currentChainState.lastUpdatedBlockNumber + 1n);\n startBlock = currentChainState.lastUpdatedBlockNumber + 1n;\n }\n }\n} catch (error) {\n // ignore errors, this is optional\n}\n\nconst { latestBlockNumber$, storedBlockLogs$ } = await syncToSqlite({\n database,\n publicClient,\n startBlock,\n maxBlockRange: env.MAX_BLOCK_RANGE,\n address: env.STORE_ADDRESS,\n});\n\nlet isCaughtUp = false;\ncombineLatest([latestBlockNumber$, storedBlockLogs$])\n .pipe(\n filter(\n ([latestBlockNumber, { blockNumber: lastBlockNumberProcessed }]) => latestBlockNumber === lastBlockNumberProcessed\n ),\n first()\n )\n .subscribe(() => {\n isCaughtUp = true;\n console.log(\"all caught up\");\n });\n\nconst server = new Koa();\nserver.use(cors());\n\nconst router = new Router();\n\nrouter.get(\"/\", (ctx) => {\n ctx.body = \"emit HelloWorld();\";\n});\n\n// k8s healthchecks\nrouter.get(\"/healthz\", (ctx) => {\n ctx.status = 200;\n});\nrouter.get(\"/readyz\", (ctx) => {\n if (isCaughtUp) {\n ctx.status = 200;\n ctx.body = \"ready\";\n } else {\n ctx.status = 424;\n ctx.body = \"backfilling\";\n }\n});\n\nserver.use(router.routes());\nserver.use(router.allowedMethods());\n\nserver.use(\n createKoaMiddleware({\n prefix: \"/trpc\",\n router: createAppRouter(),\n createContext: async () => ({\n queryAdapter: await createQueryAdapter(database),\n }),\n })\n);\n\nserver.listen({ host: env.HOST, port: env.PORT });\nconsole.log(`sqlite indexer frontend listening on http://${env.HOST}:${env.PORT}`);\n","import { asc, eq } from \"drizzle-orm\";\nimport { BaseSQLiteDatabase } from \"drizzle-orm/sqlite-core\";\nimport { buildTable, chainState, getTables } from \"@latticexyz/store-sync/sqlite\";\nimport { Hex, getAddress } from \"viem\";\nimport { decodeDynamicField } from \"@latticexyz/protocol-parser\";\nimport { SyncFilter, TableWithRecords } from \"@latticexyz/store-sync\";\n\n// TODO: refactor sqlite and replace this with getLogs to match postgres (https://github.com/latticexyz/mud/issues/1970)\n\n/**\n * @deprecated\n * */\nexport function getTablesWithRecords(\n database: BaseSQLiteDatabase<\"sync\", any>,\n {\n chainId,\n address,\n filters = [],\n }: {\n readonly chainId: number;\n readonly address?: Hex;\n readonly filters?: readonly SyncFilter[];\n }\n): { blockNumber: bigint | null; tables: readonly TableWithRecords[] } {\n const metadata = database\n .select()\n .from(chainState)\n .where(eq(chainState.chainId, chainId))\n .limit(1)\n .all()\n .find(() => true);\n\n // If _any_ filter has a table ID, this will filter down all data to just those tables. Which mean we can't yet mix table filters with key-only filters.\n // TODO: improve this so we can express this in the query (need to be able to query data across tables more easily)\n const tableIds = Array.from(new Set(filters.map((filter) => filter.tableId)));\n const tables = getTables(database)\n .filter((table) => address == null || getAddress(address) === getAddress(table.address))\n .filter((table) => !tableIds.length || tableIds.includes(table.tableId));\n\n const tablesWithRecords = tables.map((table) => {\n const sqliteTable = buildTable(table);\n const records = database\n .select()\n .from(sqliteTable)\n .where(eq(sqliteTable.__isDeleted, false))\n .orderBy(\n asc(sqliteTable.__lastUpdatedBlockNumber)\n // TODO: add logIndex (https://github.com/latticexyz/mud/issues/1979)\n )\n .all();\n const filteredRecords = !filters.length\n ? records\n : records.filter((record) => {\n const keyTuple = decodeDynamicField(\"bytes32[]\", record.__key);\n return filters.some(\n (filter) =>\n filter.tableId === table.tableId &&\n (filter.key0 == null || filter.key0 === keyTuple[0]) &&\n (filter.key1 == null || filter.key1 === keyTuple[1])\n );\n });\n return {\n ...table,\n records: filteredRecords.map((record) => ({\n key: Object.fromEntries(Object.entries(table.keySchema).map(([name]) => [name, record[name]])),\n value: Object.fromEntries(Object.entries(table.valueSchema).map(([name]) => [name, record[name]])),\n })),\n };\n });\n\n return {\n blockNumber: metadata?.lastUpdatedBlockNumber ?? null,\n tables: tablesWithRecords,\n };\n}\n","import { BaseSQLiteDatabase } from \"drizzle-orm/sqlite-core\";\nimport { QueryAdapter } from \"@latticexyz/store-sync/trpc-indexer\";\nimport { getTablesWithRecords } from \"./getTablesWithRecords\";\nimport { tablesWithRecordsToLogs } from \"@latticexyz/store-sync\";\n\n/**\n * Creates a storage adapter for the tRPC server/client to query data from SQLite.\n *\n * @param {BaseSQLiteDatabase<\"sync\", any>} database SQLite database object from Drizzle\n * @returns {Promise<QueryAdapter>} A set of methods used by tRPC endpoints.\n */\nexport async function createQueryAdapter(database: BaseSQLiteDatabase<\"sync\", any>): Promise<QueryAdapter> {\n const adapter: QueryAdapter = {\n async getLogs(opts) {\n const { blockNumber, tables } = getTablesWithRecords(database, opts);\n const logs = tablesWithRecordsToLogs(tables);\n return { blockNumber: blockNumber ?? 0n, logs };\n },\n async findAll(opts) {\n return getTablesWithRecords(database, opts);\n },\n };\n return adapter;\n}\n"],"mappings":";uDACA,MAAO,gBACP,OAAOA,MAAQ,UACf,OAAS,KAAAC,MAAS,MAClB,OAAS,MAAAC,MAAU,cACnB,OAAS,WAAAC,MAAe,6BACxB,OAAOC,MAAc,iBACrB,OAAS,sBAAAC,EAAoB,YAAAC,EAAU,aAAAC,EAAW,QAAAC,MAAuB,OACzE,OAAOC,MAAS,MAChB,OAAOC,MAAU,YACjB,OAAOC,MAAY,cACnB,OAAS,uBAAAC,MAA2B,mBACpC,OAAS,mBAAAC,MAAuB,sCAChC,OAAS,cAAAC,EAAY,iBAAAC,EAAe,gBAAAC,MAAoB,gCCbxD,OAAS,OAAAC,EAAK,MAAAC,MAAU,cAExB,OAAS,cAAAC,EAAY,cAAAC,EAAY,aAAAC,MAAiB,gCAClD,OAAc,cAAAC,MAAkB,OAChC,OAAS,sBAAAC,MAA0B,8BAQ5B,SAASC,EACdC,EACA,CACE,QAAAC,EACA,QAAAC,EACA,QAAAC,EAAU,CAAC,CACb,EAKqE,CACrE,IAAMC,EAAWJ,EACd,OAAO,EACP,KAAKL,CAAU,EACf,MAAMF,EAAGE,EAAW,QAASM,CAAO,CAAC,EACrC,MAAM,CAAC,EACP,IAAI,EACJ,KAAK,IAAM,EAAI,EAIZI,EAAW,MAAM,KAAK,IAAI,IAAIF,EAAQ,IAAKG,GAAWA,EAAO,OAAO,CAAC,CAAC,EAKtEC,EAJSX,EAAUI,CAAQ,EAC9B,OAAQQ,GAAUN,GAAW,MAAQL,EAAWK,CAAO,IAAML,EAAWW,EAAM,OAAO,CAAC,EACtF,OAAQA,GAAU,CAACH,EAAS,QAAUA,EAAS,SAASG,EAAM,OAAO,CAAC,EAExC,IAAKA,GAAU,CAC9C,IAAMC,EAAcf,EAAWc,CAAK,EAC9BE,EAAUV,EACb,OAAO,EACP,KAAKS,CAAW,EAChB,MAAMhB,EAAGgB,EAAY,YAAa,EAAK,CAAC,EACxC,QACCjB,EAAIiB,EAAY,wBAAwB,CAE1C,EACC,IAAI,EACDE,EAAmBR,EAAQ,OAE7BO,EAAQ,OAAQE,GAAW,CACzB,IAAMC,EAAWf,EAAmB,YAAac,EAAO,KAAK,EAC7D,OAAOT,EAAQ,KACZG,GACCA,EAAO,UAAYE,EAAM,UACxBF,EAAO,MAAQ,MAAQA,EAAO,OAASO,EAAS,CAAC,KACjDP,EAAO,MAAQ,MAAQA,EAAO,OAASO,EAAS,CAAC,EACtD,CACF,CAAC,EATDH,EAUJ,MAAO,CACL,GAAGF,EACH,QAASG,EAAgB,IAAKC,IAAY,CACxC,IAAK,OAAO,YAAY,OAAO,QAAQJ,EAAM,SAAS,EAAE,IAAI,CAAC,CAACM,CAAI,IAAM,CAACA,EAAMF,EAAOE,CAAI,CAAC,CAAC,CAAC,EAC7F,MAAO,OAAO,YAAY,OAAO,QAAQN,EAAM,WAAW,EAAE,IAAI,CAAC,CAACM,CAAI,IAAM,CAACA,EAAMF,EAAOE,CAAI,CAAC,CAAC,CAAC,CACnG,EAAE,CACJ,CACF,CAAC,EAED,MAAO,CACL,YAAaV,GAAU,wBAA0B,KACjD,OAAQG,CACV,CACF,CCvEA,OAAS,2BAAAQ,MAA+B,yBAQxC,eAAsBC,EAAmBC,EAAkE,CAWzG,MAV8B,CAC5B,MAAM,QAAQC,EAAM,CAClB,GAAM,CAAE,YAAAC,EAAa,OAAAC,CAAO,EAAIC,EAAqBJ,EAAUC,CAAI,EAC7DI,EAAOP,EAAwBK,CAAM,EAC3C,MAAO,CAAE,YAAaD,GAAe,GAAI,KAAAG,CAAK,CAChD,EACA,MAAM,QAAQJ,EAAM,CAClB,OAAOG,EAAqBJ,EAAUC,CAAI,CAC5C,CACF,CAEF,CFRA,OAAS,aAAAK,MAAiB,2BAC1B,OAAS,iBAAAC,EAAe,UAAAC,EAAQ,SAAAC,OAAa,OAG7C,IAAMC,EAAMC,EACVC,EAAE,aACAA,EAAE,aAAaC,EAAkBC,CAAiB,EAClDF,EAAE,OAAO,CACP,gBAAiBA,EAAE,OAAO,EAAE,QAAQ,YAAY,CAClD,CAAC,CACH,CACF,EAEMG,GAA0B,CAE9BL,EAAI,WAAaM,EAAUN,EAAI,UAAU,EAAI,OAE7CA,EAAI,aAAeO,EAAKP,EAAI,YAAY,EAAI,MAC9C,EAAE,OAAOQ,CAAS,EAEZC,EAAeC,EAAmB,CACtC,UAAWC,EAASN,EAAU,EAC9B,gBAAiBL,EAAI,gBACvB,CAAC,EAEKY,GAAU,MAAMH,EAAa,WAAW,EACxCI,EAAWC,EAAQ,IAAIC,EAASf,EAAI,eAAe,CAAC,EAEtDgB,EAAahB,EAAI,YAGrB,GAAI,CAGF,IAAMiB,EAFqBJ,EAAS,OAAO,EAAE,KAAKK,CAAU,EAAE,MAAMC,EAAGD,EAAW,QAASN,EAAO,CAAC,EAAE,IAAI,EAEX,CAAC,EAE3FK,GAAqB,OACnBA,EAAkB,eAAiBG,GACrC,QAAQ,IACN,8BACAH,EAAkB,cAClB,KACAG,EACA,qBACF,EACAC,EAAG,aAAarB,EAAI,eAAe,GAC1BiB,EAAkB,wBAA0B,OACrD,QAAQ,IAAI,6BAA8BA,EAAkB,uBAAyB,EAAE,EACvFD,EAAaC,EAAkB,uBAAyB,IAG9D,MAAE,CAEF,CAEA,GAAM,CAAE,mBAAAK,GAAoB,iBAAAC,EAAiB,EAAI,MAAMC,EAAa,CAClE,SAAAX,EACA,aAAAJ,EACA,WAAAO,EACA,cAAehB,EAAI,gBACnB,QAASA,EAAI,aACf,CAAC,EAEGyB,EAAa,GACjBC,EAAc,CAACJ,GAAoBC,EAAgB,CAAC,EACjD,KACCI,EACE,CAAC,CAACC,EAAmB,CAAE,YAAaC,CAAyB,CAAC,IAAMD,IAAsBC,CAC5F,EACAC,GAAM,CACR,EACC,UAAU,IAAM,CACfL,EAAa,GACb,QAAQ,IAAI,eAAe,CAC7B,CAAC,EAEH,IAAMM,EAAS,IAAIC,EACnBD,EAAO,IAAIE,EAAK,CAAC,EAEjB,IAAMC,EAAS,IAAIC,EAEnBD,EAAO,IAAI,IAAME,GAAQ,CACvBA,EAAI,KAAO,oBACb,CAAC,EAGDF,EAAO,IAAI,WAAaE,GAAQ,CAC9BA,EAAI,OAAS,GACf,CAAC,EACDF,EAAO,IAAI,UAAYE,GAAQ,CACzBX,GACFW,EAAI,OAAS,IACbA,EAAI,KAAO,UAEXA,EAAI,OAAS,IACbA,EAAI,KAAO,cAEf,CAAC,EAEDL,EAAO,IAAIG,EAAO,OAAO,CAAC,EAC1BH,EAAO,IAAIG,EAAO,eAAe,CAAC,EAElCH,EAAO,IACLM,EAAoB,CAClB,OAAQ,QACR,OAAQC,EAAgB,EACxB,cAAe,UAAa,CAC1B,aAAc,MAAMC,EAAmB1B,CAAQ,CACjD,EACF,CAAC,CACH,EAEAkB,EAAO,OAAO,CAAE,KAAM/B,EAAI,KAAM,KAAMA,EAAI,IAAK,CAAC,EAChD,QAAQ,IAAI,+CAA+CA,EAAI,QAAQA,EAAI,MAAM","names":["fs","z","eq","drizzle","Database","createPublicClient","fallback","webSocket","http","Koa","cors","Router","createKoaMiddleware","createAppRouter","chainState","schemaVersion","syncToSqlite","asc","eq","buildTable","chainState","getTables","getAddress","decodeDynamicField","getTablesWithRecords","database","chainId","address","filters","metadata","tableIds","filter","tablesWithRecords","table","sqliteTable","records","filteredRecords","record","keyTuple","name","tablesWithRecordsToLogs","createQueryAdapter","database","opts","blockNumber","tables","getTablesWithRecords","logs","isDefined","combineLatest","filter","first","env","parseEnv","z","indexerEnvSchema","frontendEnvSchema","transports","webSocket","http","isDefined","publicClient","createPublicClient","fallback","chainId","database","drizzle","Database","startBlock","currentChainState","chainState","eq","schemaVersion","fs","latestBlockNumber$","storedBlockLogs$","syncToSqlite","isCaughtUp","combineLatest","filter","latestBlockNumber","lastBlockNumberProcessed","first","server","Koa","cors","router","Router","ctx","createKoaMiddleware","createAppRouter","createQueryAdapter"]}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@latticexyz/store-indexer",
|
|
3
|
-
"version": "2.0.0-main-
|
|
3
|
+
"version": "2.0.0-main-c314badd",
|
|
4
4
|
"description": "Minimal Typescript indexer for Store",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -19,30 +19,34 @@
|
|
|
19
19
|
"sqlite-indexer": "./dist/bin/sqlite-indexer.js"
|
|
20
20
|
},
|
|
21
21
|
"dependencies": {
|
|
22
|
-
"@
|
|
23
|
-
"@
|
|
22
|
+
"@koa/cors": "^4.0.0",
|
|
23
|
+
"@koa/router": "^12.0.1",
|
|
24
24
|
"@trpc/client": "10.34.0",
|
|
25
25
|
"@trpc/server": "10.34.0",
|
|
26
26
|
"better-sqlite3": "^8.6.0",
|
|
27
27
|
"debug": "^4.3.4",
|
|
28
28
|
"dotenv": "^16.0.3",
|
|
29
29
|
"drizzle-orm": "^0.28.5",
|
|
30
|
-
"
|
|
30
|
+
"koa": "^2.14.2",
|
|
31
31
|
"postgres": "^3.3.5",
|
|
32
32
|
"rxjs": "7.5.5",
|
|
33
33
|
"superjson": "^1.12.4",
|
|
34
|
+
"trpc-koa-adapter": "^1.1.3",
|
|
34
35
|
"viem": "1.14.0",
|
|
35
36
|
"zod": "^3.21.4",
|
|
36
|
-
"@latticexyz/block-logs-stream": "2.0.0-main-
|
|
37
|
-
"@latticexyz/common": "2.0.0-main-
|
|
38
|
-
"@latticexyz/protocol-parser": "2.0.0-main-
|
|
39
|
-
"@latticexyz/store": "2.0.0-main-
|
|
40
|
-
"@latticexyz/store-sync": "2.0.0-main-
|
|
37
|
+
"@latticexyz/block-logs-stream": "2.0.0-main-c314badd",
|
|
38
|
+
"@latticexyz/common": "2.0.0-main-c314badd",
|
|
39
|
+
"@latticexyz/protocol-parser": "2.0.0-main-c314badd",
|
|
40
|
+
"@latticexyz/store": "2.0.0-main-c314badd",
|
|
41
|
+
"@latticexyz/store-sync": "2.0.0-main-c314badd"
|
|
41
42
|
},
|
|
42
43
|
"devDependencies": {
|
|
43
44
|
"@types/better-sqlite3": "^7.6.4",
|
|
44
45
|
"@types/cors": "^2.8.13",
|
|
45
46
|
"@types/debug": "^4.1.7",
|
|
47
|
+
"@types/koa": "^2.13.12",
|
|
48
|
+
"@types/koa__cors": "^4.0.3",
|
|
49
|
+
"@types/koa__router": "^12.0.4",
|
|
46
50
|
"concurrently": "^8.2.2",
|
|
47
51
|
"tsup": "^6.7.0",
|
|
48
52
|
"tsx": "^3.12.6",
|