@latticexyz/store-indexer 2.0.0-main-7eabd06f → 2.0.0-main-f38ad176
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -1,3 +1,3 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import{a as
|
|
2
|
+
import{a as g}from"../chunk-LOFAZSVJ.js";import{a as A,c as h}from"../chunk-2E5MDUA2.js";import"dotenv/config";import{z as p}from"zod";import _ from"fastify";import{fastifyTRPCPlugin as j}from"@trpc/server/adapters/fastify";import{createAppRouter as z}from"@latticexyz/store-sync/trpc-indexer";import{eq as T,getTableName as x}from"drizzle-orm";import{buildTable as q,buildInternalTables as D,getTables as E}from"@latticexyz/store-sync/postgres";import{getAddress as k}from"viem";import{decodeDynamicField as I}from"@latticexyz/protocol-parser";async function w(o){return{async findAll({chainId:d,address:c,filters:m=[]}){let u=Array.from(new Set(m.map(e=>e.tableId))),R=(await E(o)).filter(e=>c==null||k(c)===k(e.address)).filter(e=>!u.length||u.includes(e.tableId)),O=await Promise.all(R.map(async e=>{let l=q(e),b=await o.select().from(l).where(T(l.__isDeleted,!1)).execute().catch(r=>(console.error("Could not query for records, returning empty set for table",x(l),r),[])),v=m.length?b.filter(r=>{let t=I("bytes32[]",r.__key);return m.some(a=>a.tableId===e.tableId&&(a.key0==null||a.key0===t[0])&&(a.key1==null||a.key1===t[1]))}):b;return{...e,records:v.map(r=>({key:Object.fromEntries(Object.entries(e.keySchema).map(([t])=>[t,r[t]])),value:Object.fromEntries(Object.entries(e.valueSchema).map(([t])=>[t,r[t]]))}))}})),f=D(),P=await o.select().from(f.chain).where(T(f.chain.chainId,d)).execute(),{lastUpdatedBlockNumber:S}=P[0]??{},y={blockNumber:S??null,tables:O};return g("findAll",d,c,y),y}}}import{drizzle as Q}from"drizzle-orm/postgres-js";import B from"postgres";var n=h(p.intersection(A,p.object({DATABASE_URL:p.string()}))),C=Q(B(n.DATABASE_URL)),s=_({maxParamLength:5e3});await s.register(import("@fastify/cors"));s.get("/healthz",(o,i)=>i.code(200).send());s.get("/readyz",(o,i)=>i.code(200).send());s.register(j,{prefix:"/trpc",trpcOptions:{router:z(),createContext:async()=>({queryAdapter:await w(C)})}});await s.listen({host:n.HOST,port:n.PORT});console.log(`postgres indexer frontend listening on http://${n.HOST}:${n.PORT}`);
|
|
3
3
|
//# sourceMappingURL=postgres-frontend.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../bin/postgres-frontend.ts","../../src/postgres/createQueryAdapter.ts"],"sourcesContent":["#!/usr/bin/env node\nimport \"dotenv/config\";\nimport { z } from \"zod\";\nimport fastify from \"fastify\";\nimport { fastifyTRPCPlugin } from \"@trpc/server/adapters/fastify\";\nimport { AppRouter, createAppRouter } from \"@latticexyz/store-sync/trpc-indexer\";\nimport { createQueryAdapter } from \"../src/postgres/createQueryAdapter\";\nimport { drizzle } from \"drizzle-orm/postgres-js\";\nimport postgres from \"postgres\";\nimport { frontendEnvSchema, parseEnv } from \"./parseEnv\";\n\nconst env = parseEnv(\n z.intersection(\n frontendEnvSchema,\n z.object({\n DATABASE_URL: z.string(),\n })\n )\n);\n\nconst database = drizzle(postgres(env.DATABASE_URL));\n\n// @see https://fastify.dev/docs/latest/\nconst server = fastify({\n maxParamLength: 5000,\n});\n\nawait server.register(import(\"@fastify/cors\"));\n\n// k8s healthchecks\nserver.get(\"/healthz\", (req, res) => res.code(200).send());\nserver.get(\"/readyz\", (req, res) => res.code(200).send());\n\n// @see https://trpc.io/docs/server/adapters/fastify\nserver.register(fastifyTRPCPlugin<AppRouter>, {\n prefix: \"/trpc\",\n trpcOptions: {\n router: createAppRouter(),\n createContext: async () => ({\n queryAdapter: await createQueryAdapter(database),\n }),\n },\n});\n\nawait server.listen({ host: env.HOST, port: env.PORT });\nconsole.log(`postgres indexer frontend listening on http://${env.HOST}:${env.PORT}`);\n","import { eq } from \"drizzle-orm\";\nimport { PgDatabase } from \"drizzle-orm/pg-core\";\nimport { buildTable, buildInternalTables, getTables } from \"@latticexyz/store-sync/postgres\";\nimport { QueryAdapter } from \"@latticexyz/store-sync/trpc-indexer\";\nimport { debug } from \"../debug\";\nimport { getAddress } from \"viem\";\nimport { decodeDynamicField } from \"@latticexyz/protocol-parser\";\n\n/**\n * Creates a query adapter for the tRPC server/client to query data from Postgres.\n *\n * @param {PgDatabase<any>} database Postgres database object from Drizzle\n * @returns {Promise<QueryAdapter>} A set of methods used by tRPC endpoints.\n */\nexport async function createQueryAdapter(database: PgDatabase<any>): Promise<QueryAdapter> {\n const adapter: QueryAdapter = {\n async findAll({ chainId, address, filters = [] }) {\n // If _any_ filter has a table ID, this will filter down all data to just those tables. Which mean we can't yet mix table filters with key-only filters.\n // TODO: improve this so we can express this in the query (need to be able to query data across tables more easily)\n const tableIds = Array.from(new Set(filters.map((filter) => filter.tableId)));\n const tables = (await getTables(database))\n .filter((table) => address == null || getAddress(address) === getAddress(table.address))\n .filter((table) => !tableIds.length || tableIds.includes(table.tableId));\n\n const tablesWithRecords = await Promise.all(\n tables.map(async (table) => {\n const
|
|
1
|
+
{"version":3,"sources":["../../bin/postgres-frontend.ts","../../src/postgres/createQueryAdapter.ts"],"sourcesContent":["#!/usr/bin/env node\nimport \"dotenv/config\";\nimport { z } from \"zod\";\nimport fastify from \"fastify\";\nimport { fastifyTRPCPlugin } from \"@trpc/server/adapters/fastify\";\nimport { AppRouter, createAppRouter } from \"@latticexyz/store-sync/trpc-indexer\";\nimport { createQueryAdapter } from \"../src/postgres/createQueryAdapter\";\nimport { drizzle } from \"drizzle-orm/postgres-js\";\nimport postgres from \"postgres\";\nimport { frontendEnvSchema, parseEnv } from \"./parseEnv\";\n\nconst env = parseEnv(\n z.intersection(\n frontendEnvSchema,\n z.object({\n DATABASE_URL: z.string(),\n })\n )\n);\n\nconst database = drizzle(postgres(env.DATABASE_URL));\n\n// @see https://fastify.dev/docs/latest/\nconst server = fastify({\n maxParamLength: 5000,\n});\n\nawait server.register(import(\"@fastify/cors\"));\n\n// k8s healthchecks\nserver.get(\"/healthz\", (req, res) => res.code(200).send());\nserver.get(\"/readyz\", (req, res) => res.code(200).send());\n\n// @see https://trpc.io/docs/server/adapters/fastify\nserver.register(fastifyTRPCPlugin<AppRouter>, {\n prefix: \"/trpc\",\n trpcOptions: {\n router: createAppRouter(),\n createContext: async () => ({\n queryAdapter: await createQueryAdapter(database),\n }),\n },\n});\n\nawait server.listen({ host: env.HOST, port: env.PORT });\nconsole.log(`postgres indexer frontend listening on http://${env.HOST}:${env.PORT}`);\n","import { eq, getTableName } from \"drizzle-orm\";\nimport { PgDatabase } from \"drizzle-orm/pg-core\";\nimport { buildTable, buildInternalTables, getTables } from \"@latticexyz/store-sync/postgres\";\nimport { QueryAdapter } from \"@latticexyz/store-sync/trpc-indexer\";\nimport { debug } from \"../debug\";\nimport { getAddress } from \"viem\";\nimport { decodeDynamicField } from \"@latticexyz/protocol-parser\";\n\n/**\n * Creates a query adapter for the tRPC server/client to query data from Postgres.\n *\n * @param {PgDatabase<any>} database Postgres database object from Drizzle\n * @returns {Promise<QueryAdapter>} A set of methods used by tRPC endpoints.\n */\nexport async function createQueryAdapter(database: PgDatabase<any>): Promise<QueryAdapter> {\n const adapter: QueryAdapter = {\n async findAll({ chainId, address, filters = [] }) {\n // If _any_ filter has a table ID, this will filter down all data to just those tables. Which mean we can't yet mix table filters with key-only filters.\n // TODO: improve this so we can express this in the query (need to be able to query data across tables more easily)\n const tableIds = Array.from(new Set(filters.map((filter) => filter.tableId)));\n const tables = (await getTables(database))\n .filter((table) => address == null || getAddress(address) === getAddress(table.address))\n .filter((table) => !tableIds.length || tableIds.includes(table.tableId));\n\n const tablesWithRecords = await Promise.all(\n tables.map(async (table) => {\n const sqlTable = buildTable(table);\n const records = await database\n .select()\n .from(sqlTable)\n .where(eq(sqlTable.__isDeleted, false))\n .execute()\n // Apparently sometimes we can have tables that exist in the internal table but no relation/schema set up, so queries fail.\n // See https://github.com/latticexyz/mud/issues/1923\n // TODO: make a more robust fix for this\n .catch((error) => {\n console.error(\n \"Could not query for records, returning empty set for table\",\n getTableName(sqlTable),\n error\n );\n return [];\n });\n const filteredRecords = !filters.length\n ? records\n : records.filter((record) => {\n const keyTuple = decodeDynamicField(\"bytes32[]\", record.__key);\n return filters.some(\n (filter) =>\n filter.tableId === table.tableId &&\n (filter.key0 == null || filter.key0 === keyTuple[0]) &&\n (filter.key1 == null || filter.key1 === keyTuple[1])\n );\n });\n return {\n ...table,\n records: filteredRecords.map((record) => ({\n key: Object.fromEntries(Object.entries(table.keySchema).map(([name]) => [name, record[name]])),\n value: Object.fromEntries(Object.entries(table.valueSchema).map(([name]) => [name, record[name]])),\n })),\n };\n })\n );\n\n const internalTables = buildInternalTables();\n const metadata = await database\n .select()\n .from(internalTables.chain)\n .where(eq(internalTables.chain.chainId, chainId))\n .execute();\n const { lastUpdatedBlockNumber } = metadata[0] ?? {};\n\n const result = {\n blockNumber: lastUpdatedBlockNumber ?? null,\n tables: tablesWithRecords,\n };\n\n debug(\"findAll\", chainId, address, result);\n\n return result;\n },\n };\n return adapter;\n}\n"],"mappings":";yFACA,MAAO,gBACP,OAAS,KAAAA,MAAS,MAClB,OAAOC,MAAa,UACpB,OAAS,qBAAAC,MAAyB,gCAClC,OAAoB,mBAAAC,MAAuB,sCCL3C,OAAS,MAAAC,EAAI,gBAAAC,MAAoB,cAEjC,OAAS,cAAAC,EAAY,uBAAAC,EAAqB,aAAAC,MAAiB,kCAG3D,OAAS,cAAAC,MAAkB,OAC3B,OAAS,sBAAAC,MAA0B,8BAQnC,eAAsBC,EAAmBC,EAAkD,CAoEzF,MAnE8B,CAC5B,MAAM,QAAQ,CAAE,QAAAC,EAAS,QAAAC,EAAS,QAAAC,EAAU,CAAC,CAAE,EAAG,CAGhD,IAAMC,EAAW,MAAM,KAAK,IAAI,IAAID,EAAQ,IAAKE,GAAWA,EAAO,OAAO,CAAC,CAAC,EACtEC,GAAU,MAAMC,EAAUP,CAAQ,GACrC,OAAQQ,GAAUN,GAAW,MAAQL,EAAWK,CAAO,IAAML,EAAWW,EAAM,OAAO,CAAC,EACtF,OAAQA,GAAU,CAACJ,EAAS,QAAUA,EAAS,SAASI,EAAM,OAAO,CAAC,EAEnEC,EAAoB,MAAM,QAAQ,IACtCH,EAAO,IAAI,MAAOE,GAAU,CAC1B,IAAME,EAAWC,EAAWH,CAAK,EAC3BI,EAAU,MAAMZ,EACnB,OAAO,EACP,KAAKU,CAAQ,EACb,MAAMG,EAAGH,EAAS,YAAa,EAAK,CAAC,EACrC,QAAQ,EAIR,MAAOI,IACN,QAAQ,MACN,6DACAC,EAAaL,CAAQ,EACrBI,CACF,EACO,CAAC,EACT,EACGE,EAAmBb,EAAQ,OAE7BS,EAAQ,OAAQK,GAAW,CACzB,IAAMC,EAAWpB,EAAmB,YAAamB,EAAO,KAAK,EAC7D,OAAOd,EAAQ,KACZE,GACCA,EAAO,UAAYG,EAAM,UACxBH,EAAO,MAAQ,MAAQA,EAAO,OAASa,EAAS,CAAC,KACjDb,EAAO,MAAQ,MAAQA,EAAO,OAASa,EAAS,CAAC,EACtD,CACF,CAAC,EATDN,EAUJ,MAAO,CACL,GAAGJ,EACH,QAASQ,EAAgB,IAAKC,IAAY,CACxC,IAAK,OAAO,YAAY,OAAO,QAAQT,EAAM,SAAS,EAAE,IAAI,CAAC,CAACW,CAAI,IAAM,CAACA,EAAMF,EAAOE,CAAI,CAAC,CAAC,CAAC,EAC7F,MAAO,OAAO,YAAY,OAAO,QAAQX,EAAM,WAAW,EAAE,IAAI,CAAC,CAACW,CAAI,IAAM,CAACA,EAAMF,EAAOE,CAAI,CAAC,CAAC,CAAC,CACnG,EAAE,CACJ,CACF,CAAC,CACH,EAEMC,EAAiBC,EAAoB,EACrCC,EAAW,MAAMtB,EACpB,OAAO,EACP,KAAKoB,EAAe,KAAK,EACzB,MAAMP,EAAGO,EAAe,MAAM,QAASnB,CAAO,CAAC,EAC/C,QAAQ,EACL,CAAE,uBAAAsB,CAAuB,EAAID,EAAS,CAAC,GAAK,CAAC,EAE7CE,EAAS,CACb,YAAaD,GAA0B,KACvC,OAAQd,CACV,EAEA,OAAAgB,EAAM,UAAWxB,EAASC,EAASsB,CAAM,EAElCA,CACT,CACF,CAEF,CD5EA,OAAS,WAAAE,MAAe,0BACxB,OAAOC,MAAc,WAGrB,IAAMC,EAAMC,EACVC,EAAE,aACAC,EACAD,EAAE,OAAO,CACP,aAAcA,EAAE,OAAO,CACzB,CAAC,CACH,CACF,EAEME,EAAWC,EAAQC,EAASN,EAAI,YAAY,CAAC,EAG7CO,EAASC,EAAQ,CACrB,eAAgB,GAClB,CAAC,EAED,MAAMD,EAAO,SAAS,OAAO,eAAe,CAAC,EAG7CA,EAAO,IAAI,WAAY,CAACE,EAAKC,IAAQA,EAAI,KAAK,GAAG,EAAE,KAAK,CAAC,EACzDH,EAAO,IAAI,UAAW,CAACE,EAAKC,IAAQA,EAAI,KAAK,GAAG,EAAE,KAAK,CAAC,EAGxDH,EAAO,SAASI,EAA8B,CAC5C,OAAQ,QACR,YAAa,CACX,OAAQC,EAAgB,EACxB,cAAe,UAAa,CAC1B,aAAc,MAAMC,EAAmBT,CAAQ,CACjD,EACF,CACF,CAAC,EAED,MAAMG,EAAO,OAAO,CAAE,KAAMP,EAAI,KAAM,KAAMA,EAAI,IAAK,CAAC,EACtD,QAAQ,IAAI,iDAAiDA,EAAI,QAAQA,EAAI,MAAM","names":["z","fastify","fastifyTRPCPlugin","createAppRouter","eq","getTableName","buildTable","buildInternalTables","getTables","getAddress","decodeDynamicField","createQueryAdapter","database","chainId","address","filters","tableIds","filter","tables","getTables","table","tablesWithRecords","sqlTable","buildTable","records","eq","error","getTableName","filteredRecords","record","keyTuple","name","internalTables","buildInternalTables","metadata","lastUpdatedBlockNumber","result","debug","drizzle","postgres","env","parseEnv","z","frontendEnvSchema","database","drizzle","postgres","server","fastify","req","res","fastifyTRPCPlugin","createAppRouter","createQueryAdapter"]}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@latticexyz/store-indexer",
|
|
3
|
-
"version": "2.0.0-main-
|
|
3
|
+
"version": "2.0.0-main-f38ad176",
|
|
4
4
|
"description": "Minimal Typescript indexer for Store",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -33,11 +33,11 @@
|
|
|
33
33
|
"superjson": "^1.12.4",
|
|
34
34
|
"viem": "1.14.0",
|
|
35
35
|
"zod": "^3.21.4",
|
|
36
|
-
"@latticexyz/block-logs-stream": "2.0.0-main-
|
|
37
|
-
"@latticexyz/common": "2.0.0-main-
|
|
38
|
-
"@latticexyz/protocol-parser": "2.0.0-main-
|
|
39
|
-
"@latticexyz/store": "2.0.0-main-
|
|
40
|
-
"@latticexyz/store-sync": "2.0.0-main-
|
|
36
|
+
"@latticexyz/block-logs-stream": "2.0.0-main-f38ad176",
|
|
37
|
+
"@latticexyz/common": "2.0.0-main-f38ad176",
|
|
38
|
+
"@latticexyz/protocol-parser": "2.0.0-main-f38ad176",
|
|
39
|
+
"@latticexyz/store": "2.0.0-main-f38ad176",
|
|
40
|
+
"@latticexyz/store-sync": "2.0.0-main-f38ad176"
|
|
41
41
|
},
|
|
42
42
|
"devDependencies": {
|
|
43
43
|
"@types/better-sqlite3": "^7.6.4",
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { eq } from "drizzle-orm";
|
|
1
|
+
import { eq, getTableName } from "drizzle-orm";
|
|
2
2
|
import { PgDatabase } from "drizzle-orm/pg-core";
|
|
3
3
|
import { buildTable, buildInternalTables, getTables } from "@latticexyz/store-sync/postgres";
|
|
4
4
|
import { QueryAdapter } from "@latticexyz/store-sync/trpc-indexer";
|
|
@@ -24,8 +24,23 @@ export async function createQueryAdapter(database: PgDatabase<any>): Promise<Que
|
|
|
24
24
|
|
|
25
25
|
const tablesWithRecords = await Promise.all(
|
|
26
26
|
tables.map(async (table) => {
|
|
27
|
-
const
|
|
28
|
-
const records = await database
|
|
27
|
+
const sqlTable = buildTable(table);
|
|
28
|
+
const records = await database
|
|
29
|
+
.select()
|
|
30
|
+
.from(sqlTable)
|
|
31
|
+
.where(eq(sqlTable.__isDeleted, false))
|
|
32
|
+
.execute()
|
|
33
|
+
// Apparently sometimes we can have tables that exist in the internal table but no relation/schema set up, so queries fail.
|
|
34
|
+
// See https://github.com/latticexyz/mud/issues/1923
|
|
35
|
+
// TODO: make a more robust fix for this
|
|
36
|
+
.catch((error) => {
|
|
37
|
+
console.error(
|
|
38
|
+
"Could not query for records, returning empty set for table",
|
|
39
|
+
getTableName(sqlTable),
|
|
40
|
+
error
|
|
41
|
+
);
|
|
42
|
+
return [];
|
|
43
|
+
});
|
|
29
44
|
const filteredRecords = !filters.length
|
|
30
45
|
? records
|
|
31
46
|
: records.filter((record) => {
|