@hugomrdias/foxer 0.1.3 → 0.1.11

package/src/db/actions/blocks.ts

@@ -1,6 +1,6 @@
 /** biome-ignore-all lint/style/noNonNullAssertion: its ok */
 
-import { gte } from 'drizzle-orm'
+import { and, gte, inArray } from 'drizzle-orm'
 import {
   getTableConfig,
   type PgAsyncTransaction,
@@ -8,13 +8,15 @@ import {
   type PgQueryResultHKT,
   type PgTable,
 } from 'drizzle-orm/pg-core'
-import type { PublicClient } from 'viem'
-import type { FilteredContracts } from '../../config/config.ts'
+import type { Hash, PublicClient } from 'viem'
+
 import { MAX_QUERY_PARAMS } from '../../contants.ts'
 import { safeGetBlock } from '../../rpc/get-block.ts'
 import type {
-  EncodedBlockWithTransactions,
+  BlocksMap,
+  EncodedBlock,
   EncodedTransaction,
+  TransactionsMap,
 } from '../../types.ts'
 import type { Logger } from '../../utils/logger.ts'
 import { startClock } from '../../utils/timer.ts'
@@ -72,19 +74,20 @@ function getTablesWithBlockNumberColumn(fullSchema: Record<string, unknown>) {
  */
 export async function cacheBlockAndTransactions(args: {
   db: Database<typeof schema, typeof relations>
-  block: EncodedBlockWithTransactions
+  blocks: EncodedBlock[]
+  transactions: EncodedTransaction[]
   logger: Logger
 }): Promise<void> {
-  const { db, block } = args
+  const { db, blocks, transactions } = args
 
   await db.transaction(async (tx) => {
     await insertBlocksInChunks({
       db: tx,
-      blocks: [block],
+      blocks,
     })
     await insertTransactionsInChunks({
       db: tx,
-      transactions: block.transactions,
+      transactions,
     })
   })
 }
@@ -104,8 +107,8 @@ export async function getBlocksInRange(
   db: Database<typeof schema, typeof relations>,
   blockNumbers: bigint[],
   client: PublicClient,
-  contracts: FilteredContracts
-): Promise<Map<bigint, EncodedBlockWithTransactions>> {
+  logsTxs: Hash[]
+): Promise<{ blocks: BlocksMap; transactions: TransactionsMap }> {
   const endClock = startClock()
   const firstBlockNumber = blockNumbers[0]!
   const lastBlockNumber = blockNumbers[blockNumbers.length - 1]!
@@ -116,74 +119,72 @@ export async function getBlocksInRange(
   //   contractAddresses: contracts.addresses,
   // })
 
-  const r = await db.query.blocks.findMany({
-    with: {
-      transactions: {
-        where: {
-          AND: [
-            { blockNumber: { gte: firstBlockNumber } },
-            { blockNumber: { lte: lastBlockNumber } },
-            {
-              to: {
-                in: contracts.addresses,
-              },
-            },
-          ],
-        },
+  const [blocks, txs] = await Promise.all([
+    db.query.blocks.findMany({
+      where: {
+        AND: [
+          { number: { gte: firstBlockNumber } },
+          { number: { lte: lastBlockNumber } },
+        ],
       },
-    },
-    where: {
-      AND: [
-        { number: { gte: firstBlockNumber } },
-        { number: { lte: lastBlockNumber } },
-      ],
-    },
-  })
+    }),
+    db
+      .select()
+      .from(schema.transactions)
+      .where(and(inArray(schema.transactions.hash, logsTxs))),
+  ])
+
+  const transactionByHash = new Map<`0x${string}`, EncodedTransaction>()
+  for (const tx of txs) {
+    transactionByHash.set(tx.hash, tx)
+  }
 
-  const blocksByNumber = new Map<bigint, EncodedBlockWithTransactions>()
+  const blocksByNumber = new Map<bigint, EncodedBlock>()
   const missing = new Set(blockNumbers)
 
-  for (const block of r) {
+  for (const block of blocks) {
     blocksByNumber.set(block.number, block)
     missing.delete(block.number)
   }
 
   const missingBlockNumbers = [...missing]
-  const newBlocks: EncodedBlockWithTransactions[] = []
+  const newBlocks: EncodedBlock[] = []
   const newTransactions: EncodedTransaction[] = []
 
   await Promise.all(
     missingBlockNumbers.map(async (blockNumber) => {
       const block = await safeGetBlock({ client, blockNumber, db })
-      const transactions = block.transactions
       blocksByNumber.set(blockNumber, block)
-      newBlocks.push(block)
+      const { transactions, ..._block } = block
+      newBlocks.push(_block)
+
       if (transactions.length > 0) {
         newTransactions.push(...transactions)
       }
+      for (const tx of transactions) {
+        transactionByHash.set(tx.hash, tx)
+      }
     })
   )
 
-  await db.transaction(async (tx) => {
-    await insertBlocksInChunks({
-      db: tx,
-      blocks: newBlocks,
-    })
-    await insertTransactionsInChunks({
-      db: tx,
-      transactions: newTransactions,
-    })
+  await cacheBlockAndTransactions({
+    db,
+    blocks: newBlocks,
+    transactions: newTransactions,
+    logger,
   })
 
-  logger.info(
+  logger.trace(
     {
-      blocks: blocksByNumber.size,
-      missing: missingBlockNumbers.length,
+      blocks: blocks.length,
+      txs: txs.length,
+      newBlocks: newBlocks.length,
+      newTxs: newTransactions.length,
       duration: endClock(),
     },
-    'get blocks'
+    'get blocks and txs'
   )
-  return blocksByNumber
+  return { blocks: blocksByNumber, transactions: transactionByHash }
 }
 
 /**
@@ -191,7 +192,7 @@ export async function getBlocksInRange(
  */
 export async function insertBlocksInChunks(args: {
   db: PgAsyncTransaction<PgQueryResultHKT, typeof schema>
-  blocks: EncodedBlockWithTransactions[]
+  blocks: EncodedBlock[]
 }): Promise<void> {
   const { db, blocks } = args
   if (blocks.length === 0) return

package/src/db/actions/transactions.ts

@@ -1,4 +1,5 @@
 import type { PgAsyncTransaction, PgQueryResultHKT } from 'drizzle-orm/pg-core'
+
 import { MAX_QUERY_PARAMS } from '../../contants.ts'
 import type { EncodedTransaction } from '../../types.ts'
 import { schema } from '../schema/index.ts'

package/src/db/client.ts

@@ -10,6 +10,7 @@ import {
 } from 'drizzle-orm/pglite'
 import type { AnyRelations, EmptyRelations } from 'drizzle-orm/relations'
 import { Pool, type PoolConfig } from 'pg'
+
 import type { DatabaseConfig } from '../config/config.ts'
 import type { Env } from '../config/env.ts'
 import { type relations, schema } from './schema/index.ts'
@@ -39,7 +40,7 @@ export type DatabaseContext<
         $prepared: ReturnType<typeof generatePrepared>
       }
       driver: 'postgres'
-      close: () => Promise<void>
+      stop: () => Promise<void>
     }
   | {
       db: PgliteDatabase<TSchema, TRelations> & {
@@ -47,7 +48,7 @@ export type DatabaseContext<
         $prepared: ReturnType<typeof generatePrepared>
       }
       driver: 'pglite'
-      close: () => Promise<void>
+      stop: () => Promise<void>
     }
 
 /**
@@ -80,10 +81,15 @@ export function createDatabase<
     options = config.options
   }
 
+  // Postgres
   if (driver === 'postgres' && url) {
     const pool = new Pool({
-      ...options,
+      application_name: 'foxer',
+      connectionTimeoutMillis: 5_000,
+      idleTimeoutMillis: 30_000,
+      max: 10,
       connectionString: url,
+      ...options,
     })
     const db = drizzleNodePostgres({
       client: pool,
@@ -101,12 +107,13 @@ export function createDatabase<
     return {
       db,
       driver: 'postgres',
-      close: async () => {
+      stop: async () => {
         await pool.end()
       },
     }
   }
 
+  // PGlite
   const client = new PGlite(
     config?.driver === 'pglite' && config.directory
       ? config.directory
@@ -128,7 +135,7 @@ export function createDatabase<
   return {
     db,
     driver: 'pglite',
-    close: async () => {
+    stop: async () => {
       await client.close()
     },
   }

package/src/db/column-types.ts

@@ -1,4 +1,5 @@
 import { customType } from 'drizzle-orm/pg-core'
+import { hex as hexCodec } from 'iso-base/rfc4648'
 import { type Address, type Hash, type Hex, stringify } from 'viem'
 
 export const numeric78 = customType<{ data: bigint; driverData: string }>({
@@ -90,16 +91,26 @@ export const jsonb = customType<{ data: unknown; driverData: string }>({
   },
 })
 
-export const bytea = customType<{ data: Hex; driverData: Buffer }>({
+export const bytea = customType<{ data: Hex; driverData: Uint8Array }>({
   dataType() {
     return 'bytea'
   },
-  toDriver(value: string): Buffer {
+  toDriver(value: string): Uint8Array {
     return Buffer.from(value.slice(2), 'hex')
   },
-  fromDriver(value: Buffer): Hex {
-    const hex = value.toString('hex')
-    const _value = hex.startsWith('\\x') ? hex.slice(2) : hex
-    return `0x${_value}` as Hex
+  fromDriver(value: unknown): Hex {
+    if (typeof value === 'string') {
+      return `0x${value.slice(2)}` as Hex
+    }
+
+    if (value instanceof Buffer) {
+      return `0x${value.toString('hex')}` as Hex
+    }
+
+    if (value instanceof Uint8Array) {
+      return `0x${hexCodec.encode(value)}` as Hex
+    }
+
+    throw new Error('Invalid value')
   },
 })

package/src/db/encode.ts

@@ -1,4 +1,5 @@
 import type { Hash } from 'viem'
+
 import type {
   ChainBlock,
   ChainTransaction,

package/src/db/migrate.ts

@@ -2,6 +2,7 @@ import { migrate as migratePostgresJs } from 'drizzle-orm/node-postgres/migrator
 import { getTableConfig, type PgTable } from 'drizzle-orm/pg-core'
 import { IndexedColumn } from 'drizzle-orm/pg-core/columns/common'
 import { migrate as migratePglite } from 'drizzle-orm/pglite/migrator'
+
 import { FOXER_TABLES, PUBLICATION_NAME } from '../contants.ts'
 import type { Logger } from '../utils/logger.ts'
 import { startClock } from '../utils/timer.ts'

package/src/db/schema/blocks.ts

@@ -1,4 +1,5 @@
 import { pgTable } from 'drizzle-orm/pg-core'
+
 import { address, bigint, bytea, hash, numeric78 } from '../column-types.ts'
 
 export const blocks = pgTable('blocks', {

package/src/db/schema/transactions.ts

@@ -1,5 +1,6 @@
 import { index, integer, jsonb, pgEnum, pgTable } from 'drizzle-orm/pg-core'
 import type { AccessList } from 'viem'
+
 import { address, bigint, bytea, hash, numeric78 } from '../column-types.ts'
 
 export const transactionTypeEnum = pgEnum('transaction_type', [
@@ -34,6 +35,8 @@ export const transactions = pgTable(
   },
   (table) => [
     index('transactions_block_number_index').on(table.blockNumber),
-    index('transactions_to_index').on(table.to),
+    index('transactions_to_block_number_index')
+      .on(table.to, table.blockNumber)
+      .concurrently(),
   ]
 )

package/src/hooks/registry.ts

@@ -1,7 +1,9 @@
 import type { AnyRelations, EmptyRelations } from 'drizzle-orm/relations'
 import type { GetEventArgs, Log } from 'viem'
+
+import type { InternalConfig } from '../config/config'
 import type { Database } from '../db/client'
-import type { EncodedBlockWithTransactions, EncodedTransaction } from '../types'
+import type { EncodedBlock, EncodedTransaction } from '../types'
 import type { Logger } from '../utils/logger'
 import type {
   ContractAbiByEventKey,
@@ -32,7 +34,7 @@ export type DecodedEvent<
     { EnableUnion: false; IndexedOnly: false; Required: true }
   >
   log: Log<bigint, number, false, ContractAbiEventByEventKey<C, Event>>
-  block: EncodedBlockWithTransactions
+  block: EncodedBlock
   transaction: EncodedTransaction
 }
 
@@ -79,7 +81,7 @@ export class HookRegistry<
       { EnableUnion: false; IndexedOnly: false; Required: true }
     >
     log: Log<bigint, number, false, ContractAbiEventByEventKey<C, K>>
-    block: EncodedBlockWithTransactions
+    block: EncodedBlock
     transaction: EncodedTransaction
     context: HookContext<TSchema, TRelations>
   }): Promise<void> {
@@ -105,3 +107,13 @@ export class HookRegistry<
     })
   }
 }
+
+export function createRegistry({
+  config,
+}: {
+  config: InternalConfig
+}): HookRegistry {
+  const registry = new HookRegistry()
+  config.hooks({ registry })
+  return registry
+}

package/src/indexer/backfill.ts

@@ -1,3 +1,4 @@
+import type { Hash } from 'viem'
 import { filterContracts, type InternalConfig } from '../config/config.ts'
 import { getBlocksInRange } from '../db/actions/blocks.ts'
 import type { Database } from '../db/client.ts'
@@ -40,7 +41,7 @@ export async function runBackfill(args: {
   }
 
   const batchSize = config.batchSize
-  logger.debug(
+  logger.info(
     {
       fromBlock: cursor.toString(),
       toBlock: safeHead.toString(),
@@ -54,14 +55,6 @@ export async function runBackfill(args: {
     const toBlock = windowEnd(cursor, batchSize, safeHead)
     const windowContracts = filterContracts(config, cursor, toBlock)
 
-    logger.debug(
-      {
-        batchFromBlock: cursor.toString(),
-        batchToBlock: toBlock.toString(),
-        streamCount: windowContracts.addresses.length,
-      },
-      'processing backfill batch'
-    )
     const batchBlockNumbers: bigint[] = []
     let blockNumber = cursor
     while (blockNumber <= toBlock) {
@@ -69,17 +62,30 @@ export async function runBackfill(args: {
       blockNumber += 1n
     }
 
-    const [blocksByNumber, logsByBlock] = await Promise.all([
-      getBlocksInRange(logger, db, batchBlockNumbers, client, windowContracts),
-      getLogsInRange({
+    const logsByBlock = await getLogsInRange({
+      logger,
+      client,
+      addresses: windowContracts.addresses,
+      events: windowContracts.eventAbis,
+      fromBlock: cursor,
+      toBlock,
+    })
+
+    const logsTxsSet = new Set<Hash>()
+    for (const logs of logsByBlock.values()) {
+      for (const log of logs) {
+        logsTxsSet.add(log.transactionHash)
+      }
+    }
+
+    const { blocks: blocksByNumber, transactions: transactionsMap } =
+      await getBlocksInRange(
         logger,
+        db,
+        batchBlockNumbers,
         client,
-        addresses: windowContracts.addresses,
-        events: windowContracts.eventAbis,
-        fromBlock: cursor,
-        toBlock,
-      }),
-    ])
+        Array.from(logsTxsSet)
+      )
 
     let blockIndex = 0
 
@@ -89,22 +95,26 @@ export async function runBackfill(args: {
         const blockNumber = batchBlockNumbers[blockIndex]
         const prefetchedBlock = blocksByNumber.get(blockNumber)
 
+        if (!prefetchedBlock) {
+          throw new Error(`Block ${blockNumber} not found`)
+        }
+
         await processBlock({
           logger,
           config,
           db: tx,
           client,
           registry,
-          blockNumber,
           logs: logsByBlock.get(blockNumber) ?? [],
           block: prefetchedBlock,
+          transactionsMap,
           type: 'backfill',
           contracts: windowContracts,
         })
         blockIndex += 1
       }
     })
-    logger.info(
+    logger.debug(
       { duration: endClockBatch() },
       'batch block and events processed'
     )
@@ -118,6 +128,7 @@ export async function runBackfill(args: {
       {
         indexedUpTo: toBlock.toString(),
         duration: batchElapsedMs,
+        contracts: windowContracts.addresses.length,
         throughput: Number(blocksPerSecond.toFixed(2)),
       },
       'backfill batch completed'

package/src/indexer/{runner.ts → create-indexer.ts}

@@ -7,7 +7,7 @@ import { runBackfill } from './backfill.ts'
 import { startLiveSync } from './live.ts'
 import { verifyRecentBlocks } from './reorg.ts'
 
-export async function bootstrapIndexer(options: {
+export async function createIndexer(options: {
   logger: Logger
   db: Database<typeof schema, typeof relations>
   registry: HookRegistry

package/src/indexer/live.ts

@@ -1,5 +1,6 @@
 import PQueue from 'p-queue'
 import type { PublicClient } from 'viem'
+
 import type { InternalConfig } from '../config/config.ts'
 import type { Database } from '../db/client.ts'
 import type { relations, schema } from '../db/schema/index.ts'
@@ -25,7 +26,7 @@ export function startLiveSync(args: {
   const contracts = config.contractsForLive
 
   if (contracts.length === 0) {
-    logger.debug(
+    logger.info(
       'all configured contracts have endBlock set; live sync disabled'
     )
     return { stop: noop }
@@ -44,7 +45,7 @@ export function startLiveSync(args: {
     onBlockNumber: (head) => {
       while (nextBlockToQueue <= head) {
         const blockNumber = nextBlockToQueue
-        pqueue.add(async () => {
+        void pqueue.add(async () => {
           await queueBlock({
             logger,
             blockNumber,
@@ -67,7 +68,7 @@ export function startLiveSync(args: {
     },
   })
 
-  logger.debug(
+  logger.info(
     { startBlock: nextBlockToQueue.toString() },
     'watching latest chain head'
   )

package/src/indexer/process-block.ts

@@ -1,12 +1,12 @@
 import type { AbiEvent, Log, PublicClient } from 'viem'
+
 import type { FilteredContracts, InternalConfig } from '../config/config.ts'
 import { cacheBlockAndTransactions } from '../db/actions/blocks.ts'
 import type { Database } from '../db/client.ts'
 import type { relations, schema } from '../db/schema/index.ts'
 import { withTransaction } from '../db/transaction.ts'
 import type { HookRegistry } from '../hooks/registry.ts'
-import { safeGetBlock } from '../rpc/get-block.ts'
-import type { EncodedBlockWithTransactions, EncodedTransaction } from '../types'
+import type { EncodedBlock, TransactionsMap } from '../types'
 import type { Logger } from '../utils/logger.ts'
 import { ensureParentContinuity } from './reorg.ts'
 
@@ -23,9 +23,9 @@ export async function processBlock(args: {
   db: Database<typeof schema, typeof relations>
   client: PublicClient
   registry: HookRegistry<NonNullable<unknown>>
-  blockNumber: bigint
-  logs?: Log<bigint, number, false, AbiEvent>[]
-  block?: EncodedBlockWithTransactions
+  logs: Log<bigint, number, false, AbiEvent>[]
+  block: EncodedBlock
+  transactionsMap: TransactionsMap
   type: 'backfill' | 'live'
   contracts: FilteredContracts
 }): Promise<ProcessBlockResult> {
@@ -35,41 +35,12 @@ export async function processBlock(args: {
     db,
     client,
     registry,
-    blockNumber,
-    block: prefetchedBlock,
-    logs: prefetchedLogs,
+    block,
+    transactionsMap,
+    logs,
     type,
     contracts,
   } = args
-  const transactionByHash = new Map<`0x${string}`, EncodedTransaction>()
-
-  let block: EncodedBlockWithTransactions | undefined
-  let logs: Log<bigint, number, false, AbiEvent>[] | undefined
-
-  if (prefetchedBlock) {
-    block = prefetchedBlock
-  }
-  if (prefetchedLogs) {
-    logs = prefetchedLogs
-  }
-  if (!block || !logs) {
-    const [blockResult, logsResult] = await Promise.all([
-      safeGetBlock({ client, blockNumber, db }),
-      client.getLogs({
-        address: contracts.addresses,
-        events: contracts.eventAbis,
-        fromBlock: blockNumber,
-        toBlock: blockNumber,
-      }),
-    ])
-
-    block = blockResult
-    logs = logsResult
-  }
-
-  for (const tx of block.transactions) {
-    transactionByHash.set(tx.hash, tx)
-  }
 
   if (type === 'live') {
     const rewindTo = await ensureParentContinuity({
@@ -87,7 +58,8 @@ export async function processBlock(args: {
     if (type === 'live') {
       await cacheBlockAndTransactions({
         db: tx,
-        block,
+        blocks: [block],
+        transactions: Array.from(transactionsMap.values()),
         logger,
       })
     }
@@ -96,7 +68,7 @@ export async function processBlock(args: {
       const contractName = contracts.contractNameByAddress[log.address]
 
       if (!contractName) {
-        logger.trace(
+        logger.debug(
           { address: log.address },
           'contract not found in contract name by address'
         )
@@ -107,10 +79,10 @@ export async function processBlock(args: {
       if (!contracts.eventNames.has(eventName)) {
         continue
       }
-      const transaction = transactionByHash.get(log.transactionHash)
+      const transaction = transactionsMap.get(log.transactionHash)
 
       if (!transaction) {
-        logger.trace(
+        logger.debug(
           { transactionHash: log.transactionHash },
 
           'transaction not found in block transaction list'

package/src/indexer/queue-block.ts

@@ -1,9 +1,12 @@
 import type { Logger } from 'pino'
 import type { PublicClient } from 'viem'
+
 import { filterContracts, type InternalConfig } from '../config/config.ts'
 import type { Database } from '../db/client.ts'
 import type { relations, schema } from '../db/schema/index.ts'
 import type { HookRegistry } from '../hooks/registry.ts'
+import { safeGetBlock } from '../rpc/get-block.ts'
+import type { EncodedBlock, EncodedTransaction } from '../types.ts'
 import { startClock } from '../utils/timer.ts'
 import { processBlock } from './process-block.ts'
 
@@ -33,15 +36,36 @@ export async function queueBlock(args: QueueBlockArgs): Promise<void> {
   const endClock = startClock()
   try {
     const contracts = filterContracts(config, blockNumber, blockNumber)
+
+    const [blockResult, logsResult] = await Promise.all([
+      safeGetBlock({ client, blockNumber, db }),
+      client.getLogs({
+        address: contracts.addresses,
+        events: contracts.eventAbis,
+        fromBlock: blockNumber,
+        toBlock: blockNumber,
+      }),
+    ])
+
+    const { transactions, ..._block } = blockResult
+    const block: EncodedBlock = _block
+    const transactionsMap = new Map<`0x${string}`, EncodedTransaction>()
+
+    for (const tx of transactions) {
+      transactionsMap.set(tx.hash, tx)
+    }
+
     const result = await processBlock({
       logger,
       config,
       db,
       client,
       registry,
-      blockNumber,
       type: 'live',
       contracts,
+      block,
+      transactionsMap,
+      logs: logsResult,
     })
 
     if (result.status === 'reorg') {