@hugomrdias/foxer 0.0.1

package/src/indexer/process-block.ts

@@ -0,0 +1,142 @@
+import type { AbiEvent, Log, PublicClient } from 'viem'
+import type { FilteredContracts, InternalConfig } from '../config/config.ts'
+import { cacheBlockAndTransactions } from '../db/actions/blocks.ts'
+import type { Database } from '../db/client.ts'
+import type { relations, schema } from '../db/schema/index.ts'
+import { withTransaction } from '../db/transaction.ts'
+import type { HookRegistry } from '../hooks/registry.ts'
+import { safeGetBlock } from '../rpc/get-block.ts'
+import type { EncodedBlockWithTransactions, EncodedTransaction } from '../types'
+import type { Logger } from '../utils/logger.ts'
+import { ensureParentContinuity } from './reorg.ts'
+
+export type ProcessBlockResult =
+  | { status: 'processed' }
+  | { status: 'reorg'; rewindTo: bigint }
+
+/**
+ * Processes one block: continuity check, event writes, and optional cursor update.
+ */
+export async function processBlock(args: {
+  logger: Logger
+  config: InternalConfig
+  db: Database<typeof schema, typeof relations>
+  client: PublicClient
+  registry: HookRegistry<NonNullable<unknown>>
+  blockNumber: bigint
+  logs?: Log<bigint, number, false, AbiEvent>[]
+  block?: EncodedBlockWithTransactions
+  type: 'backfill' | 'live'
+  contracts: FilteredContracts
+}): Promise<ProcessBlockResult> {
+  const {
+    logger,
+    config,
+    db,
+    client,
+    registry,
+    blockNumber,
+    block: prefetchedBlock,
+    logs: prefetchedLogs,
+    type,
+    contracts,
+  } = args
+  const transactionByHash = new Map<`0x${string}`, EncodedTransaction>()
+
+  let block: EncodedBlockWithTransactions | undefined
+  let logs: Log<bigint, number, false, AbiEvent>[] | undefined
+
+  if (prefetchedBlock) {
+    block = prefetchedBlock
+  }
+  if (prefetchedLogs) {
+    logs = prefetchedLogs
+  }
+  if (!block || !logs) {
+    const [blockResult, logsResult] = await Promise.all([
+      safeGetBlock({ client, blockNumber, db }),
+      client.getLogs({
+        address: contracts.addresses,
+        events: contracts.eventAbis,
+        fromBlock: blockNumber,
+        toBlock: blockNumber,
+      }),
+    ])
+
+    block = blockResult
+    logs = logsResult
+  }
+
+  for (const tx of block.transactions) {
+    transactionByHash.set(tx.hash, tx)
+  }
+
+  if (type === 'live') {
+    const rewindTo = await ensureParentContinuity({
+      logger,
+      db,
+      client,
+      block,
+    })
+    if (rewindTo != null) {
+      return { status: 'reorg', rewindTo }
+    }
+  }
+
+  const write = async (tx: Database<typeof schema, typeof relations>) => {
+    if (type === 'live') {
+      await cacheBlockAndTransactions({
+        db: tx,
+        block,
+        logger,
+      })
+    }
+
+    for (const log of logs) {
+      const contractName = contracts.contractNameByAddress[log.address]
+
+      if (!contractName) {
+        logger.trace(
+          { address: log.address },
+          'contract not found in contract name by address'
+        )
+        continue
+      }
+      const eventName = log.eventName
+
+      if (!contracts.eventNames.has(eventName)) {
+        continue
+      }
+      const transaction = transactionByHash.get(log.transactionHash)
+
+      if (!transaction) {
+        logger.trace(
+          { transactionHash: log.transactionHash },
+
+          'transaction not found in block transaction list'
+        )
+        continue
+      }
+      await registry.dispatch({
+        key: `${contractName}:${eventName}` as never,
+        args: log.args as never,
+        log: log as never,
+        block,
+        transaction,
+        context: {
+          db: tx,
+          chainId: config.client.chain.id,
+          logger,
+        },
+      })
+    }
+  }
+
+  if (type === 'backfill') {
+    await write(db)
+  } else {
+    await withTransaction(db, write)
+  }
+
+  return { status: 'processed' }
+}

package/src/indexer/queue-block.ts

@@ -0,0 +1,74 @@
+import type { Logger } from 'pino'
+import type { PublicClient } from 'viem'
+import { filterContracts, type InternalConfig } from '../config/config.ts'
+import type { Database } from '../db/client.ts'
+import type { relations, schema } from '../db/schema/index.ts'
+import type { HookRegistry } from '../hooks/registry.ts'
+import { startClock } from '../utils/timer.ts'
+import { processBlock } from './process-block.ts'
+
+export type QueueBlockArgs = {
+  logger: Logger
+  blockNumber: bigint
+  onRewind: (rewindTo: bigint) => void
+  config: InternalConfig
+  db: Database<typeof schema, typeof relations>
+  client: PublicClient
+  registry: HookRegistry
+  queueSize: number
+}
+
+export async function queueBlock(args: QueueBlockArgs): Promise<void> {
+  const {
+    config,
+    db,
+    client,
+    registry,
+    blockNumber,
+    logger,
+    onRewind,
+    queueSize,
+  } = args
+
+  const endClock = startClock()
+  try {
+    const contracts = filterContracts(config, blockNumber, blockNumber)
+    const result = await processBlock({
+      logger,
+      config,
+      db,
+      client,
+      registry,
+      blockNumber,
+      type: 'live',
+      contracts,
+    })
+
+    if (result.status === 'reorg') {
+      logger.warn(
+        {
+          blockNumber: blockNumber.toString(),
+          rewindTo: result.rewindTo.toString(),
+        },
+        'reorg detected during live processing; rewinding'
+      )
+      onRewind(result.rewindTo)
+      return
+    }
+
+    logger.info(
+      {
+        duration: endClock(),
+        blockNumber: blockNumber.toString(),
+        queueSize,
+      },
+      'processed live block'
+    )
+  } catch (error) {
+    logger.error(
+      { error, blockNumber: blockNumber.toString() },
+      'block processing failed; rewinding'
+    )
+    onRewind(blockNumber - 1n)
+  }
+}

package/src/indexer/reorg.ts

@@ -0,0 +1,120 @@
+import type { PublicClient } from 'viem'
+import { deleteBlocksFrom } from '../db/actions/blocks.ts'
+import type { Database } from '../db/client.ts'
+import { safeGetBlock } from '../rpc/get-block.ts'
+import type { EncodedBlockWithTransactions } from '../types'
+import { hashEquals } from '../utils/hash.ts'
+import type { Logger } from '../utils/logger.ts'
+import { startClock } from '../utils/timer.ts'
+
+/**
+ * Verifies parent-hash continuity and rolls back divergent canonical rows if needed.
+ * Returns the rewind start block when a reorg is detected.
+ */
+export async function ensureParentContinuity(args: {
+  logger: Logger
+  db: Database
+  client: PublicClient
+  block: EncodedBlockWithTransactions
+}): Promise<bigint | null> {
+  const { logger, db, client, block } = args
+  if (block.number === 0n) return null
+
+  // get the previous block
+  const previous = (
+    await db.$prepared.getBlockById.execute({
+      db,
+      blockNumber: block.number - 1n,
+    })
+  )[0]
+  if (!previous) return null
+
+  // check if the previous block's hash is the same as the block's parent hash
+  if (hashEquals(previous.hash, block.parentHash)) {
+    return null
+  }
+
+  logger.warn(
+    { blockNumber: block.number.toString() },
+    'parent mismatch detected; rolling back'
+  )
+
+  // Walk backward from the immediate parent of the failing block until we find
+  // a block number where DB and chain hashes agree again.
+  let cursor = block.number - 1n
+
+  while (true) {
+    // 1) Read the DB's canonical block at this height.
+    const dbBlock = (
+      await db.$prepared.getBlockById.execute({ blockNumber: cursor })
+    )[0]
+    if (!dbBlock) {
+      cursor -= 1n
+      continue
+    }
+
+    // 2) Read chain block at the same height.
+    const chainBlock = await safeGetBlock({ client, blockNumber: cursor, db })
+    // if (blockResult.status === 'null_round') {
+    //   cursor -= 1n
+    //   continue
+    // }
+
+    // 3) Found the last common ancestor. Rewind to the first divergent height.
+    if (hashEquals(chainBlock.hash, dbBlock.hash)) {
+      const rewindTo = cursor
+      await deleteBlocksFrom(db, rewindTo)
+      return rewindTo
+    }
+
+    // 4) Still divergent at this height; keep scanning backward.
+    // TODO This should finality depth and it should delete and throw critical error
+    if (cursor === 0n) {
+      await deleteBlocksFrom(db, 0n)
+      return 0n
+    }
+    cursor -= 1n
+  }
+}
+
+/**
+ * Validates recent indexed blocks against chain state on startup.
+ */
+export async function verifyRecentBlocks(args: {
+  logger: Logger
+  db: Database
+  client: PublicClient
+  depth: bigint
+}): Promise<void> {
+  const { logger, db, client, depth } = args
+  const endClock = startClock()
+  const latest =
+    (await db.$prepared.getLatestBlock.execute())[0]?.number ?? null
+
+  if (latest == null) return
+
+  const start = latest - depth >= 0n ? latest - depth : 0n
+  let blockNumber = start
+  while (blockNumber <= latest) {
+    const dbBlock = (
+      await db.$prepared.getBlockById.execute({ blockNumber })
+    )[0]
+
+    if (!dbBlock) {
+      blockNumber += 1n
+      continue
+    }
+    const chainBlock = await safeGetBlock({ client, blockNumber, db })
+
+    if (!hashEquals(chainBlock.hash, dbBlock.hash)) {
+      logger.warn(
+        { blockNumber: blockNumber.toString() },
+        'startup sanity check mismatch detected'
+      )
+      await deleteBlocksFrom(db, blockNumber)
+      return
+    }
+    blockNumber += 1n
+  }
+  logger.info({ duration: endClock() }, 'startup sanity check completed')
+}

package/src/indexer/runner.ts

@@ -0,0 +1,35 @@
+import type { InternalConfig } from '../config/config.ts'
+import type { Database } from '../db/client.ts'
+import type { relations, schema } from '../db/schema/index.ts'
+import type { HookRegistry } from '../hooks/registry.ts'
+import type { Logger } from '../utils/logger.ts'
+import { runBackfill } from './backfill.ts'
+import { startLiveSync } from './live.ts'
+import { verifyRecentBlocks } from './reorg.ts'
+
+export async function bootstrapIndexer(options: {
+  logger: Logger
+  db: Database<typeof schema, typeof relations>
+  registry: HookRegistry
+  config: InternalConfig
+}): Promise<{ stop: () => void }> {
+  await verifyRecentBlocks({
+    logger: options.logger,
+    db: options.db,
+    client: options.config.clients.backfill,
+    depth: options.config.finality,
+  })
+
+  const nextCursor = await runBackfill(options)
+
+  const live = startLiveSync({
+    logger: options.logger,
+    config: options.config,
+    db: options.db,
+    client: options.config.clients.live,
+    registry: options.registry,
+    initialCursor: nextCursor,
+  })
+
+  return live
+}

package/src/rpc/client.ts

@@ -0,0 +1,27 @@
+import { createPublicClient, type PublicClient } from 'viem'
+import type { ClientConfig } from '../config/config.ts'
+
+export type RpcClients = {
+  backfill: PublicClient
+  live: PublicClient
+}
+
+/**
+ * Creates a viem public client configured for the target FEVM chain.
+ */
+export function createRpcClients(options: ClientConfig): RpcClients {
+  const backfill = createPublicClient(options)
+
+  const liveTransport = options.realtimeTransport ?? options.transport
+
+  const live = createPublicClient({
+    chain: options.chain,
+    transport: liveTransport,
+    pollingInterval: 1000,
+  })
+
+  return {
+    backfill,
+    live,
+  }
+}

package/src/rpc/get-block.ts

@@ -0,0 +1,100 @@
+import type { Hash, PublicClient } from 'viem'
+import type { Database } from '../db/client.ts'
+import {
+  encodeBlockWithTransactions,
+  encodeNullRoundBlock,
+} from '../db/encode.ts'
+import type { EncodedBlockWithTransactions } from '../types.ts'
+
+/**
+ * Fetches a block while normalizing null-round behavior into an explicit result.
+ */
+export async function safeGetBlock(options: {
+  client: PublicClient
+  blockNumber: bigint
+  db: Database
+}): Promise<EncodedBlockWithTransactions> {
+  const { client, blockNumber, db } = options
+  try {
+    const block = await client.getBlock({
+      blockNumber,
+      includeTransactions: true,
+    })
+
+    return encodeBlockWithTransactions(block)
+  } catch (error) {
+    if (isNullRoundRpcError(error)) {
+      let previousBlock:
+        | { number: bigint; hash: Hash; parentHash: Hash }
+        | undefined
+
+      previousBlock = (
+        await db.$prepared.getBlockById.execute({
+          blockNumber: blockNumber - 1n,
+        })
+      )[0]
+
+      let previousBlockNumber = blockNumber - 1n
+      // go to the chain and loop back until a full block is found
+      if (!previousBlock) {
+        while (!previousBlock) {
+          try {
+            const block = await client.getBlock({
+              blockNumber: previousBlockNumber,
+            })
+            previousBlock = {
+              number: block.number,
+              hash: block.hash,
+              parentHash: block.parentHash,
+            }
+          } catch (error) {
+            // catched another null round, keep going
+            if (isNullRoundRpcError(error)) {
+              previousBlockNumber -= 1n
+              continue
+            }
+            throw error
+          }
+        }
+      }
+      return encodeNullRoundBlock({
+        number: blockNumber,
+        hash: previousBlock.hash,
+      })
+    }
+    throw error
+  }
+}
+
+/**
+ * Detects Filecoin null-round RPC errors so callers can skip non-existent rounds.
+ */
+export function isNullRoundRpcError(error: unknown): boolean {
+  if (!(error instanceof Error)) return false
+
+  const message = error.message.toLowerCase()
+  if (message.includes('null round')) {
+    return true
+  }
+
+  const details = (error as { details?: unknown }).details
+  if (
+    typeof details === 'string' &&
+    details.toLowerCase().includes('null round')
+  ) {
+    return true
+  }
+
+  const cause = (error as { cause?: unknown }).cause
+  if (cause && typeof cause === 'object') {
+    const causeMessage = (cause as { message?: unknown }).message
+    if (
+      typeof causeMessage === 'string' &&
+      causeMessage.toLowerCase().includes('null round')
+    ) {
+      return true
+    }
+  }
+
+  return false
+}

package/src/rpc/get-logs.ts

@@ -0,0 +1,38 @@
+import type { AbiEvent, Address, Log, PublicClient } from 'viem'
+import type { Logger } from '../utils/logger.ts'
+import { startClock } from '../utils/timer.ts'
+
+export async function getLogsInRange(args: {
+  logger: Logger
+  client: PublicClient
+  addresses: Address[]
+  events: readonly AbiEvent[]
+  fromBlock: bigint
+  toBlock: bigint
+}): Promise<Map<bigint, Log<bigint, number, false, AbiEvent>[]>> {
+  const { logger, client, addresses, events, fromBlock, toBlock } = args
+
+  const endClock = startClock()
+
+  const logsByBlock = new Map<bigint, Log<bigint, number, false, AbiEvent>[]>()
+  const logs = await client.getLogs({
+    address: addresses,
+    events: events,
+    fromBlock: fromBlock,
+    toBlock: toBlock,
+  })
+
+  for (const log of logs) {
+    const byBlock = logsByBlock.get(log.blockNumber) ?? []
+    byBlock.push(log)
+    logsByBlock.set(log.blockNumber, byBlock)
+  }
+  logger.info(
+    {
+      logs: logsByBlock.size,
+      duration: endClock(),
+    },
+    'get logs'
+  )
+  return logsByBlock
+}

package/src/schema.ts

@@ -0,0 +1,10 @@
+import { dirname, resolve } from 'node:path'
+import { fileURLToPath } from 'node:url'
+
+const __dirname = dirname(fileURLToPath(import.meta.url))
+
+export const schemaFiles = [
+  resolve(__dirname, './db/schema/blocks.js'),
+  resolve(__dirname, './db/schema/transactions.js'),
+]
+export { schema } from './db/schema/index.ts'

package/src/types.ts

@@ -0,0 +1,32 @@
+import type { Simplify } from 'type-fest'
+import type { Block, Transaction } from 'viem'
+import type { Schema } from './db/schema/index'
+/**
+ * Generic result with error
+ */
+export type MaybeResult<ResultType = unknown, ErrorType = Error> =
+  | {
+      error: ErrorType
+      result?: undefined
+    }
+  | {
+      result: ResultType
+      error?: undefined
+    }
+
+export type UnknownObject = NonNullable<unknown>
+
+export type ChainTransaction = Transaction<bigint, number, false>
+export type ChainBlock = Block<
+  bigint,
+  true,
+  'latest' | 'safe' | 'finalized',
+  ChainTransaction
+>
+export type EncodedBlock = Schema['blocks']['$inferInsert']
+export type EncodedTransaction = Schema['transactions']['$inferInsert']
+export type EncodedBlockWithTransactions = Simplify<
+  EncodedBlock & {
+    transactions: EncodedTransaction[]
+  }
+>

package/src/utils/bloom.ts

@@ -0,0 +1,41 @@
+/** biome-ignore-all lint/style/noNonNullAssertion: no need to check for null */
+import { type Hex, hexToBytes, keccak256 } from 'viem'
+
+export const zeroLogsBloom =
+  '0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000'
+
+// const BLOOM_SIZE_BYTES = 256;
+
+export const isInBloom = (bloomHex: Hex, value: Hex): boolean => {
+  const bloom = hexToBytes(bloomHex)
+  const hash = hexToBytes(keccak256(value))
+
+  // Ethereum uses 3 pairs of bytes from the hash to determine 3 bits
+  for (let i = 0; i < 6; i += 2) {
+    // Calculate the bit index (0 to 2047)
+    const bitIndex = ((hash[i]! << 8) | hash[i + 1]!) & 0x7ff
+
+    // Check if that bit is set in the 256-byte bloom array
+    const byteIndex = 255 - Math.floor(bitIndex / 8)
+    const bitMask = 1 << (bitIndex % 8)
+
+    if ((bloom[byteIndex]! & bitMask) === 0) {
+      return false // Definitely NOT in this block
+    }
+  }
+  return true // PROBABLY in this block (could be a false positive)
+}
+
+export function isBlockInteresting(
+  bloomHex: Hex,
+  addresses: Hex[],
+  topics: Hex[]
+): boolean {
+  // Check if ANY of our target contracts might be in this block
+  const hasContract = addresses.some((addr) => isInBloom(bloomHex, addr))
+  if (!hasContract) return false
+
+  // Check if ANY of our target event signatures might be in this block
+  const hasEvent = topics.some((topic) => isInBloom(bloomHex, topic))
+  return hasEvent
+}

package/src/utils/build-conflict-columns.ts

@@ -0,0 +1,26 @@
+import { type SQL, sql } from 'drizzle-orm'
+import type { PgTable } from 'drizzle-orm/pg-core'
+import { getColumns } from 'drizzle-orm/utils'
+import { snakeCase } from 'scule'
+
+export const buildConflictUpdateColumns = <
+  T extends PgTable,
+  Q extends keyof T['_']['columns'],
+>(
+  table: T,
+  columns?: Q[]
+) => {
+  const cls = getColumns(table)
+  const cols = columns ?? (Object.keys(cls) as Q[])
+  const r = cols.reduce(
+    (acc, column) => {
+      const colName = snakeCase(cls[column].name)
+
+      acc[column] = sql.raw(`excluded.${colName}`)
+      return acc
+    },
+    {} as Record<Q, SQL>
+  )
+
+  return r
+}

package/src/utils/common.ts

@@ -0,0 +1,3 @@
+export const noop = (): void => {
+  /** no-op */
+}

package/src/utils/cursor.ts

@@ -0,0 +1,7 @@
+/**
+ * Returns an inclusive window end, capped by a max block.
+ */
+export function windowEnd(start: bigint, size: bigint, max: bigint): bigint {
+  const end = start + size - 1n
+  return end <= max ? end : max
+}

package/src/utils/format.ts

@@ -0,0 +1,18 @@
+export const formatLogDuration = (ms: number) => {
+  // If less than 1 second, return ms.
+  if (ms < 1000) return `${Math.round(ms)}ms`
+  const seconds = Math.floor(ms / 1000)
+
+  const h = Math.floor(seconds / 3600)
+  const m = Math.floor((seconds - h * 3600) / 60)
+  const s = seconds - h * 3600 - m * 60
+  const secWithMs = ((ms % 60000) / 1000).toFixed(3).replace(/\.?0+$/, '') // seconds including ms fraction, max 59.999
+
+  const hstr = h > 0 ? `${h}h ` : ''
+  const mstr = m > 0 || h > 0 ? `${m < 10 && h > 0 ? '0' : ''}${m}m ` : ''
+  // Add milliseconds fraction to the seconds string, e.g., 2.322s
+  const sstr =
+    s > 0 || m > 0 ? `${s < 10 && m > 0 ? '0' : ''}${secWithMs}s` : ''
+
+  return `${hstr}${mstr}${sstr}`
+}