@hugomrdias/foxer 0.0.1

package/src/db/migrate.ts

@@ -0,0 +1,222 @@
+import { migrate as migratePostgresJs } from 'drizzle-orm/node-postgres/migrator'
+import { getTableConfig, type PgTable } from 'drizzle-orm/pg-core'
+import { IndexedColumn } from 'drizzle-orm/pg-core/columns/common'
+import { migrate as migratePglite } from 'drizzle-orm/pglite/migrator'
+import { FOXER_TABLES, PUBLICATION_NAME } from '../contants.ts'
+import type { Logger } from '../utils/logger.ts'
+import { startClock } from '../utils/timer.ts'
+import type { Database, DatabaseContext } from './client.ts'
+
+/**
+ * Applies pending SQL migrations at runtime.
+ */
+export async function runMigrations({
+  dbContext,
+  folder,
+  logger,
+}: {
+  folder: string
+  dbContext: DatabaseContext
+  logger: Logger
+}): Promise<void> {
+  const endClock = startClock()
+  const { db, driver } = dbContext
+  // apply migrations
+  if (driver === 'postgres') {
+    await migratePostgresJs(db, { migrationsFolder: folder })
+  } else {
+    await migratePglite(db, { migrationsFolder: folder })
+  }
+
+  // get tables to migrate
+  const tables = Object.keys(dbContext.db._.fullSchema).filter(
+    (table) => !FOXER_TABLES.includes(table)
+  )
+  // assert tables have blockNumber column and index
+  assertTablesHaveBlockNumberIndex(dbContext.db._.fullSchema, tables)
+
+  // check if wal is enabled
+  const wal = await isWalEnabled(db)
+  if (!wal) {
+    throw new Error(
+      'WAL is not enabled, set wal_level=logical in your postgresql.conf or pass -c wal_level=logical to the postgres client'
+    )
+  }
+
+  // create publications
+  await createPublications(db, tables)
+
+  logger.info({ driver, duration: endClock() }, 'migrations applied')
+}
+
+/**
+ * Creates or updates a publication for the provided tables in the database.
+ * The resulting publication contains only the provided tables.
+ * This is needed for the live sync to work.
+ *
+ * @param db - The database to create the publication for
+ * @param tables - The list of table names to include
+ */
+export async function createPublications(db: Database, tables: string[]) {
+  if (tables.length === 0) return
+
+  const quotedTables = tables.map((table) => `"${table.replaceAll('"', '""')}"`)
+  const publication = await db.execute(
+    `SELECT puballtables FROM pg_publication WHERE pubname = '${PUBLICATION_NAME}'`
+  )
+
+  if (publication.rows.length === 0) {
+    await db.execute(
+      `CREATE PUBLICATION ${PUBLICATION_NAME} FOR TABLE ${quotedTables.join(', ')}`
+    )
+    return
+  }
+
+  const isForAllTables = Boolean(publication.rows[0].puballtables)
+  if (isForAllTables) {
+    await db.execute(`DROP PUBLICATION ${PUBLICATION_NAME}`)
+    await db.execute(
+      `CREATE PUBLICATION ${PUBLICATION_NAME} FOR TABLE ${quotedTables.join(', ')}`
+    )
+    return
+  }
+
+  await db.execute(
+    `ALTER PUBLICATION ${PUBLICATION_NAME} SET TABLE ${quotedTables.join(', ')}`
+  )
+}
+
+/**
+ * Ensures every provided table has a blockNumber column and an index on it.
+ */
+export function assertTablesHaveBlockNumberIndex(
+  fullSchema: Record<string, unknown>,
+  tableNames: string[]
+) {
+  const missingBlockNumberColumn: string[] = []
+  const missingBlockNumberIndex: string[] = []
+  const tableConfigs = new Map<string, ReturnType<typeof getTableConfig>>()
+  const tableHasBlockNumberColumn = new Map<string, boolean>()
+
+  for (const tableName of tableNames) {
+    const table = fullSchema[tableName]
+    if (!table) continue
+
+    const config = getTableConfig(table as PgTable)
+    tableConfigs.set(tableName, config)
+    tableHasBlockNumberColumn.set(
+      tableName,
+      config.columns.some((column) =>
+        ['blockNumber', 'block_number'].includes(column.name)
+      )
+    )
+  }
+
+  for (const tableName of tableNames) {
+    const config = tableConfigs.get(tableName)
+    if (!config) continue
+    const blockNumberColumns = config.columns.filter((column) =>
+      ['blockNumber', 'block_number'].includes(column.name)
+    )
+
+    if (blockNumberColumns.length === 0) {
+      if (
+        hasCascadeForeignKeyToBlockNumberTable(
+          config,
+          tableHasBlockNumberColumn
+        )
+      ) {
+        continue
+      }
+      missingBlockNumberColumn.push(tableName)
+      continue
+    }
+
+    const blockNumberColumnNames = new Set(
+      blockNumberColumns.map((column) => column.name)
+    )
+    const hasBlockNumberIndex = config.indexes.some((index) => {
+      const indexColumns = index.config?.columns
+
+      if (!indexColumns || indexColumns.length === 0) return false
+      return indexColumns.some((column) => {
+        if (!(column instanceof IndexedColumn)) return false
+        if (!column.name) return false
+        return blockNumberColumnNames.has(column.name)
+      })
+    })
+
+    if (!hasBlockNumberIndex) {
+      missingBlockNumberIndex.push(tableName)
+    }
+  }
+
+  if (
+    missingBlockNumberColumn.length > 0 ||
+    missingBlockNumberIndex.length > 0
+  ) {
+    const missingColumnTables = missingBlockNumberColumn.sort()
+    const missingIndexTables = missingBlockNumberIndex.sort()
+    const lines = [
+      'Invalid schema for Foxer sync.',
+      '',
+      'Each published table must have:',
+      "1) a 'blockNumber' column (db name can be 'block_number')",
+      "2) an index that includes 'blockNumber'",
+      "Exception: table can skip both when it has a foreign key with onDelete('cascade') to a table with blockNumber.",
+      '',
+    ]
+
+    if (missingColumnTables.length > 0) {
+      lines.push(
+        `Tables missing blockNumber column: ${missingColumnTables.join(', ')}`
+      )
+    }
+    if (missingIndexTables.length > 0) {
+      lines.push(
+        `Tables missing blockNumber index: ${missingIndexTables.join(', ')}`
+      )
+    }
+
+    lines.push(
+      '',
+      'Drizzle example:',
+      "const myTable = pgTable('my_table', {",
+      '  // 1) Add the blockNumber column',
+      '  blockNumber: bigint().notNull(),',
+      '  // ...other columns',
+      '}, (table) => [',
+      '  // 2) Add an index on blockNumber',
+      "  index('my_table_block_number_index').on(table.blockNumber),",
+      '])'
+    )
+
+    throw new Error(lines.join('\n'))
+  }
+}
+
+function hasCascadeForeignKeyToBlockNumberTable(
+  tableConfig: ReturnType<typeof getTableConfig>,
+  tableHasBlockNumberColumn: Map<string, boolean>
+) {
+  return tableConfig.foreignKeys.some((foreignKey) => {
+    if (foreignKey.onDelete !== 'cascade') return false
+
+    const referencedTable = foreignKey.reference().foreignTable
+    const referencedTableName = getTableConfig(referencedTable).name
+    return tableHasBlockNumberColumn.get(referencedTableName) === true
+  })
+}
+
+/**
+ * Checks if WAL is enabled.
+ *
+ * @param db - The database to check
+ * @returns True if WAL is enabled, false otherwise
+ */
+export async function isWalEnabled(db: Database) {
+  const wal = await db
+    .execute('SHOW WAL_LEVEL')
+    .then((result) => result.rows[0].wal_level)
+  return wal === 'logical'
+}

package/src/db/schema/blocks.ts

@@ -0,0 +1,24 @@
+import { pgTable } from 'drizzle-orm/pg-core'
+import { address, bigint, bytea, hash, numeric78 } from '../column-types.ts'
+
+export const blocks = pgTable('blocks', {
+  number: bigint().notNull().primaryKey(),
+  timestamp: bigint().notNull(),
+  hash: hash().notNull(),
+  parentHash: hash().notNull(),
+  logsBloom: bytea().notNull(),
+  miner: address().notNull(),
+  gasUsed: numeric78().notNull(),
+  gasLimit: numeric78().notNull(),
+  baseFeePerGas: numeric78(),
+  nonce: bytea().notNull(),
+  mixHash: bytea().notNull(),
+  stateRoot: bytea().notNull(),
+  receiptsRoot: bytea().notNull(),
+  transactionsRoot: bytea().notNull(),
+  sha3Uncles: bytea().notNull(),
+  size: numeric78().notNull(),
+  difficulty: numeric78().notNull(),
+  totalDifficulty: numeric78(),
+  extraData: bytea().notNull(),
+})

package/src/db/schema/index.ts

@@ -0,0 +1,21 @@
+import { defineRelations } from 'drizzle-orm'
+
+import { blocks } from './blocks.ts'
+import { transactions } from './transactions.ts'
+
+export const relations = defineRelations({ blocks, transactions }, (r) => {
+  return {
+    blocks: {
+      transactions: r.many.transactions(),
+    },
+    transactions: {
+      block: r.one.blocks({
+        from: r.transactions.blockNumber,
+        to: r.blocks.number,
+      }),
+    },
+  }
+})
+
+export const schema = { blocks, transactions }
+export type Schema = typeof schema

package/src/db/schema/transactions.ts

@@ -0,0 +1,39 @@
+import { index, integer, jsonb, pgEnum, pgTable } from 'drizzle-orm/pg-core'
+import type { AccessList } from 'viem'
+import { address, bigint, bytea, hash, numeric78 } from '../column-types.ts'
+
+export const transactionTypeEnum = pgEnum('transaction_type', [
+  'legacy',
+  'eip1559',
+  'eip2930',
+  'eip4844',
+  'eip7702',
+])
+
+export const transactions = pgTable(
+  'transactions',
+  {
+    hash: hash().primaryKey(),
+    blockNumber: bigint().notNull(),
+    transactionIndex: integer().notNull(),
+    blockHash: hash().notNull(),
+    from: address().notNull(),
+    to: address(),
+    input: bytea().notNull(),
+    value: numeric78().notNull(),
+    nonce: integer().notNull(),
+    r: bytea().notNull(),
+    s: bytea().notNull(),
+    v: numeric78().notNull(),
+    type: transactionTypeEnum().notNull(),
+    gas: numeric78().notNull(),
+    gasPrice: numeric78(),
+    maxFeePerGas: numeric78(),
+    maxPriorityFeePerGas: numeric78(),
+    accessList: jsonb().$type<AccessList>(),
+  },
+  (table) => [
+    index('transactions_block_number_index').on(table.blockNumber),
+    index('transactions_to_index').on(table.to),
+  ]
+)

package/src/db/transaction.ts

@@ -0,0 +1,20 @@
+import type { Database } from './client'
+import type { relations, schema } from './schema/index.ts'
+
+/**
+ * Runs work in a transaction for either postgres or pglite drivers.
+ */
+export function withTransaction<T>(
+  db: Database<typeof schema, typeof relations>,
+  run: (tx: Database<typeof schema, typeof relations>) => Promise<T>
+): Promise<T> {
+  const executor = db as unknown as Database<
+    typeof schema,
+    typeof relations
+  > & {
+    transaction: <R>(
+      fn: (tx: Database<typeof schema, typeof relations>) => Promise<R>
+    ) => Promise<R>
+  }
+  return executor.transaction(run)
+}

package/src/hooks/registry.ts

@@ -0,0 +1,107 @@
+import type { AnyRelations, EmptyRelations } from 'drizzle-orm/relations'
+import type { GetEventArgs, Log } from 'viem'
+import type { Database } from '../db/client'
+import type { EncodedBlockWithTransactions, EncodedTransaction } from '../types'
+import type { Logger } from '../utils/logger'
+import type {
+  ContractAbiByEventKey,
+  ContractAbiEventByEventKey,
+  ContractsConfig,
+  EventKey,
+  EventNameFromEventKey,
+  MergedContractEvents,
+} from '../utils/types'
+
+export type HookContext<
+  TSchema extends Record<string, unknown> = Record<string, unknown>,
+  TRelations extends AnyRelations = EmptyRelations,
+> = {
+  db: Database<TSchema, TRelations>
+  chainId: number
+  logger: Logger
+}
+
+export type DecodedEvent<
+  C extends ContractsConfig<NonNullable<unknown>>,
+  Event extends EventKey,
+> = {
+  // Resolve the concrete ABI for the `contract:event` key.
+  args: GetEventArgs<
+    ContractAbiByEventKey<C, Event>,
+    EventNameFromEventKey<Event>,
+    { EnableUnion: false; IndexedOnly: false; Required: true }
+  >
+  log: Log<bigint, number, false, ContractAbiEventByEventKey<C, Event>>
+  block: EncodedBlockWithTransactions
+  transaction: EncodedTransaction
+}
+
+export type EventHook<
+  C extends ContractsConfig<NonNullable<unknown>>,
+  Event extends EventKey = EventKey,
+  TSchema extends Record<string, unknown> = Record<string, unknown>,
+  TRelations extends AnyRelations = EmptyRelations,
+> = (args: {
+  context: HookContext<TSchema, TRelations>
+  event: DecodedEvent<C, Event>
+}) => Promise<void> | void
+
+/**
+ * Registry for strongly typed contract-event hooks.
+ */
+export class HookRegistry<
+  C extends ContractsConfig<NonNullable<unknown>> = ContractsConfig<
+    NonNullable<unknown>
+  >,
+  TSchema extends Record<string, unknown> = Record<string, unknown>,
+  TRelations extends AnyRelations = EmptyRelations,
+> {
+  private readonly hooks = new Map<MergedContractEvents<C>, unknown>()
+
+  /**
+   * Registers a hook for a specific `contract:event` key.
+   */
+  on<K extends MergedContractEvents<C>>(
+    streamKey: K,
+    hook: EventHook<C, K, TSchema, TRelations>
+  ): void {
+    this.hooks.set(streamKey, hook)
+  }
+
+  /**
+   * Decodes a log using stream ABI metadata and dispatches to the registered hook.
+   */
+  async dispatch<K extends MergedContractEvents<C>>(options: {
+    key: K
+    args: GetEventArgs<
+      ContractAbiByEventKey<C, K>,
+      EventNameFromEventKey<K>,
+      { EnableUnion: false; IndexedOnly: false; Required: true }
+    >
+    log: Log<bigint, number, false, ContractAbiEventByEventKey<C, K>>
+    block: EncodedBlockWithTransactions
+    transaction: EncodedTransaction
+    context: HookContext<TSchema, TRelations>
+  }): Promise<void> {
+    const { key, args, log, block, transaction, context } = options
+    const hook = this.hooks.get(key) as unknown as EventHook<
+      C,
+      K,
+      TSchema,
+      TRelations
+    >
+    if (!hook) return
+
+    const event = {
+      args,
+      log,
+      block,
+      transaction,
+    } as DecodedEvent<C, K>
+
+    await hook({
+      context,
+      event,
+    })
+  }
+}

package/src/index.ts

@@ -0,0 +1,9 @@
+export type * from './config/config.ts'
+export { createConfig } from './config/config.ts'
+export type { Database } from './db/client.ts'
+export * from './db/column-types.ts'
+export type { HookRegistry } from './hooks/registry.ts'
+export type * from './rpc/client.ts'
+export { buildConflictUpdateColumns } from './utils/build-conflict-columns.ts'
+export type { Logger } from './utils/logger.ts'
+export type * from './utils/types.ts'

package/src/indexer/backfill.ts

@@ -0,0 +1,133 @@
+import { filterContracts, type InternalConfig } from '../config/config.ts'
+import { getBlocksInRange } from '../db/actions/blocks.ts'
+import type { Database } from '../db/client.ts'
+import type { relations, schema } from '../db/schema/index.ts'
+import { withTransaction } from '../db/transaction.ts'
+import type { HookRegistry } from '../hooks/registry.ts'
+import { getLogsInRange } from '../rpc/get-logs.ts'
+import { windowEnd } from '../utils/cursor.ts'
+import type { Logger } from '../utils/logger.ts'
+import { startClock } from '../utils/timer.ts'
+import { processBlock } from './process-block.ts'
+
+/**
+ * Executes historical catch-up from the current cursor to the safe head.
+ */
+export async function runBackfill(args: {
+  logger: Logger
+  config: InternalConfig
+  db: Database<typeof schema, typeof relations>
+  registry: HookRegistry
+}): Promise<bigint> {
+  const endClock = startClock()
+  const { db, registry, config, logger } = args
+  const client = config.clients.backfill
+  const chainHead = await client.getBlockNumber()
+  const safeHead =
+    chainHead > config.finality ? chainHead - config.finality : 0n
+  let cursor = config.startBlockNumber
+
+  if (cursor > safeHead) {
+    logger.debug(
+      {
+        cursor: cursor.toString(),
+        backfillHead: safeHead.toString(),
+        head: chainHead.toString(),
+      },
+      'no historical catch-up needed'
+    )
+    return cursor
+  }
+
+  const batchSize = config.batchSize
+  logger.debug(
+    {
+      fromBlock: cursor.toString(),
+      toBlock: safeHead.toString(),
+      batchSize: batchSize.toString(),
+    },
+    'starting backfill'
+  )
+
+  while (cursor <= safeHead) {
+    const batchStartMs = Date.now()
+    const toBlock = windowEnd(cursor, batchSize, safeHead)
+    const windowContracts = filterContracts(config, cursor, toBlock)
+
+    logger.debug(
+      {
+        batchFromBlock: cursor.toString(),
+        batchToBlock: toBlock.toString(),
+        streamCount: windowContracts.addresses.length,
+      },
+      'processing backfill batch'
+    )
+    const batchBlockNumbers: bigint[] = []
+    let blockNumber = cursor
+    while (blockNumber <= toBlock) {
+      batchBlockNumbers.push(blockNumber)
+      blockNumber += 1n
+    }
+
+    const [blocksByNumber, logsByBlock] = await Promise.all([
+      getBlocksInRange(logger, db, batchBlockNumbers, client, windowContracts),
+      getLogsInRange({
+        logger,
+        client,
+        addresses: windowContracts.addresses,
+        events: windowContracts.eventAbis,
+        fromBlock: cursor,
+        toBlock,
+      }),
+    ])
+
+    let blockIndex = 0
+
+    const endClockBatch = startClock()
+    await withTransaction(db, async (tx) => {
+      while (blockIndex < batchBlockNumbers.length) {
+        const blockNumber = batchBlockNumbers[blockIndex]
+        const prefetchedBlock = blocksByNumber.get(blockNumber)
+
+        await processBlock({
+          logger,
+          config,
+          db: tx,
+          client,
+          registry,
+          blockNumber,
+          logs: logsByBlock.get(blockNumber) ?? [],
+          block: prefetchedBlock,
+          type: 'backfill',
+          contracts: windowContracts,
+        })
+        blockIndex += 1
+      }
+    })
+    logger.info(
+      { duration: endClockBatch() },
+      'batch block and events processed'
+    )
+    const batchElapsedMs = Date.now() - batchStartMs
+    const blocksInRange = Number(toBlock - cursor + 1n)
+    const blocksPerSecond =
+      batchElapsedMs > 0
+        ? blocksInRange / (batchElapsedMs / 1000)
+        : blocksInRange
+    logger.info(
+      {
+        indexedUpTo: toBlock.toString(),
+        duration: batchElapsedMs,
+        throughput: Number(blocksPerSecond.toFixed(2)),
+      },
+      'backfill batch completed'
+    )
+    cursor = toBlock + 1n
+  }
+
+  logger.info(
+    { duration: endClock(), blocks: cursor - config.startBlockNumber },
+    'backfill completed'
+  )
+  return cursor
+}

package/src/indexer/live.ts

@@ -0,0 +1,76 @@
+import PQueue from 'p-queue'
+import type { PublicClient } from 'viem'
+import type { InternalConfig } from '../config/config.ts'
+import type { Database } from '../db/client.ts'
+import type { relations, schema } from '../db/schema/index.ts'
+import type { HookRegistry } from '../hooks/registry.ts'
+import { noop } from '../utils/common.ts'
+import type { Logger } from '../utils/logger.ts'
+import { queueBlock } from './queue-block.ts'
+
+/**
+ * Starts live head following and sequential block processing.
+ */
+export function startLiveSync(args: {
+  logger: Logger
+  config: InternalConfig
+  db: Database<typeof schema, typeof relations>
+  client: PublicClient
+  registry: HookRegistry
+  initialCursor: bigint
+}): { stop: () => void } {
+  const { config, db, client, registry, logger } = args
+
+  // filter out contracts that have endBlock set
+  const contracts = config.contractsForLive
+
+  if (contracts.length === 0) {
+    logger.debug(
+      'all configured contracts have endBlock set; live sync disabled'
+    )
+    return { stop: noop }
+  }
+
+  const pqueue = new PQueue({ concurrency: 1 })
+  pqueue.on('error', (error) => {
+    logger.error({ error }, 'live queue error')
+  })
+
+  let nextBlockToQueue = args.initialCursor
+
+  const unwatch = client.watchBlockNumber({
+    emitMissed: true,
+    emitOnBegin: true,
+    onBlockNumber: (head) => {
+      while (nextBlockToQueue <= head) {
+        const blockNumber = nextBlockToQueue
+        pqueue.add(async () => {
+          await queueBlock({
+            logger,
+            blockNumber,
+            config,
+            db,
+            client,
+            registry,
+            queueSize: pqueue.size,
+            onRewind: (rewindTo) => {
+              nextBlockToQueue = rewindTo
+              pqueue.clear()
+            },
+          })
+        })
+        nextBlockToQueue += 1n
+      }
+    },
+    onError: (error) => {
+      logger.error({ error }, 'watchBlockNumber stream error')
+    },
+  })
+
+  logger.debug(
+    { startBlock: nextBlockToQueue.toString() },
+    'watching latest chain head'
+  )
+
+  return { stop: unwatch }
+}