@hugomrdias/foxer 0.0.1

package/src/db/actions/blocks.ts

@@ -0,0 +1,209 @@
+/** biome-ignore-all lint/style/noNonNullAssertion: its ok */
+
+import { gte } from 'drizzle-orm'
+import {
+  getTableConfig,
+  type PgAsyncTransaction,
+  type PgColumn,
+  type PgQueryResultHKT,
+  type PgTable,
+} from 'drizzle-orm/pg-core'
+import type { PublicClient } from 'viem'
+import type { FilteredContracts } from '../../config/config.ts'
+import { MAX_QUERY_PARAMS } from '../../contants.ts'
+import { safeGetBlock } from '../../rpc/get-block.ts'
+import type {
+  EncodedBlockWithTransactions,
+  EncodedTransaction,
+} from '../../types.ts'
+import type { Logger } from '../../utils/logger.ts'
+import { startClock } from '../../utils/timer.ts'
+import type { Database } from '../client.ts'
+import { type relations, schema } from '../schema/index.ts'
+import { insertTransactionsInChunks } from './transactions.ts'
+
+/**
+ * Deletes canonical block rows from a specific block onward.
+ */
+export async function deleteBlocksFrom(
+  db: Database,
+  fromBlock: bigint
+): Promise<void> {
+  const deleteTargets = getTablesWithBlockNumberColumn(
+    db._.fullSchema as Record<string, unknown>
+  )
+
+  await db.transaction(async (tx) => {
+    for (const target of deleteTargets) {
+      await tx
+        .delete(target.table)
+        .where(gte(target.blockNumberColumn, fromBlock))
+    }
+
+    // blocks uses `number` instead of blockNumber
+    await tx.delete(schema.blocks).where(gte(schema.blocks.number, fromBlock))
+  })
+}
+
+function getTablesWithBlockNumberColumn(fullSchema: Record<string, unknown>) {
+  const targets: Array<{ table: PgTable; blockNumberColumn: PgColumn }> = []
+
+  for (const table of Object.values(fullSchema)) {
+    const pgTable = table as PgTable
+    const config = getTableConfig(pgTable)
+    const blockNumberColumn = config.columns.find((column) =>
+      ['blockNumber', 'block_number'].includes(column.name)
+    )
+    if (!blockNumberColumn) continue
+
+    targets.push({ table: pgTable, blockNumberColumn })
+  }
+
+  return targets
+}
+
+/**
+ * Caches a block and its transactions in the database.
+ *
+ * @param args - The arguments for the function
+ * @param args.db - The database instance
+ * @param args.block - The block to cache
+ * @returns The cached block and transactions
+ */
+export async function cacheBlockAndTransactions(args: {
+  db: Database<typeof schema, typeof relations>
+  block: EncodedBlockWithTransactions
+  logger: Logger
+}): Promise<void> {
+  const { db, block } = args
+
+  await db.transaction(async (tx) => {
+    await insertBlocksInChunks({
+      db: tx,
+      blocks: [block],
+    })
+    await insertTransactionsInChunks({
+      db: tx,
+      transactions: block.transactions,
+    })
+  })
+}
+
+/**
+ * Gets blocks and their transactions from the database in a range and fetches missing blocks from the RPC.
+ *
+ * @param args - The arguments for the function
+ * @param args.logger - The logger instance
+ * @param args.db - The database instance
+ * @param args.blockNumbers - The block numbers to get
+ * @param args.client - The client instance
+ * @returns The blocks and their transactions
+ */
+export async function getBlocksInRange(
+  logger: Logger,
+  db: Database<typeof schema, typeof relations>,
+  blockNumbers: bigint[],
+  client: PublicClient,
+  contracts: FilteredContracts
+): Promise<Map<bigint, EncodedBlockWithTransactions>> {
+  const endClock = startClock()
+  const firstBlockNumber = blockNumbers[0]!
+  const lastBlockNumber = blockNumbers[blockNumbers.length - 1]!
+
+  // const r = await db.$prepared.getBlocksInRange.execute({
+  //   firstBlockNumber,
+  //   lastBlockNumber,
+  //   contractAddresses: contracts.addresses,
+  // })
+
+  const r = await db.query.blocks.findMany({
+    with: {
+      transactions: {
+        where: {
+          AND: [
+            { blockNumber: { gte: firstBlockNumber } },
+            { blockNumber: { lte: lastBlockNumber } },
+            {
+              to: {
+                in: contracts.addresses,
+              },
+            },
+          ],
+        },
+      },
+    },
+    where: {
+      AND: [
+        { number: { gte: firstBlockNumber } },
+        { number: { lte: lastBlockNumber } },
+      ],
+    },
+  })
+
+  const blocksByNumber = new Map<bigint, EncodedBlockWithTransactions>()
+  const missing = new Set(blockNumbers)
+
+  for (const block of r) {
+    blocksByNumber.set(block.number, block)
+    missing.delete(block.number)
+  }
+
+  const missingBlockNumbers = [...missing]
+  const newBlocks: EncodedBlockWithTransactions[] = []
+  const newTransactions: EncodedTransaction[] = []
+
+  await Promise.all(
+    missingBlockNumbers.map(async (blockNumber) => {
+      const block = await safeGetBlock({ client, blockNumber, db })
+      const transactions = block.transactions
+      blocksByNumber.set(blockNumber, block)
+      newBlocks.push(block)
+      if (transactions.length > 0) {
+        newTransactions.push(...transactions)
+      }
+    })
+  )
+
+  await db.transaction(async (tx) => {
+    await insertBlocksInChunks({
+      db: tx,
+      blocks: newBlocks,
+    })
+    await insertTransactionsInChunks({
+      db: tx,
+      transactions: newTransactions,
+    })
+  })
+
+  logger.info(
+    {
+      blocks: blocksByNumber.size,
+      missing: missingBlockNumbers.length,
+      duration: endClock(),
+    },
+    'get blocks'
+  )
+  return blocksByNumber
+}
+
+/**
+ * Inserts blocks in chunks to avoid query parameter limit.
+ */
+export async function insertBlocksInChunks(args: {
+  db: PgAsyncTransaction<PgQueryResultHKT, typeof schema>
+  blocks: EncodedBlockWithTransactions[]
+}): Promise<void> {
+  const { db, blocks } = args
+  if (blocks.length === 0) return
+
+  const batchSize = Math.floor(MAX_QUERY_PARAMS / Object.keys(blocks[0]).length)
+  for (let i = 0; i < blocks.length; i += batchSize) {
+    const chunk = blocks.slice(i, i + batchSize)
+    await db
+      .insert(schema.blocks)
+      .values(chunk)
+      .onConflictDoNothing({
+        target: [schema.blocks.number],
+      })
+  }
+}

package/src/db/actions/index.ts

@@ -0,0 +1 @@
+export * as blocks from './blocks.ts'

package/src/db/actions/transactions.ts

@@ -0,0 +1,32 @@
+import type { PgAsyncTransaction, PgQueryResultHKT } from 'drizzle-orm/pg-core'
+import { MAX_QUERY_PARAMS } from '../../contants.ts'
+import type { EncodedTransaction } from '../../types.ts'
+import { schema } from '../schema/index.ts'
+
+/**
+ * Inserts transactions in chunks to avoid query parameter limit.
+ */
+export async function insertTransactionsInChunks(args: {
+  db: PgAsyncTransaction<PgQueryResultHKT, typeof schema>
+  transactions: EncodedTransaction[]
+}): Promise<void> {
+  const { db, transactions } = args
+  if (transactions.length === 0) {
+    return
+  }
+
+  const batchSize = Math.floor(
+    MAX_QUERY_PARAMS / Object.keys(transactions[0]).length
+  )
+
+  for (let i = 0; i < transactions.length; i += batchSize) {
+    const chunk = transactions.slice(i, i + batchSize)
+
+    await db
+      .insert(schema.transactions)
+      .values(chunk)
+      .onConflictDoNothing({
+        target: [schema.transactions.hash],
+      })
+  }
+}

package/src/db/client.ts

@@ -0,0 +1,186 @@
+import { PGlite } from '@electric-sql/pglite'
+import { desc, eq, sql } from 'drizzle-orm'
+import {
+  drizzle as drizzleNodePostgres,
+  type NodePgDatabase,
+} from 'drizzle-orm/node-postgres'
+import {
+  drizzle as drizzlePglite,
+  type PgliteDatabase,
+} from 'drizzle-orm/pglite'
+import type { AnyRelations, EmptyRelations } from 'drizzle-orm/relations'
+import { Pool, type PoolConfig } from 'pg'
+import type { DatabaseConfig } from '../config/config.ts'
+import type { Env } from '../config/env.ts'
+import { type relations, schema } from './schema/index.ts'
+
+export type Database<
+  TSchema extends Record<string, unknown> = Record<string, unknown>,
+  TRelations extends AnyRelations = EmptyRelations,
+> =
+  | (PgliteDatabase<TSchema, TRelations> & {
+      $client: PGlite
+      $prepared: ReturnType<typeof generatePrepared>
+    })
+  | (NodePgDatabase<TSchema, TRelations> & {
+      $client: Pool
+      $prepared: ReturnType<typeof generatePrepared>
+    })
+
+export type DbDriver = 'pglite' | 'postgres'
+
+export type DatabaseContext<
+  TSchema extends Record<string, unknown> = Record<string, unknown>,
+  TRelations extends AnyRelations = EmptyRelations,
+> =
+  | {
+      db: NodePgDatabase<TSchema, TRelations> & {
+        $client: Pool
+        $prepared: ReturnType<typeof generatePrepared>
+      }
+      driver: 'postgres'
+      close: () => Promise<void>
+    }
+  | {
+      db: PgliteDatabase<TSchema, TRelations> & {
+        $client: PGlite
+        $prepared: ReturnType<typeof generatePrepared>
+      }
+      driver: 'pglite'
+      close: () => Promise<void>
+    }
+
+/**
+ * Creates a typed Drizzle database context for either Postgres or PGlite.
+ */
+export function createDatabase<
+  TSchema extends Record<string, unknown> = Record<string, unknown>,
+  TRelations extends AnyRelations = EmptyRelations,
+>({
+  env,
+  config,
+  schema,
+  relations,
+}: {
+  env: Env
+  config?: DatabaseConfig
+  schema: TSchema
+  relations: TRelations
+}): DatabaseContext<TSchema, TRelations> {
+  let driver: string = 'pglite'
+  let url: string | undefined
+  let options: PoolConfig | undefined
+
+  if (env.DATABASE_URL && typeof env.DATABASE_URL === 'string') {
+    driver = 'postgres'
+    url = env.DATABASE_URL
+  } else if (config?.driver === 'postgres') {
+    driver = config.driver
+    url = config.url
+    options = config.options
+  }
+
+  if (driver === 'postgres' && url) {
+    const pool = new Pool({
+      ...options,
+      connectionString: url,
+    })
+    const db = drizzleNodePostgres({
+      client: pool,
+      relations: relations,
+      schema: schema,
+      casing: 'snake_case',
+    }) as NodePgDatabase<TSchema, TRelations> & {
+      $client: Pool
+      $prepared: ReturnType<typeof generatePrepared>
+    }
+
+    // @ts-expect-error - TODO: fix this
+    db.$prepared = generatePrepared(db)
+
+    return {
+      db,
+      driver: 'postgres',
+      close: async () => {
+        await pool.end()
+      },
+    }
+  }
+
+  const client = new PGlite(
+    config?.driver === 'pglite' && config.directory
+      ? config.directory
+      : '.pglite'
+  )
+  const db = drizzlePglite({
+    client: client,
+    relations: relations,
+    schema: schema,
+    casing: 'snake_case',
+  }) as PgliteDatabase<TSchema, TRelations> & {
+    $client: PGlite
+    $prepared: ReturnType<typeof generatePrepared>
+  }
+
+  // @ts-expect-error - TODO: fix this
+  db.$prepared = generatePrepared(db)
+
+  return {
+    db,
+    driver: 'pglite',
+    close: async () => {
+      await client.close()
+    },
+  }
+}
+
+function generatePrepared(
+  db: Omit<Database<typeof schema, typeof relations>, '$prepared'>
+) {
+  const getLatestBlock = db
+    .select({
+      number: schema.blocks.number,
+      hash: schema.blocks.hash,
+      parentHash: schema.blocks.parentHash,
+    })
+    .from(schema.blocks)
+    .orderBy(desc(schema.blocks.number))
+    .limit(1)
+    .prepare('get_latest_block')
+
+  const getBlockById = db
+    .select({
+      number: schema.blocks.number,
+      hash: schema.blocks.hash,
+      parentHash: schema.blocks.parentHash,
+    })
+    .from(schema.blocks)
+    .where(eq(schema.blocks.number, sql.placeholder('blockNumber')))
+    .prepare('get_block_by_id')
+
+  const getBlocksInRange = db.query.blocks
+    .findMany({
+      with: {
+        transactions: {
+          where: {
+            to: {
+              in: sql.placeholder('contractAddresses'),
+            },
+          },
+        },
+      },
+      where: {
+        AND: [
+          { number: { gte: sql.placeholder('firstBlockNumber') } },
+          { number: { lte: sql.placeholder('lastBlockNumber') } },
+        ],
+      },
+    })
+    .prepare('get_blocks_in_range')
+
+  return {
+    getLatestBlock,
+    getBlockById,
+    getBlocksInRange,
+  }
+}

package/src/db/column-types.ts

@@ -0,0 +1,105 @@
+import { customType } from 'drizzle-orm/pg-core'
+import { type Address, type Hash, type Hex, stringify } from 'viem'
+
+export const numeric78 = customType<{ data: bigint; driverData: string }>({
+  dataType() {
+    return 'numeric(78,0)'
+  },
+  fromDriver(value: string) {
+    return BigInt(value)
+  },
+})
+
+export const hex = customType<{
+  data: Hex
+  driverData: string
+  config?: { length: number | undefined }
+}>({
+  dataType(config) {
+    if (config?.length) {
+      return `varchar(${config.length})`
+    }
+    return 'varchar'
+  },
+  fromDriver(value: string) {
+    return value as Hex
+  },
+})
+
+/**
+ * Hash column type
+ * @param config - Configuration for the column
+ * @param config.length - Length of the column (default: 66)
+ * @returns Hash column type
+ */
+export const hash = customType<{
+  data: Hash
+  driverData: string
+  config?: { length: number | undefined }
+}>({
+  dataType(config) {
+    if (config?.length) {
+      return `varchar(${config.length})`
+    }
+    return 'varchar(66)'
+  },
+  fromDriver(value: string) {
+    return value as Hash
+  },
+})
+
+export const address = customType<{
+  data: Address
+  driverData: string
+  config?: { length: number | undefined }
+}>({
+  dataType(config) {
+    if (config?.length) {
+      return `varchar(${config.length})`
+    }
+    return 'varchar(42)'
+  },
+  fromDriver(value: string) {
+    return value as Address
+  },
+})
+
+/**
+ * Bigint column type (8 bytes)
+ * For bigger number see {@link numeric78}, {@link uint256} and {@link int256}
+ * @returns Bigint column type
+ */
+export const bigint = customType<{ data: bigint; driverData: string }>({
+  dataType() {
+    return 'bigint'
+  },
+  fromDriver(value: string) {
+    return BigInt(value)
+  },
+})
+
+export const uint256 = numeric78
+export const int256 = numeric78
+
+export const jsonb = customType<{ data: unknown; driverData: string }>({
+  dataType() {
+    return 'jsonb'
+  },
+  toDriver(value: unknown): string {
+    return stringify(value)
+  },
+})
+
+export const bytea = customType<{ data: Hex; driverData: Buffer }>({
+  dataType() {
+    return 'bytea'
+  },
+  toDriver(value: string): Buffer {
+    return Buffer.from(value.slice(2), 'hex')
+  },
+  fromDriver(value: Buffer): Hex {
+    const hex = value.toString('hex')
+    const _value = hex.startsWith('\\x') ? hex.slice(2) : hex
+    return `0x${_value}` as Hex
+  },
+})

package/src/db/encode.ts

@@ -0,0 +1,99 @@
+import type { Hash } from 'viem'
+import type {
+  ChainBlock,
+  ChainTransaction,
+  EncodedBlock,
+  EncodedBlockWithTransactions,
+  EncodedTransaction,
+} from '../types'
+
+export function encodeTransaction(tx: ChainTransaction): EncodedTransaction {
+  return {
+    hash: tx.hash,
+    blockNumber: tx.blockNumber,
+    transactionIndex: tx.transactionIndex,
+    blockHash: tx.blockHash,
+    from: tx.from,
+    to: tx.to ?? null,
+    input: tx.input,
+    value: tx.value,
+    nonce: tx.nonce,
+    r: tx.r,
+    s: tx.s,
+    v: tx.v,
+    type: tx.type,
+    gas: tx.gas ?? null,
+    gasPrice: tx.gasPrice ?? null,
+    maxFeePerGas: tx.maxFeePerGas ?? null,
+    maxPriorityFeePerGas: tx.maxPriorityFeePerGas ?? null,
+    accessList: tx.accessList ?? null,
+  }
+}
+
+export function encodeBlock(block: ChainBlock): EncodedBlock {
+  return {
+    number: block.number,
+    timestamp: block.timestamp,
+    hash: block.hash,
+    parentHash: block.parentHash,
+    logsBloom: block.logsBloom,
+    miner: block.miner,
+    gasUsed: block.gasUsed,
+    gasLimit: block.gasLimit,
+    baseFeePerGas: block.baseFeePerGas ?? null,
+    nonce: block.nonce,
+    mixHash: block.mixHash,
+    stateRoot: block.stateRoot,
+    receiptsRoot: block.receiptsRoot,
+    transactionsRoot: block.transactionsRoot,
+    sha3Uncles: block.sha3Uncles,
+    size: block.size,
+    difficulty: block.difficulty,
+    totalDifficulty: block.totalDifficulty ?? null,
+    extraData: block.extraData,
+  }
+}
+
+export function encodeBlockWithTransactions(
+  block: ChainBlock
+): EncodedBlockWithTransactions {
+  return {
+    ...encodeBlock(block),
+    transactions: block.transactions.map(encodeTransaction),
+  }
+}
+
+const EMPTY_TRIE_HASH =
+  '0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421'
+const EMPTY_LOGS_BLOOM =
+  '0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff'
+export function encodeNullRoundBlock(options: {
+  number: bigint
+  hash: Hash
+}): EncodedBlockWithTransactions {
+  return {
+    number: options.number,
+    // TODO: probably should be previous block timestamp plus 30 seconds
+    timestamp: BigInt(Math.floor(Date.now() / 1000)),
+    hash: options.hash,
+    parentHash: options.hash,
+    logsBloom: EMPTY_LOGS_BLOOM,
+    miner: '0x0000000000000000000000000000000000000000',
+    gasUsed: 0n,
+    gasLimit: 30_000_000n,
+    baseFeePerGas: 1_000_000_000n,
+    nonce: '0x0000000000000000',
+    mixHash:
+      '0x0000000000000000000000000000000000000000000000000000000000000000',
+    stateRoot: EMPTY_TRIE_HASH,
+    receiptsRoot: EMPTY_TRIE_HASH,
+    transactionsRoot: EMPTY_TRIE_HASH,
+    sha3Uncles:
+      '0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347',
+    size: 0n,
+    difficulty: 0n,
+    totalDifficulty: 0n,
+    extraData: '0x',
+    transactions: [],
+  }
+}