@hugomrdias/foxer 0.0.7 → 0.1.9

package/src/bin/index.ts

@@ -3,12 +3,16 @@
 import { readFileSync } from 'node:fs'
 import { dirname, resolve } from 'node:path'
 import { fileURLToPath } from 'node:url'
+
 import { cli } from 'cleye'
+
 import { create } from './create.ts'
 import { dev } from './dev.ts'
+import { serve } from './serve.ts'
+import { start } from './start.ts'
 
 const __dirname = dirname(fileURLToPath(import.meta.url))
-const packageJsonPath = resolve(__dirname, '../../../package.json')
+const packageJsonPath = resolve(__dirname, '../../package.json')
 const packageJson = JSON.parse(
   readFileSync(packageJsonPath, { encoding: 'utf8' })
 )
@@ -16,7 +20,7 @@ const packageJson = JSON.parse(
 const argv = cli({
   name: 'foxer',
   version: packageJson.version,
-  commands: [dev, create],
+  commands: [dev, create, start, serve],
   help: {
     version: packageJson.version,
   },

package/src/bin/serve.ts

@@ -0,0 +1,65 @@
+import { type Command, command } from 'cleye'
+import { gracefulExit } from 'exit-hook'
+
+import { createApi } from '../api/create-api.ts'
+import { createEnv } from '../config/env.ts'
+import { createDatabase } from '../db/client.ts'
+import * as InternalSchema from '../db/schema/index.ts'
+import { createLogger } from '../utils/logger.ts'
+import { createExit } from '../utils/shutdown.ts'
+import { globalFlags } from './flags.ts'
+import { loadConfig } from './utils.ts'
+
+export const serve: Command = command(
+  {
+    name: 'serve',
+    flags: { ...globalFlags },
+    help: {
+      description: 'Serve the production HTTP API and without the indexer',
+    },
+  },
+  async (argv) => {
+    const logger = createLogger({
+      level: argv.flags.logLevel,
+      mode: 'json',
+    })
+
+    if (process.env.DATABASE_URL == null) {
+      logger.error('DATABASE_URL environment variable is not set')
+      gracefulExit(1)
+    }
+
+    try {
+      const env = createEnv(logger)
+      const config = await loadConfig(
+        logger,
+        argv.flags.root,
+        argv.flags.config
+      )
+
+      const dbContext = createDatabase({
+        env,
+        schema: { ...config.schema, ...InternalSchema.schema },
+        relations: { ...config.relations, ...InternalSchema.relations },
+      })
+
+      const api = createApi({
+        db: dbContext.db,
+        config,
+        logger,
+        port: argv.flags.port,
+      })
+
+      createExit({
+        logger,
+        stop: async () => {
+          api.stop()
+          await dbContext.stop()
+        },
+      })
+    } catch (error) {
+      logger.error({ error }, 'HTTP API server failed')
+      gracefulExit(1)
+    }
+  }
+)

package/src/bin/start.ts

@@ -0,0 +1,87 @@
+import path from 'node:path'
+
+import { type Command, command } from 'cleye'
+import { gracefulExit } from 'exit-hook'
+
+import { createApi } from '../api/create-api.ts'
+import { createEnv } from '../config/env.ts'
+import { createDatabase } from '../db/client.ts'
+import { runMigrations } from '../db/migrate.ts'
+import * as InternalSchema from '../db/schema/index.ts'
+import { createRegistry } from '../hooks/registry.ts'
+import { createIndexer } from '../indexer/create-indexer.ts'
+import { createLogger } from '../utils/logger.ts'
+import { createExit } from '../utils/shutdown.ts'
+import { globalFlags } from './flags.ts'
+import { loadConfig } from './utils.ts'
+
+export const start: Command = command(
+  {
+    name: 'start',
+    flags: { ...globalFlags },
+    help: {
+      description: 'Start the production server',
+    },
+  },
+  async (argv) => {
+    const logger = createLogger({
+      level: argv.flags.logLevel,
+      mode: 'json',
+    })
+
+    if (process.env.DATABASE_URL == null) {
+      logger.error('DATABASE_URL environment variable is not set')
+      gracefulExit(1)
+    }
+
+    try {
+      const env = createEnv(logger)
+      const config = await loadConfig(
+        logger,
+        argv.flags.root,
+        argv.flags.config
+      )
+
+      const dbContext = createDatabase({
+        env,
+        schema: { ...config.schema, ...InternalSchema.schema },
+        relations: { ...config.relations, ...InternalSchema.relations },
+      })
+
+      await runMigrations({
+        dbContext,
+        folder: path.resolve(argv.flags.root, config.drizzleFolder),
+        logger,
+      })
+
+      const registry = createRegistry({ config })
+
+      const [api, indexer] = await Promise.all([
+        createApi({
+          db: dbContext.db,
+          config,
+          logger,
+          port: argv.flags.port,
+        }),
+        createIndexer({
+          logger,
+          db: dbContext.db,
+          registry,
+          config,
+        }),
+      ])
+
+      createExit({
+        logger,
+        stop: async () => {
+          indexer.stop()
+          api.stop()
+          await dbContext.stop()
+        },
+      })
+    } catch (error) {
+      logger.error({ error }, 'start server failed')
+      gracefulExit(1)
+    }
+  }
+)

package/src/bin/utils.ts

@@ -1,50 +1,47 @@
 import path from 'node:path'
-import { pathToFileURL } from 'node:url'
-import { type LilconfigResult, lilconfig } from 'lilconfig'
+import {
+  type LoadConfigResult,
+  loadConfig as unconfigLoadConfig,
+} from 'unconfig'
+
 import type { InternalConfig } from '../config/config'
 import type { Logger } from '../utils/logger'
 
 const CLI_NAME = 'foxer'
 
-const loadEsm = async (filepath: string) => {
-  const res = await import(pathToFileURL(filepath).href)
-  return res.default ?? res
-}
-
-const configLoaders = {
-  '.js': loadEsm,
-  '.mjs': loadEsm,
-  '.ts': loadEsm,
-  '.mts': loadEsm,
-}
-
 export async function loadConfig(
   logger: Logger,
   root: string,
   filePath?: string
 ) {
-  let configFile: LilconfigResult | undefined
+  let configFile: LoadConfigResult<{ config: InternalConfig }> | undefined
 
   try {
     if (filePath) {
       const configPath = path.resolve(root, filePath)
 
-      configFile = await lilconfig(configPath, {
-        loaders: configLoaders,
-        searchPlaces: [],
-      }).load(configPath)
+      configFile = await unconfigLoadConfig({
+        sources: [
+          {
+            files: configPath,
+          },
+        ],
+      })
     } else {
-      configFile = await lilconfig(CLI_NAME, {
-        loaders: configLoaders,
-        searchPlaces: [`${CLI_NAME}.config.ts`, `${CLI_NAME}.config.mts`],
-      }).search()
+      configFile = await unconfigLoadConfig({
+        sources: [
+          {
+            files: `${CLI_NAME}.config`,
+          },
+        ],
+      })
     }
   } catch (error) {
-    logger.error(error, 'config evaluation failed')
+    logger.error({ error }, 'config evaluation failed')
     // ignore
   }
 
-  if (!configFile || configFile.isEmpty) {
+  if (!configFile) {
     logger.error({
       msg: 'Config file not found',
     })

package/src/config/config.ts

@@ -13,6 +13,7 @@ import {
   type PublicClientConfig,
   type WebSocketTransport,
 } from 'viem'
+
 import type { Database } from '../db/client.ts'
 import type { HookRegistry } from '../hooks/registry.ts'
 import { createRpcClients, type RpcClients } from '../rpc/client.ts'

package/src/config/env.ts

@@ -1,9 +1,11 @@
 import dotenv from 'dotenv'
 import { z } from 'zod'
+
 import type { Logger } from '../utils/logger'
 
 dotenv.config({
   path: '.env.local',
+  quiet: true,
 })
 
 export type Env = z.infer<typeof envSchema>
@@ -20,7 +22,7 @@ export function createEnv(logger: Logger) {
 
   if (!parsed.success) {
     throw new Error(
-      `Failed to parse environment variables: \n ${z.flattenError(parsed.error)}`
+      `Failed to parse environment variables: \n ${z.prettifyError(parsed.error)}`
     )
   }
   logger.debug({ env: parsed.data }, 'env parsed')

package/src/db/actions/blocks.ts

@@ -9,6 +9,7 @@ import {
   type PgTable,
 } from 'drizzle-orm/pg-core'
 import type { PublicClient } from 'viem'
+
 import type { FilteredContracts } from '../../config/config.ts'
 import { MAX_QUERY_PARAMS } from '../../contants.ts'
 import { safeGetBlock } from '../../rpc/get-block.ts'
@@ -175,7 +176,7 @@ export async function getBlocksInRange(
     })
   })
 
-  logger.info(
+  logger.trace(
     {
       blocks: blocksByNumber.size,
       missing: missingBlockNumbers.length,

package/src/db/actions/transactions.ts

@@ -1,4 +1,5 @@
 import type { PgAsyncTransaction, PgQueryResultHKT } from 'drizzle-orm/pg-core'
+
 import { MAX_QUERY_PARAMS } from '../../contants.ts'
 import type { EncodedTransaction } from '../../types.ts'
 import { schema } from '../schema/index.ts'

package/src/db/client.ts

@@ -10,6 +10,7 @@ import {
 } from 'drizzle-orm/pglite'
 import type { AnyRelations, EmptyRelations } from 'drizzle-orm/relations'
 import { Pool, type PoolConfig } from 'pg'
+
 import type { DatabaseConfig } from '../config/config.ts'
 import type { Env } from '../config/env.ts'
 import { type relations, schema } from './schema/index.ts'
@@ -39,7 +40,7 @@ export type DatabaseContext<
         $prepared: ReturnType<typeof generatePrepared>
       }
       driver: 'postgres'
-      close: () => Promise<void>
+      stop: () => Promise<void>
     }
   | {
       db: PgliteDatabase<TSchema, TRelations> & {
@@ -47,7 +48,7 @@ export type DatabaseContext<
         $prepared: ReturnType<typeof generatePrepared>
       }
       driver: 'pglite'
-      close: () => Promise<void>
+      stop: () => Promise<void>
     }
 
 /**
@@ -80,6 +81,7 @@ export function createDatabase<
     options = config.options
   }
 
+  // Postgres
   if (driver === 'postgres' && url) {
     const pool = new Pool({
       ...options,
@@ -101,12 +103,13 @@ export function createDatabase<
     return {
       db,
       driver: 'postgres',
-      close: async () => {
+      stop: async () => {
         await pool.end()
       },
     }
   }
 
+  // PGlite
   const client = new PGlite(
     config?.driver === 'pglite' && config.directory
       ? config.directory
@@ -128,7 +131,7 @@ export function createDatabase<
   return {
     db,
     driver: 'pglite',
-    close: async () => {
+    stop: async () => {
       await client.close()
     },
   }

package/src/db/encode.ts

@@ -1,4 +1,5 @@
 import type { Hash } from 'viem'
+
 import type {
   ChainBlock,
   ChainTransaction,

package/src/db/migrate.ts

@@ -2,6 +2,7 @@ import { migrate as migratePostgresJs } from 'drizzle-orm/node-postgres/migrator
 import { getTableConfig, type PgTable } from 'drizzle-orm/pg-core'
 import { IndexedColumn } from 'drizzle-orm/pg-core/columns/common'
 import { migrate as migratePglite } from 'drizzle-orm/pglite/migrator'
+
 import { FOXER_TABLES, PUBLICATION_NAME } from '../contants.ts'
 import type { Logger } from '../utils/logger.ts'
 import { startClock } from '../utils/timer.ts'
@@ -214,8 +215,12 @@ function hasCascadeForeignKeyToBlockNumberTable(
  * @returns True if WAL is enabled, false otherwise
  */
 export async function isWalEnabled(db: Database) {
-  const wal = await db
-    .execute('SHOW WAL_LEVEL')
-    .then((result) => result.rows[0].wal_level)
-  return wal === 'logical'
+  try {
+    const wal = await db
+      .execute('SHOW WAL_LEVEL')
+      .then((result) => result.rows[0].wal_level)
+    return wal === 'logical'
+  } catch (error) {
+    throw new Error('Failed to check if WAL is enabled', { cause: error })
+  }
 }

package/src/db/schema/blocks.ts

@@ -1,4 +1,5 @@
 import { pgTable } from 'drizzle-orm/pg-core'
+
 import { address, bigint, bytea, hash, numeric78 } from '../column-types.ts'
 
 export const blocks = pgTable('blocks', {

package/src/db/schema/transactions.ts

@@ -1,5 +1,6 @@
 import { index, integer, jsonb, pgEnum, pgTable } from 'drizzle-orm/pg-core'
 import type { AccessList } from 'viem'
+
 import { address, bigint, bytea, hash, numeric78 } from '../column-types.ts'
 
 export const transactionTypeEnum = pgEnum('transaction_type', [

package/src/hooks/registry.ts

@@ -1,5 +1,7 @@
 import type { AnyRelations, EmptyRelations } from 'drizzle-orm/relations'
 import type { GetEventArgs, Log } from 'viem'
+
+import type { InternalConfig } from '../config/config'
 import type { Database } from '../db/client'
 import type { EncodedBlockWithTransactions, EncodedTransaction } from '../types'
 import type { Logger } from '../utils/logger'
@@ -105,3 +107,13 @@ export class HookRegistry<
     })
   }
 }
+
+export function createRegistry({
+  config,
+}: {
+  config: InternalConfig
+}): HookRegistry {
+  const registry = new HookRegistry()
+  config.hooks({ registry })
+  return registry
+}

package/src/indexer/backfill.ts

@@ -40,7 +40,7 @@ export async function runBackfill(args: {
   }
 
   const batchSize = config.batchSize
-  logger.debug(
+  logger.info(
     {
       fromBlock: cursor.toString(),
       toBlock: safeHead.toString(),
@@ -54,14 +54,6 @@ export async function runBackfill(args: {
     const toBlock = windowEnd(cursor, batchSize, safeHead)
     const windowContracts = filterContracts(config, cursor, toBlock)
 
-    logger.debug(
-      {
-        batchFromBlock: cursor.toString(),
-        batchToBlock: toBlock.toString(),
-        streamCount: windowContracts.addresses.length,
-      },
-      'processing backfill batch'
-    )
     const batchBlockNumbers: bigint[] = []
     let blockNumber = cursor
     while (blockNumber <= toBlock) {
@@ -104,7 +96,7 @@ export async function runBackfill(args: {
         blockIndex += 1
       }
     })
-    logger.info(
+    logger.debug(
       { duration: endClockBatch() },
       'batch block and events processed'
     )
@@ -118,6 +110,7 @@ export async function runBackfill(args: {
       {
         indexedUpTo: toBlock.toString(),
         duration: batchElapsedMs,
+        contracts: windowContracts.addresses.length,
         throughput: Number(blocksPerSecond.toFixed(2)),
       },
       'backfill batch completed'

package/src/indexer/{runner.ts → create-indexer.ts}

@@ -7,7 +7,7 @@ import { runBackfill } from './backfill.ts'
 import { startLiveSync } from './live.ts'
 import { verifyRecentBlocks } from './reorg.ts'
 
-export async function bootstrapIndexer(options: {
+export async function createIndexer(options: {
   logger: Logger
   db: Database<typeof schema, typeof relations>
   registry: HookRegistry

package/src/indexer/live.ts

@@ -1,5 +1,6 @@
 import PQueue from 'p-queue'
 import type { PublicClient } from 'viem'
+
 import type { InternalConfig } from '../config/config.ts'
 import type { Database } from '../db/client.ts'
 import type { relations, schema } from '../db/schema/index.ts'
@@ -25,7 +26,7 @@ export function startLiveSync(args: {
   const contracts = config.contractsForLive
 
   if (contracts.length === 0) {
-    logger.debug(
+    logger.info(
       'all configured contracts have endBlock set; live sync disabled'
     )
     return { stop: noop }
@@ -44,7 +45,7 @@ export function startLiveSync(args: {
     onBlockNumber: (head) => {
       while (nextBlockToQueue <= head) {
         const blockNumber = nextBlockToQueue
-        pqueue.add(async () => {
+        void pqueue.add(async () => {
           await queueBlock({
             logger,
             blockNumber,
@@ -67,7 +68,7 @@ export function startLiveSync(args: {
     },
   })
 
-  logger.debug(
+  logger.info(
     { startBlock: nextBlockToQueue.toString() },
     'watching latest chain head'
   )

package/src/indexer/process-block.ts

@@ -1,4 +1,5 @@
 import type { AbiEvent, Log, PublicClient } from 'viem'
+
 import type { FilteredContracts, InternalConfig } from '../config/config.ts'
 import { cacheBlockAndTransactions } from '../db/actions/blocks.ts'
 import type { Database } from '../db/client.ts'
@@ -96,7 +97,7 @@ export async function processBlock(args: {
       const contractName = contracts.contractNameByAddress[log.address]
 
       if (!contractName) {
-        logger.trace(
+        logger.debug(
           { address: log.address },
           'contract not found in contract name by address'
         )
@@ -110,7 +111,7 @@ export async function processBlock(args: {
       const transaction = transactionByHash.get(log.transactionHash)
 
       if (!transaction) {
-        logger.trace(
+        logger.debug(
           { transactionHash: log.transactionHash },
 
           'transaction not found in block transaction list'

package/src/indexer/queue-block.ts

@@ -1,5 +1,6 @@
 import type { Logger } from 'pino'
 import type { PublicClient } from 'viem'
+
 import { filterContracts, type InternalConfig } from '../config/config.ts'
 import type { Database } from '../db/client.ts'
 import type { relations, schema } from '../db/schema/index.ts'

package/src/indexer/reorg.ts

@@ -1,4 +1,5 @@
 import type { PublicClient } from 'viem'
+
 import { deleteBlocksFrom } from '../db/actions/blocks.ts'
 import type { Database } from '../db/client.ts'
 import { safeGetBlock } from '../rpc/get-block.ts'

package/src/rpc/client.ts

@@ -1,4 +1,5 @@
 import { createPublicClient, type PublicClient } from 'viem'
+
 import type { ClientConfig } from '../config/config.ts'
 
 export type RpcClients = {

package/src/rpc/get-block.ts

@@ -1,4 +1,5 @@
 import type { Hash, PublicClient } from 'viem'
+
 import type { Database } from '../db/client.ts'
 import {
   encodeBlockWithTransactions,

package/src/rpc/get-logs.ts

@@ -1,4 +1,5 @@
 import type { AbiEvent, Address, Log, PublicClient } from 'viem'
+
 import type { Logger } from '../utils/logger.ts'
 import { startClock } from '../utils/timer.ts'
 
@@ -27,7 +28,7 @@ export async function getLogsInRange(args: {
     byBlock.push(log)
     logsByBlock.set(log.blockNumber, byBlock)
   }
-  logger.info(
+  logger.trace(
     {
       logs: logsByBlock.size,
       duration: endClock(),

package/src/types.ts

@@ -1,5 +1,6 @@
 import type { Simplify } from 'type-fest'
 import type { Block, Transaction } from 'viem'
+
 import type { Schema } from './db/schema/index'
 /**
  * Generic result with error

package/src/utils/logger.ts

@@ -3,6 +3,7 @@ import type { DestinationStream, LevelWithSilent } from 'pino'
 import pino from 'pino'
 import type { Simplify } from 'type-fest'
 import { stringify } from 'viem'
+
 import { formatLogDuration } from './format.ts'
 
 export type LogMode = 'pretty' | 'json'
@@ -47,9 +48,6 @@ export function createLogger({
   } else {
     logger = pino({
       level,
-      serializers: {
-        error: errorSerializer,
-      },
       // Removes "pid" and "hostname" properties from the log.
       base: undefined,
     })

package/src/utils/shutdown.ts

@@ -8,15 +8,16 @@ export function createExit({
   logger: Logger
   stop: () => Promise<void>
 }) {
+  registerUnhandled({ logger })
   asyncExitHook(
     async () => {
       logger.warn('shutting down...')
       await stop()
       logger.info('shutdown complete')
-      process.exit(0)
+      // process.exit(0)
     },
     {
-      wait: 300,
+      wait: 1000,
     }
   )
 }

package/src/utils/types.ts

@@ -1,4 +1,5 @@
 import type { Abi, ExtractAbiEvent, ExtractAbiEventNames } from 'abitype'
+
 import type { UnknownObject } from '../types'
 
 export type EnsureUniqueTuple<

package/template/{package.json → package.json.tpl}

@@ -1,22 +1,24 @@
 {
   "name": "foxer-repo",
   "version": "0.0.0",
-  "description": "",
-  "main": "index.js",
   "private": true,
+  "description": "",
+  "keywords": [],
+  "license": "MIT",
   "workspaces": [
     "apps/*",
     "packages/*"
   ],
+  "main": "index.js",
   "scripts": {
     "test": "echo \"Error: no test specified\" && exit 1",
-    "lint": "pnpm -r --if-present run lint",
-    "build": "pnpm -r --if-present run build",
-    "lint:fix": "biome check --write ."
+   "check": "turbo run check",
+    "build": "turbo run build",
+    "lint": "biome check ."
   },
-  "keywords": [],
-  "license": "MIT",
   "devDependencies": {
-    "@biomejs/biome": "^2.4.5"
+    "@biomejs/biome": "^2.4.7",
+    "@hugomrdias/configs": "^1.1.3",
+    "turbo": "^2.8.17"
   }
 }