@bopen-io/wallet-toolbox 1.7.18

package/src/services/chaintracker/chaintracks/Ingest/WhatsOnChainServices.ts

@@ -0,0 +1,219 @@
+import { Chain } from '../../../../sdk/types'
+import { BlockHeader } from '../../../../sdk/WalletServices.interfaces'
+import { WhatsOnChain, WocChainInfo } from '../../../providers/WhatsOnChain'
+import { ChaintracksFetchApi } from '../Api/ChaintracksFetchApi'
+import { ChaintracksFetch } from '../util/ChaintracksFetch'
+import { HeightRange } from '../util/HeightRange'
+
+/**
+ * return true to ignore error, false to close service connection
+ */
+export type ErrorHandler = (code: number, message: string) => boolean
+export type EnqueueHandler = (header: BlockHeader) => void
+
+export interface WhatsOnChainServicesOptions {
+  /**
+   * Which chain is being tracked: main, test, or stn.
+   */
+  chain: Chain
+  /**
+   * WhatsOnChain.com API Key
+   * https://docs.taal.com/introduction/get-an-api-key
+   * If unknown or empty, maximum request rate is limited.
+   * https://developers.whatsonchain.com/#rate-limits
+   */
+  apiKey?: string
+  /**
+   * Request timeout for GETs to https://api.whatsonchain.com/v1/bsv
+   */
+  timeout: number
+  /**
+   * User-Agent header value for requests to https://api.whatsonchain.com/v1/bsv
+   */
+  userAgent: string
+  /**
+   * Enable WhatsOnChain client cache option.
+   */
+  enableCache: boolean
+  /**
+   * How long chainInfo is considered still valid before updating (msecs).
+   */
+  chainInfoMsecs: number
+}
+
+export class WhatsOnChainServices {
+  static createWhatsOnChainServicesOptions(chain: Chain): WhatsOnChainServicesOptions {
+    const options: WhatsOnChainServicesOptions = {
+      chain,
+      apiKey: '',
+      timeout: 30000,
+      userAgent: 'BabbageWhatsOnChainServices',
+      enableCache: true,
+      chainInfoMsecs: 5000
+    }
+    return options
+  }
+
+  static chainInfo: (WocChainInfo | undefined)[] = []
+  static chainInfoTime: (Date | undefined)[] = []
+  static chainInfoMsecs: number[] = []
+
+  chain: Chain
+  woc: WhatsOnChain
+
+  constructor(public options: WhatsOnChainServicesOptions) {
+    const config = {
+      apiKey: this.options.apiKey,
+      timeout: this.options.timeout,
+      userAgent: this.options.userAgent,
+      enableCache: this.options.enableCache
+    }
+    this.chain = options.chain
+    WhatsOnChainServices.chainInfoMsecs[this.chain] = options.chainInfoMsecs
+    this.woc = new WhatsOnChain(this.chain, config)
+  }
+
+  async getHeaderByHash(hash: string): Promise<BlockHeader | undefined> {
+    const header = await this.woc.getBlockHeaderByHash(hash)
+    return header
+  }
+
+  async getChainInfo(): Promise<WocChainInfo> {
+    const now = new Date()
+    let update = WhatsOnChainServices.chainInfo[this.chain] === undefined
+    if (!update && WhatsOnChainServices.chainInfoTime[this.chain] !== undefined) {
+      const elapsed = now.getTime() - WhatsOnChainServices.chainInfoTime[this.chain].getTime()
+      update = elapsed > WhatsOnChainServices.chainInfoMsecs[this.chain]
+    }
+    if (update) {
+      WhatsOnChainServices.chainInfo[this.chain] = await this.woc.getChainInfo()
+      WhatsOnChainServices.chainInfoTime[this.chain] = now
+    }
+    if (!WhatsOnChainServices.chainInfo[this.chain]) throw new Error('Unexpected failure to update chainInfo.')
+    return WhatsOnChainServices.chainInfo[this.chain]
+  }
+
+  async getChainTipHeight(): Promise<number> {
+    return (await this.getChainInfo()).blocks
+  }
+
+  async getChainTipHash(): Promise<string> {
+    return (await this.getChainInfo()).bestblockhash
+  }
+
+  /**
+   * @param fetch
+   * @returns returns the last 10 block headers including height, size, chainwork...
+   */
+  async getHeaders(fetch?: ChaintracksFetchApi): Promise<WocGetHeadersHeader[]> {
+    fetch ||= new ChaintracksFetch()
+    const headers = await fetch.fetchJson<WocGetHeadersHeader[]>(
+      `https://api.whatsonchain.com/v1/bsv/${this.chain}/block/headers`
+    )
+    return headers
+  }
+
+  async getHeaderByteFileLinks(
+    neededRange: HeightRange,
+    fetch?: ChaintracksFetchApi
+  ): Promise<GetHeaderByteFileLinksResult[]> {
+    fetch ||= new ChaintracksFetch()
+    const files = await fetch.fetchJson<WocGetHeaderByteFileLinks>(
+      `https://api.whatsonchain.com/v1/bsv/${this.chain}/block/headers/resources`
+    )
+    const r: GetHeaderByteFileLinksResult[] = []
+    let range: HeightRange | undefined = undefined
+    for (const link of files.files) {
+      const parsed = parseFileLink(link)
+      if (parsed === undefined) continue // parse error, return empty result
+      if (parsed.range === 'latest') {
+        if (range === undefined) continue // should not happen on valid input
+        const fromHeight = range.maxHeight + 1
+        if (neededRange.maxHeight >= fromHeight) {
+          // We need this range but don't know maxHeight
+          const data = await fetch.download(link)
+          range = new HeightRange(fromHeight, fromHeight + data.length / 80 - 1)
+          if (!neededRange.intersect(range).isEmpty)
+            r.push({ sourceUrl: parsed.sourceUrl, fileName: parsed.fileName, range, data })
+        }
+      } else {
+        range = new HeightRange(parsed.range.fromHeight, parsed.range.toHeight)
+        if (!neededRange.intersect(range).isEmpty)
+          r.push({ sourceUrl: parsed.sourceUrl, fileName: parsed.fileName, range, data: undefined })
+      }
+    }
+    return r
+
+    function parseFileLink(
+      file: string
+    ): { range: { fromHeight: number; toHeight: number } | 'latest'; sourceUrl: string; fileName: string } | undefined {
+      const url = new URL(file)
+      const parts = url.pathname.split('/')
+      const fileName = parts.pop()
+      if (!fileName) return undefined // no file name, invalid link
+      const sourceUrl = `${url.protocol}//${url.hostname}${parts.join('/')}`
+      const bits = fileName.split('_')
+      if (bits.length === 1 && bits[0] === 'latest') {
+        return { range: 'latest', sourceUrl, fileName }
+      }
+      if (bits.length === 3) {
+        const fromHeight = parseInt(bits[0], 10)
+        const toHeight = parseInt(bits[1], 10)
+        if (Number.isInteger(fromHeight) && Number.isInteger(toHeight)) {
+          return { range: { fromHeight, toHeight }, sourceUrl, fileName }
+        }
+      }
+      return undefined
+    }
+  }
+}
+
+export interface WocGetHeaderByteFileLinks {
+  files: string[]
+}
+
+export interface WocGetHeadersHeader {
+  hash: string
+  confirmations: number
+  size: number
+  height: number
+  version: number
+  versionHex: string
+  merkleroot: string
+  time: number
+  mediantime: number
+  nonce: number
+  bits: string
+  difficulty: number
+  chainwork: string
+  previousblockhash: string
+  nextblockhash: string
+  nTx: number
+  num_tx: number
+}
+
+export function wocGetHeadersHeaderToBlockHeader(h: WocGetHeadersHeader): BlockHeader {
+  const bits: number = typeof h.bits === 'string' ? parseInt(h.bits, 16) : h.bits
+  if (!h.previousblockhash) {
+    h.previousblockhash = '0000000000000000000000000000000000000000000000000000000000000000' // genesis
+  }
+  const bh: BlockHeader = {
+    height: h.height,
+    hash: h.hash,
+    version: h.version,
+    previousHash: h.previousblockhash,
+    merkleRoot: h.merkleroot,
+    time: h.time,
+    bits,
+    nonce: h.nonce
+  }
+
+  return bh
+}
+
+export interface GetHeaderByteFileLinksResult {
+  sourceUrl: string
+  fileName: string
+  range: HeightRange
+  data: Uint8Array | undefined
+}

package/src/services/chaintracker/chaintracks/Ingest/__tests/BulkIngestorCDNBabbage.test.ts

@@ -0,0 +1,54 @@
+import { Knex, knex as makeKnex } from 'knex'
+import { Chain } from '../../../../../sdk'
+import { BulkIngestorCDNBabbage } from '../BulkIngestorCDNBabbage'
+import { ChaintracksFetch } from '../../util/ChaintracksFetch'
+import { ChaintracksFs } from '../../util/ChaintracksFs'
+import { HeightRange } from '../../util/HeightRange'
+import { ChaintracksStorageKnex } from '../../Storage/ChaintracksStorageKnex'
+import { BulkFilesReaderStorage } from '../../util/BulkFilesReader'
+
+const rootFolder = './src/services/chaintracker/chaintracks/__tests/data'
+const fs = ChaintracksFs
+const fetch = new ChaintracksFetch()
+
+describe('BulkIngestorCDNBabbage tests', () => {
+  jest.setTimeout(99999999)
+
+  test('0 mainNet', async () => {
+    const { cdn, r } = await testUpdateLocalCache('main', '0')
+    expect(cdn.availableBulkFiles?.files.length).toBeGreaterThan(8)
+    expect(r.liveHeaders.length).toBe(0)
+    expect(r.reader.range.minHeight).toBe(0)
+    expect(r.reader.range.maxHeight).toBeGreaterThan(800000)
+  })
+
+  test('1 testNet', async () => {
+    const { cdn, r } = await testUpdateLocalCache('test', '1')
+    expect(cdn.availableBulkFiles?.files.length).toBeGreaterThan(15)
+    expect(r.liveHeaders.length).toBe(0)
+    expect(r.reader.range.minHeight).toBe(0)
+    expect(r.reader.range.maxHeight).toBeGreaterThan(1500000)
+  })
+})
+
+async function testUpdateLocalCache(chain: Chain, test: string) {
+  const bulkCDNOptions = BulkIngestorCDNBabbage.createBulkIngestorCDNBabbageOptions(chain, fetch)
+
+  const cdn = new BulkIngestorCDNBabbage(bulkCDNOptions)
+
+  const localSqlite: Knex.Config = {
+    client: 'sqlite3',
+    connection: { filename: fs.pathJoin(rootFolder, `BulkIngestorCDNBabbage.test_${test}.sqlite`) },
+    useNullAsDefault: true
+  }
+  const knexOptions = ChaintracksStorageKnex.createStorageKnexOptions(chain, makeKnex(localSqlite))
+  const storage = new ChaintracksStorageKnex(knexOptions)
+  const before = await storage.getAvailableHeightRanges()
+  await cdn.setStorage(storage, console.log)
+
+  const range = new HeightRange(0, 9900000)
+  const liveHeaders = await cdn.fetchHeaders(before, range, range, [])
+  const reader = await BulkFilesReaderStorage.fromStorage(storage, fetch, range)
+  await storage.knex.destroy()
+  return { cdn, r: { reader, liveHeaders } }
+}

package/src/services/chaintracker/chaintracks/Ingest/__tests/LiveIngestorWhatsOnChainPoll.test.ts

@@ -0,0 +1,33 @@
+import { wait } from '../../../../../utility/utilityHelpers'
+import { BlockHeader } from '../../Api/BlockHeaderApi'
+import { LiveIngestorWhatsOnChainPoll } from '../LiveIngestorWhatsOnChainPoll'
+
+describe('LiveIngestorWhatsOnChainPoll tests', () => {
+  jest.setTimeout(99999999)
+
+  test('0 listen for first new header', async () => {
+    const liveHeaders: BlockHeader[] = []
+    const options = LiveIngestorWhatsOnChainPoll.createLiveIngestorWhatsOnChainOptions('main')
+    const ingestor = new LiveIngestorWhatsOnChainPoll(options)
+    const p = ingestor.startListening(liveHeaders)
+    let log = ''
+    let count = 0
+    for (;;) {
+      const h = liveHeaders.shift()
+      if (h) {
+        log += `${h.height} ${h.hash}\n`
+        count++
+      } else {
+        if (log) {
+          console.log(`LiveIngestorWhatsOnChain received ${count} headers:\n${log}`)
+          log = ''
+          break
+        }
+        //if (count >= 11) break
+        await wait(100)
+      }
+    }
+    ingestor.stopListening()
+    await p
+  })
+})

package/src/services/chaintracker/chaintracks/Ingest/__tests/WhatsOnChainServices.test.ts

@@ -0,0 +1,124 @@
+import { wait } from '../../../../../utility/utilityHelpers'
+import { BlockHeader } from '../../Api/BlockHeaderApi'
+import { deserializeBaseBlockHeader, deserializeBlockHeader } from '../../util/blockHeaderUtilities'
+import { ChaintracksFetch } from '../../util/ChaintracksFetch'
+import { ChaintracksFs } from '../../util/ChaintracksFs'
+import { EnqueueHandler, ErrorHandler, WhatsOnChainServices, WocGetHeadersHeader } from '../WhatsOnChainServices'
+import { StopListenerToken, WocHeadersBulkListener, WocHeadersLiveListener } from '../WhatsOnChainIngestorWs'
+import { Chain } from '../../../../../sdk'
+import { URL } from 'url'
+import { HeightRange } from '../../util/HeightRange'
+
+describe('WhatsOnChainServices tests', () => {
+  jest.setTimeout(999999999)
+
+  const chain: Chain = 'main'
+  const options = WhatsOnChainServices.createWhatsOnChainServicesOptions(chain)
+  const woc = new WhatsOnChainServices(options)
+
+  test('getHeaderByHash', async () => {
+    const header = await woc.getHeaderByHash('000000000000000001b3e99847d57ff3e0bfc4222cea5c29f10bf24387a250a2')
+    expect(header?.height === 781348).toBe(true)
+  })
+
+  test('getChainTipHeight', async () => {
+    const height = await woc.getChainTipHeight()
+    expect(height > 600000).toBe(true)
+  })
+
+  const stopOldListenersToken: StopListenerToken = { stop: undefined }
+  function stopOldListener() {
+    stopOldListenersToken.stop?.()
+  }
+
+  test.skip('0 listenForOldBlockHeaders', async () => {
+    // The service this depends on appears to be deprecated...
+    const height = await woc.getChainTipHeight()
+    expect(height > 600000).toBe(true)
+
+    const headersOld: BlockHeader[] = []
+    const errorsOld: { code: number; message: string }[] = []
+    const okOld = await WocHeadersBulkListener(
+      height - 4,
+      height,
+      h => headersOld.push(h),
+      (code, message) => {
+        errorsOld.push({ code, message })
+        return true
+      },
+      stopOldListenersToken,
+      chain
+    )
+    expect(okOld).toBe(true)
+    expect(errorsOld.length).toBe(0)
+    expect(headersOld.length >= 4).toBe(true)
+  })
+
+  const stopNewListenersToken: StopListenerToken = { stop: undefined }
+
+  test.skip('1 listenForNewBlockHeaders', async () => {
+    // The service this depends on appears to be deprecated...
+    const height = await woc.getChainTipHeight()
+    expect(height > 600000).toBe(true)
+
+    // Comment out this line to just wait for next new header...
+    //setTimeout(() => woc.stopNewListener(), 5000)
+    const headersNew: BlockHeader[] = []
+    const errorsNew: { code: number; message: string }[] = []
+    const eh: EnqueueHandler = h => {
+      headersNew.push(h)
+      if (headersNew.length >= 1) stopNewListenersToken.stop?.()
+    }
+    const errh: ErrorHandler = (code, message) => {
+      errorsNew.push({ code, message })
+      return true
+    }
+    const okNew = await WocHeadersLiveListener(eh, errh, stopNewListenersToken, chain, console.log.bind(console))
+    if (errorsNew.length > 0) console.log(JSON.stringify(errorsNew))
+    expect(errorsNew.length).toBe(0)
+    expect(okNew).toBe(true)
+    expect(headersNew.length >= 0).toBe(true)
+  })
+
+  test('2 get latest header bytes', async () => {
+    const fetch = new ChaintracksFetch()
+
+    //for (;;) {
+    const bytes = await fetch.download(`https://api.whatsonchain.com/v1/bsv/main/block/headers/latest`)
+    console.log(`headers: ${bytes.length / 80}`)
+    const latest = await fetch.download(`https://api.whatsonchain.com/v1/bsv/main/block/headers/latest?count=1`)
+    const bh = deserializeBlockHeader(latest, 0, 0)
+    console.log(`latest hash: ${bh.hash} at ${new Date().toISOString()}`)
+    //  await wait(60 * 1000)
+    //}
+  })
+
+  test('3 get headers', async () => {
+    const fetch = new ChaintracksFetch()
+
+    //for (;;) {
+    const headers = await fetch.fetchJson<WocGetHeadersHeader[]>(
+      `https://api.whatsonchain.com/v1/bsv/main/block/headers`
+    )
+    let log = ''
+    for (const h of headers) {
+      log += `${h.height} ${h.hash} ${h.confirmations} ${h.nTx}\n`
+    }
+    console.log(`${new Date().toISOString()}\n${log}`)
+    //await wait(60 * 1000)
+    //}
+  })
+
+  test('4 get header byte file links', async () => {
+    const fetch = new ChaintracksFetch()
+    const woc = new WhatsOnChainServices(WhatsOnChainServices.createWhatsOnChainServicesOptions('main'))
+    const files = await woc.getHeaderByteFileLinks(new HeightRange(907123, 911000))
+    expect(files.length).toBe(3)
+    expect(files[0].range.minHeight).toBe(906001)
+    expect(files[0].range.maxHeight).toBe(908000)
+    expect(files[1].range.minHeight).toBe(908001)
+    expect(files[1].range.maxHeight).toBe(910000)
+    expect(files[2].range.minHeight).toBe(910001)
+    expect(files[2].range.maxHeight).toBeGreaterThan(910001)
+  })
+})

package/src/services/chaintracker/chaintracks/Storage/BulkStorageBase.ts

@@ -0,0 +1,92 @@
+// /* eslint-disable @typescript-eslint/no-empty-function */
+/* eslint-disable @typescript-eslint/no-unused-vars */
+import { BulkStorageApi, BulkStorageBaseOptions } from '../Api/BulkStorageApi'
+
+import { ChaintracksStorageBase } from './ChaintracksStorageBase'
+
+import { HeightRange } from '../util/HeightRange'
+import { BulkFilesReader } from '../util/BulkFilesReader'
+import { BulkHeaderFileInfo } from '../util/BulkHeaderFile'
+import { BulkHeaderFilesInfo } from '../util/BulkHeaderFile'
+
+import { addWork, convertBitsToWork, deserializeBlockHeaders, genesisBuffer } from '../util/blockHeaderUtilities'
+import { Chain } from '../../../../sdk/types'
+import { BlockHeader, LiveBlockHeader } from '../Api/BlockHeaderApi'
+import { ChaintracksFsApi } from '../Api/ChaintracksFsApi'
+import { Utils } from '@bsv/sdk'
+import { asUint8Array } from '../../../../utility/utilityHelpers.noBuffer'
+
+export abstract class BulkStorageBase implements BulkStorageApi {
+  static createBulkStorageBaseOptions(chain: Chain, fs: ChaintracksFsApi): BulkStorageBaseOptions {
+    const options: BulkStorageBaseOptions = {
+      chain,
+      fs
+    }
+    return options
+  }
+
+  chain: Chain
+  fs: ChaintracksFsApi
+  log: (...args: any[]) => void = () => {}
+
+  constructor(options: BulkStorageBaseOptions) {
+    this.chain = options.chain
+    this.fs = options.fs
+  }
+
+  async shutdown(): Promise<void> {}
+
+  abstract appendHeaders(minHeight: number, count: number, newBulkHeaders: Uint8Array): Promise<void>
+  abstract getMaxHeight(): Promise<number>
+  abstract headersToBuffer(height: number, count: number): Promise<Uint8Array>
+  abstract findHeaderForHeightOrUndefined(height: number): Promise<BlockHeader | undefined>
+
+  async findHeaderForHeight(height: number): Promise<BlockHeader> {
+    const header = await this.findHeaderForHeightOrUndefined(height)
+    if (!header) throw new Error(`No header found for height ${height}`)
+    return header
+  }
+
+  async getHeightRange(): Promise<HeightRange> {
+    return new HeightRange(0, await this.getMaxHeight())
+  }
+
+  async setStorage(storage: ChaintracksStorageBase, log: (...args: any[]) => void): Promise<void> {}
+
+  async exportBulkHeaders(rootFolder: string, jsonFilename: string, maxPerFile: number): Promise<void> {
+    const info: BulkHeaderFilesInfo = {
+      rootFolder: rootFolder,
+      jsonFilename: jsonFilename,
+      files: [],
+      headersPerFile: maxPerFile
+    }
+    const maxHeight = await this.getMaxHeight()
+    const baseFilename = jsonFilename.slice(0, -5) // remove ".json"
+    let prevHash = '00'.repeat(32)
+    let prevChainWork = '00'.repeat(32)
+    for (let height = 0; height <= maxHeight; height += maxPerFile) {
+      const count = Math.min(maxPerFile, maxHeight - height + 1)
+      let file: BulkHeaderFileInfo = {
+        fileName: `${baseFilename}_${info.files.length}.headers`,
+        firstHeight: height,
+        prevHash: prevHash,
+        prevChainWork: prevChainWork,
+        count: count,
+        lastHash: null,
+        fileHash: null,
+        lastChainWork: ''
+      }
+      const buffer = await this.headersToBuffer(height, count)
+      await this.fs.writeFile(this.fs.pathJoin(rootFolder, file.fileName), buffer)
+      /*
+      file = await BulkFilesReader.validateHeaderFile(this.fs, rootFolder, file)
+      if (!file.lastHash) throw new Error('Unexpected result.')
+      prevHash = file.lastHash
+      prevChainWork = file.lastChainWork
+      info.files.push(file)
+      */
+    }
+    const bytes = asUint8Array(JSON.stringify(info), 'utf8')
+    await this.fs.writeFile(this.fs.pathJoin(rootFolder, jsonFilename), bytes)
+  }
+}

package/src/services/chaintracker/chaintracks/Storage/ChaintracksKnexMigrations.ts

@@ -0,0 +1,104 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
+import { Knex } from 'knex'
+import { Chain } from '../../../../sdk'
+
+interface Migration {
+  up: (knex: Knex) => PromiseLike<any>
+  down?: (knex: Knex) => PromiseLike<any>
+}
+
+interface MigrationSource<TMigrationSpec> {
+  getMigrations(loadExtensions: readonly string[]): Promise<TMigrationSpec[]>
+  getMigrationName(migration: TMigrationSpec): string
+  getMigration(migration: TMigrationSpec): Promise<Migration>
+}
+
+export class ChaintracksKnexMigrations implements MigrationSource<string> {
+  migrations: Record<string, Migration> = {}
+
+  constructor(public chain: Chain) {
+    this.migrations = this.setupMigrations()
+  }
+
+  async getMigrations(): Promise<string[]> {
+    return Object.keys(this.migrations).sort()
+  }
+
+  getMigrationName(migration: string) {
+    return migration
+  }
+
+  async getMigration(migration: string): Promise<Migration> {
+    return this.migrations[migration]
+  }
+
+  async getLatestMigration(): Promise<string> {
+    const ms = await this.getMigrations()
+    return ms[ms.length - 1]
+  }
+
+  static async latestMigration(): Promise<string> {
+    const km = new ChaintracksKnexMigrations('test')
+    return await km.getLatestMigration()
+  }
+
+  setupMigrations(): Record<string, Migration> {
+    const migrations: Record<string, Migration> = {}
+
+    const liveHeadersTableName = `live_headers`
+    const bulkFilesTableName = `bulk_files`
+
+    migrations['2025-06-28-001 initial migration'] = {
+      async up(knex) {
+        await knex.schema.createTable(liveHeadersTableName, table => {
+          table.increments('headerId')
+          table.integer('previousHeaderId').unsigned().references('headerId').inTable(liveHeadersTableName)
+          table.binary('previousHash', 32)
+          table.integer('height').unsigned().notNullable
+          table.boolean('isActive').notNullable
+          table.boolean('isChainTip').notNullable
+          table.binary('hash', 32).notNullable
+          table.binary('chainWork', 32).notNullable
+          table.integer('version').unsigned().notNullable
+          table.binary('merkleRoot', 32).notNullable
+          table.integer('time').unsigned().notNullable
+          table.integer('bits').unsigned().notNullable
+          table.integer('nonce').unsigned().notNullable
+
+          table.unique(['hash'])
+          table.index(['previousHeaderId'])
+          table.index(['height'])
+          table.index(['previousHash'])
+          table.index(['merkleRoot'])
+          table.index(['isChainTip'])
+          table.index(['isActive'])
+          table.index(['isActive', 'isChainTip'])
+        })
+
+        await knex.schema.createTable(bulkFilesTableName, table => {
+          table.increments('fileId')
+          table.string('chain').notNullable()
+          table.string('fileName').notNullable()
+          table.integer('firstHeight').unsigned().notNullable()
+          table.integer('count').unsigned().notNullable()
+          table.string('prevHash', 64).notNullable() // hex encoded
+          table.string('lastHash', 64).notNullable() // hex encoded
+          table.string('prevChainWork', 64).notNullable() // hex encoded
+          table.string('lastChainWork', 64).notNullable() // hex encoded
+          table.string('fileHash').notNullable() // base64 encoded
+          table.boolean('validated').defaultTo(false).notNullable()
+          table.string('sourceUrl').nullable()
+          table.binary('data', 32000000).nullable() // 32MB max size
+
+          table.index(['firstHeight', 'chain'])
+        })
+      },
+      async down(knex) {
+        await knex.schema.dropTable(liveHeadersTableName)
+        await knex.schema.dropTable(bulkFilesTableName)
+      }
+    }
+
+    return migrations
+  }
+}