@bsv/wallet-toolbox 1.6.23 → 1.6.25

package/src/monitor/__test/MonitorDaemon.man.test.ts

@@ -1,17 +1,45 @@
-import * as dotenv from 'dotenv'
 import { MonitorDaemon } from '../MonitorDaemon'
-dotenv.config()
-
-const cloudMySQLConnection = process.env.TEST_CLOUD_MYSQL_CONNECTION || '{}'
+import { Chain } from '../../sdk/types'
+import { Services } from '../../services/Services'
+import { _tu } from '../../../test/utils/TestUtilsWalletStorage'
+import { createDefaultNoDbChaintracksOptions } from '../../services/chaintracker/chaintracks/createDefaultNoDbChaintracksOptions'
+import { Chaintracks } from '../../services/chaintracker/chaintracks/Chaintracks'
 
 describe('MonitorDaemon tests', () => {
   jest.setTimeout(99999999)
 
-  test('0', async () => {
+  test('0 mainnet', async () => {
+    await test0Body('main')
+  })
+
+  test('0a testnet', async () => {
+    await test0Body('test')
+  })
+
+  async function test0Body(chain: Chain) {
+    const env = _tu.getEnv(chain)
+
+    const servicesOptions = Services.createDefaultOptions(chain)
+    if (env.taalApiKey) {
+      servicesOptions.taalApiKey = env.taalApiKey
+      servicesOptions.arcConfig.apiKey = env.taalApiKey
+    }
+    if (env.whatsonchainApiKey) servicesOptions.whatsOnChainApiKey = env.whatsonchainApiKey
+    if (env.bitailsApiKey) servicesOptions.bitailsApiKey = env.bitailsApiKey
+
+    const u = undefined
+    const maxRetained = 32
+    const chaintracksOptions = createDefaultNoDbChaintracksOptions(chain, env.whatsonchainApiKey, u, maxRetained)
+    const chaintracks = new Chaintracks(chaintracksOptions)
+    servicesOptions.chaintracks = chaintracks
+
     const d = new MonitorDaemon({
       chain: 'test',
-      mySQLConnection: cloudMySQLConnection
+      mySQLConnection: env.cloudMySQLConnection,
+      servicesOptions,
+      chaintracks
     })
+
     await d.runDaemon()
-  })
+  }
 })

package/src/monitor/tasks/TaskCheckNoSends.ts

@@ -26,7 +26,7 @@ export class TaskCheckNoSends extends WalletMonitorTask {
 
   constructor(
     monitor: Monitor,
-    public triggerMsecs = monitor.oneDay * 1
+    public triggerMsecs = Monitor.oneDay * 1
   ) {
     super(monitor, TaskCheckNoSends.taskName)
   }

package/src/monitor/tasks/TaskClock.ts

@@ -7,7 +7,7 @@ export class TaskClock extends WalletMonitorTask {
 
   constructor(
     monitor: Monitor,
-    public triggerMsecs = 1 * monitor.oneSecond
+    public triggerMsecs = 1 * Monitor.oneSecond
   ) {
     super(monitor, TaskClock.taskName)
     this.nextMinute = this.getNextMinute()
@@ -28,6 +28,6 @@ export class TaskClock extends WalletMonitorTask {
   }
 
   getNextMinute(): number {
-    return Math.ceil(Date.now() / this.monitor.oneMinute) * this.monitor.oneMinute
+    return Math.ceil(Date.now() / Monitor.oneMinute) * Monitor.oneMinute
   }
 }

package/src/monitor/tasks/TaskMonitorCallHistory.ts

@@ -7,7 +7,7 @@ export class TaskMonitorCallHistory extends WalletMonitorTask {
 
   constructor(
     monitor: Monitor,
-    public triggerMsecs = monitor.oneMinute * 12
+    public triggerMsecs = Monitor.oneMinute * 12
   ) {
     super(monitor, TaskMonitorCallHistory.taskName)
   }

package/src/monitor/tasks/TaskNewHeader.ts

@@ -29,7 +29,7 @@ export class TaskNewHeader extends WalletMonitorTask {
 
   constructor(
     monitor: Monitor,
-    public triggerMsecs = 1 * monitor.oneMinute
+    public triggerMsecs = 1 * Monitor.oneMinute
   ) {
     super(monitor, TaskNewHeader.taskName)
   }

package/src/monitor/tasks/TaskReorg.ts

@@ -0,0 +1,139 @@
+import { TableProvenTx } from '../../index.client'
+import { BlockHeader } from '../../services/chaintracker/chaintracks/Api/BlockHeaderApi'
+import { DeactivedHeader, Monitor } from '../Monitor'
+import { WalletMonitorTask } from './WalletMonitorTask'
+
+/**
+ * Check the `monitor.deactivatedHeaders` for any headers that have been deactivated.
+ *
+ * When headers are found, review matching ProvenTx records and update proof data as appropriate.
+ *
+ * New deactivated headers are pushed onto the `deactivatedHeaders` array.
+ * They must be shifted out as they are processed.
+ *
+ * The current implementation ages deactivation notifications by 10 minutes with each retry.
+ * If a successful proof update confirms original proof data after 3 retries, the original is retained.
+ *
+ * In normal operation there should never be any work for this task to perform.
+ * The most common result is that there are no matching proven_txs records because
+ * generating new proven_txs records intentionally lags new block generation to
+ * minimize this disruption.
+ *
+ * It is very disruptive to update a proven_txs record because:
+ * - Sync'ed storage is impacted.
+ * - Generated beefs are impacted.
+ * - Updated proof data may be unavailable at the time a reorg is first reported.
+ *
+ * Instead of reorg notification derived from new header notification, reorg repair to
+ * the proven_txs table is more effectively driven by noticing that a beef generated for a new
+ * createAction fails to verify against the chaintracker.
+ *
+ * An alternate approach to processing these events is to revert the proven_txs record to a proven_tx_reqs record.
+ * Pros:
+ * - The same multiple attempt logic that already exists is reused.
+ * - Failing to obtain a new proof already has transaction failure handling in place.
+ * - Generated beefs automatically become one generation deeper, potentially allowing transaction outputs to be spent.
+ * Cons:
+ * - Transactions must revert to un-proven / un-mined.
+ */
+export class TaskReorg extends WalletMonitorTask {
+  static taskName = 'Reorg'
+
+  process: DeactivedHeader[] = []
+
+  constructor(
+    monitor: Monitor,
+    public agedMsecs = Monitor.oneMinute * 10,
+    public maxRetries = 3
+  ) {
+    super(monitor, TaskReorg.taskName)
+  }
+
+  trigger(nowMsecsSinceEpoch: number): { run: boolean } {
+    const cutoff = nowMsecsSinceEpoch - this.agedMsecs
+    const q = this.monitor.deactivatedHeaders
+    while (q.length > 0 && cutoff > q[0].whenMsecs) {
+      // Prepare to process deactivated headers that have aged sufficiently (agedMsecs)
+      const header = q.shift()!
+      this.process.push(header)
+    }
+    return {
+      run: this.process.length > 0
+    }
+  }
+
+  async runTask(): Promise<string> {
+    let log = ''
+
+    for (;;) {
+      const header = this.process.shift()
+      if (!header) break
+
+      let ptxs: TableProvenTx[] = []
+
+      await this.storage.runAsStorageProvider(async sp => {
+        // Lookup all the proven_txs records matching the deactivated headers
+        ptxs = await sp.findProvenTxs({ partial: { blockHash: header.header.hash } })
+      })
+
+      log += `  block ${header.header.hash} orphaned with ${ptxs.length} impacted transactions\n`
+
+      let retry = false
+      for (const ptx of ptxs) {
+        const mpr = await this.monitor.services.getMerklePath(ptx.txid)
+        if (mpr.merklePath && mpr.header) {
+          const mp = mpr.merklePath
+          const h = mpr.header
+          const leaf = mp.path[0].find(leaf => leaf.txid === true && leaf.hash === ptx.txid)
+          if (leaf) {
+            const update: Partial<TableProvenTx> = {
+              height: mp.blockHeight,
+              index: leaf.offset,
+              merklePath: mp.toBinary(),
+              merkleRoot: h.merkleRoot,
+              blockHash: h.hash
+            }
+            if (update.blockHash === ptx.blockHash) {
+              log += `    txid ${ptx.txid} merkle path update still based on deactivated header ${ptx.blockHash}\n`
+              if (header.tries + 1 >= this.maxRetries) {
+                log += `      maximum retries ${this.maxRetries} exceeded\n`
+              } else {
+                retry = true
+              }
+            } else {
+              // Verify the new proof's validity.
+              const merkleRoot = mp.computeRoot(ptx.txid)
+              const chaintracker = await this.monitor.services.getChainTracker()
+              const isValid = await chaintracker.isValidRootForHeight(merkleRoot, update.height!)
+              const logUpdate = `      height ${ptx.height} ${ptx.height === update.height ? 'unchanged' : `-> ${update.height}`}\n`
+              log += `      blockHash ${ptx.blockHash} -> ${update.blockHash}\n`
+              log += `      merkleRoot ${ptx.merkleRoot} -> ${update.merkleRoot}\n`
+              log += `      index ${ptx.index} -> ${update.index}\n`
+              if (!isValid) {
+                log +=
+                  `    txid ${ptx.txid} chaintracker fails to confirm updated merkle path update invalid\n` + logUpdate
+              } else {
+                await this.storage.runAsStorageProvider(async sp => {
+                  await sp.updateProvenTx(ptx.provenTxId, update)
+                })
+                log += `    txid ${ptx.txid} proof data updated\n` + logUpdate
+              }
+            }
+          } else {
+            log += `    txid ${ptx.txid} merkle path update doesn't include txid\n`
+            retry = true
+          }
+        } else {
+          log += `    txid ${ptx.txid} merkle path update unavailable\n`
+          retry = true
+        }
+      }
+      if (retry) {
+        log += `    retrying...\n`
+        this.monitor.deactivatedHeaders.push({ header: header.header, whenMsecs: Date.now(), tries: header.tries + 1 })
+      }
+    }
+
+    return log
+  }
+}

package/src/monitor/tasks/TaskSendWaiting.ts

@@ -16,9 +16,9 @@ export class TaskSendWaiting extends WalletMonitorTask {
 
   constructor(
     monitor: Monitor,
-    public triggerMsecs = monitor.oneSecond * 8,
-    public agedMsecs = monitor.oneSecond * 7,
-    public sendingMsecs = monitor.oneMinute * 5
+    public triggerMsecs = Monitor.oneSecond * 8,
+    public agedMsecs = Monitor.oneSecond * 7,
+    public sendingMsecs = Monitor.oneMinute * 5
   ) {
     super(monitor, TaskSendWaiting.taskName)
   }

package/src/monitor/tasks/TaskUnFail.ts

@@ -25,7 +25,7 @@ export class TaskUnFail extends WalletMonitorTask {
 
   constructor(
     monitor: Monitor,
-    public triggerMsecs = monitor.oneMinute * 10
+    public triggerMsecs = Monitor.oneMinute * 10
   ) {
     super(monitor, TaskUnFail.taskName)
   }

package/src/services/chaintracker/chaintracks/Api/ChaintracksClientApi.ts

@@ -10,7 +10,12 @@ export type HeaderListener = (header: BlockHeader) => void
 /**
  * @public
  */
-export type ReorgListener = (depth: number, oldTip: BlockHeader, newTip: BlockHeader) => void
+export type ReorgListener = (
+  depth: number,
+  oldTip: BlockHeader,
+  newTip: BlockHeader,
+  deactivatedHeaders?: BlockHeader[]
+) => void
 
 /**
  * @public

package/src/services/chaintracker/chaintracks/Api/ChaintracksStorageApi.ts

@@ -256,6 +256,10 @@ export type InsertHeaderResult = {
    * If `added` is true, this header was the active chain tip before the insert. It may or may not still be the active chain tip after the insert.
    */
   priorTip: LiveBlockHeader | undefined
+  /**
+   * If a reorg has occurred, these headers where active and are now deactivated.
+   */
+  deactivatedHeaders: LiveBlockHeader[]
   /**
    * header's previousHash was not found in database
    */

package/src/services/chaintracker/chaintracks/Chaintracks.ts

@@ -416,7 +416,9 @@ export class Chaintracks implements ChaintracksManagementApi {
           const reorgListener = this.callbacks.reorg[id]
           if (reorgListener) {
             try {
-              reorgListener(ihr.reorgDepth, ihr.priorTip, header)
+              const priorTip: BlockHeader = { ...ihr.priorTip }
+              const deactivated: BlockHeader[] = ihr.deactivatedHeaders.map(lbh => ({ ...lbh }))
+              reorgListener(ihr.reorgDepth, priorTip, header, deactivated)
             } catch {
               /* ignore all errors thrown */
             }

package/src/services/chaintracker/chaintracks/ChaintracksService.ts

@@ -4,7 +4,7 @@ import { IncomingMessage, Server, ServerResponse } from 'http'
 import express, { Request, Response } from 'express'
 import bodyParser from 'body-parser'
 import { Chain } from '../../../sdk/types'
-import { createNoDbChaintracksOptions } from './createDefaultNoDbChaintracksOptions'
+import { createDefaultNoDbChaintracksOptions } from './createDefaultNoDbChaintracksOptions'
 import { Services } from '../../Services'
 import { FiatExchangeRates, WERR_INVALID_PARAMETER } from '../../../sdk'
 import { ChaintracksInfoApi } from './Api/ChaintracksClientApi'
@@ -45,7 +45,7 @@ export class ChaintracksService {
     this.options = { ...options }
     this.port = options.port
     this.chain = options.chain
-    this.chaintracks = options.chaintracks || new Chaintracks(createNoDbChaintracksOptions(this.chain))
+    this.chaintracks = options.chaintracks || new Chaintracks(createDefaultNoDbChaintracksOptions(this.chain))
     this.services = options.services || new Services(this.chain)
     // Prevent recursion...
     this.services.updateFiatExchangeRateServices.remove('ChaintracksService')

package/src/services/chaintracker/chaintracks/Storage/ChaintracksStorageIdb.ts

@@ -263,7 +263,8 @@ export class ChaintracksStorageIdb extends ChaintracksStorageBase implements Cha
       isActiveTip: false,
       reorgDepth: 0,
       priorTip: undefined,
-      noTip: false
+      noTip: false,
+      deactivatedHeaders: []
     }
 
     // Check for duplicate
@@ -359,8 +360,10 @@ export class ChaintracksStorageIdb extends ChaintracksStorageBase implements Cha
       }
 
       if (activeAncestor.headerId !== oneBack.headerId) {
+        // Deactivate reorg'ed headers
         let headerToDeactivate = this.repairStoredLiveHeader(await activeTipIndex.get([1, 1]))!
         while (headerToDeactivate && headerToDeactivate.headerId !== activeAncestor.headerId) {
+          r.deactivatedHeaders.push(headerToDeactivate)
           await store.put(this.prepareStoredLiveHeader({ ...headerToDeactivate, isActive: false }))
           headerToDeactivate = this.repairStoredLiveHeader(await store.get(headerToDeactivate.previousHeaderId!))!
         }

package/src/services/chaintracker/chaintracks/Storage/ChaintracksStorageKnex.ts

@@ -4,25 +4,15 @@ import { InsertHeaderResult, ChaintracksStorageBaseOptions } from '../Api/Chaint
 import { ChaintracksStorageBase } from './ChaintracksStorageBase'
 import { LiveBlockHeader } from '../Api/BlockHeaderApi'
 import { BlockHeader } from '../../../../sdk/WalletServices.interfaces'
-import {
-  addWork,
-  convertBitsToWork,
-  isMoreWork,
-  serializeBaseBlockHeader,
-  serializeBaseBlockHeaders
-} from '../util/blockHeaderUtilities'
+import { addWork, convertBitsToWork, isMoreWork } from '../util/blockHeaderUtilities'
 import { verifyOneOrNone } from '../../../../utility/utilityHelpers'
 import { DBType } from '../../../../storage/StorageReader'
 import { BulkHeaderFileInfo } from '../util/BulkHeaderFile'
 import { HeightRange } from '../util/HeightRange'
-import { BulkFilesReaderStorage } from '../util/BulkFilesReader'
-import { ChaintracksFetch } from '../util/ChaintracksFetch'
 import { ChaintracksStorageBulkFileApi } from '../Api/ChaintracksStorageApi'
 import { Chain } from '../../../../sdk/types'
 import { WERR_INVALID_OPERATION, WERR_INVALID_PARAMETER } from '../../../../sdk/WERR_errors'
 import { determineDBType } from '../../../../storage/schema/KnexMigrations'
-import { BulkFileDataReader } from '../index.client'
-import { asArray } from '../../../../utility/utilityHelpers.noBuffer'
 
 export interface ChaintracksStorageKnexOptions extends ChaintracksStorageBaseOptions {
   /**
@@ -231,7 +221,8 @@ export class ChaintracksStorageKnex extends ChaintracksStorageBase implements Ch
       isActiveTip: false,
       reorgDepth: 0,
       priorTip: undefined,
-      noTip: false
+      noTip: false,
+      deactivatedHeaders: []
     }
 
     await this.knex.transaction(async trx => {
@@ -337,6 +328,7 @@ export class ChaintracksStorageKnex extends ChaintracksStorageBase implements Ch
           let [headerToDeactivate] = await trx<LiveBlockHeader>(table).where({ isChainTip: true, isActive: true })
           while (headerToDeactivate.headerId !== activeAncestor.headerId) {
             // Headers are deactivated until we reach the activeAncestor
+            r.deactivatedHeaders.push(headerToDeactivate)
             await trx<LiveBlockHeader>(table)
               .where({ headerId: headerToDeactivate.headerId })
               .update({ isActive: false })

package/src/services/chaintracker/chaintracks/Storage/ChaintracksStorageNoDb.ts

@@ -177,7 +177,8 @@ export class ChaintracksStorageNoDb extends ChaintracksStorageBase {
       isActiveTip: false,
       reorgDepth: 0,
       priorTip: undefined,
-      noTip: false
+      noTip: false,
+      deactivatedHeaders: []
     }
 
     // Check for duplicate
@@ -272,6 +273,7 @@ export class ChaintracksStorageNoDb extends ChaintracksStorageBase {
       if (activeAncestor.headerId !== oneBack.headerId) {
         let headerToDeactivate = Array.from(data.liveHeaders.values()).find(h => h.isChainTip && h.isActive)
         while (headerToDeactivate && headerToDeactivate.headerId !== activeAncestor.headerId) {
+          r.deactivatedHeaders.push(headerToDeactivate)
           data.liveHeaders.set(headerToDeactivate.headerId, { ...headerToDeactivate, isActive: false })
           headerToDeactivate = data.liveHeaders.get(headerToDeactivate.previousHeaderId!)
         }

package/src/services/chaintracker/chaintracks/__tests/Chaintracks.test.ts

@@ -2,7 +2,7 @@ import { createDefaultKnexChaintracksOptions } from '../createDefaultKnexChaintr
 import { Chaintracks } from '../Chaintracks'
 import { wait } from '../../../../utility/utilityHelpers'
 import { Chain } from '../../../../sdk'
-import { createNoDbChaintracksOptions } from '../createDefaultNoDbChaintracksOptions'
+import { createDefaultNoDbChaintracksOptions } from '../createDefaultNoDbChaintracksOptions'
 import { ChaintracksFs } from '../util/ChaintracksFs'
 import { LocalCdnServer } from './LocalCdnServer'
 
@@ -52,7 +52,7 @@ describe('Chaintracks tests', () => {
   })
 
   async function NoDbBody(chain: Chain, exportHeaders?: boolean) {
-    const o = createNoDbChaintracksOptions(chain)
+    const o = createDefaultNoDbChaintracksOptions(chain)
     const c = new Chaintracks(o)
     await c.makeAvailable()
 

package/src/services/chaintracker/chaintracks/createDefaultIdbChaintracksOptions.ts

@@ -0,0 +1,92 @@
+import { Chain } from '../../../sdk'
+import { ChaintracksOptions } from './Api/ChaintracksApi'
+import { Chaintracks } from './Chaintracks'
+import { BulkIngestorCDNBabbage } from './Ingest/BulkIngestorCDNBabbage'
+import { ChaintracksFetch } from './util/ChaintracksFetch'
+import { LiveIngestorWhatsOnChainOptions, LiveIngestorWhatsOnChainPoll } from './Ingest/LiveIngestorWhatsOnChainPoll'
+import { BulkIngestorWhatsOnChainCdn, BulkIngestorWhatsOnChainOptions } from './Ingest/BulkIngestorWhatsOnChainCdn'
+import { ChaintracksFetchApi } from './Api/ChaintracksFetchApi'
+import { ChaintracksStorageIdb, ChaintracksStorageIdbOptions } from './Storage/ChaintracksStorageIdb'
+import { BulkFileDataManager, BulkFileDataManagerOptions } from './util/BulkFileDataManager'
+import { BulkIngestorCDNOptions } from './Ingest/BulkIngestorCDN'
+import { WhatsOnChainServicesOptions } from './Ingest/WhatsOnChainServices'
+
+export function createDefaultIdbChaintracksOptions(
+  chain: Chain,
+  whatsonchainApiKey: string = '',
+  maxPerFile: number = 100000,
+  maxRetained: number = 2,
+  fetch?: ChaintracksFetchApi,
+  cdnUrl: string = 'https://cdn.projectbabbage.com/blockheaders/',
+  liveHeightThreshold: number = 2000,
+  reorgHeightThreshold: number = 400,
+  bulkMigrationChunkSize: number = 500,
+  batchInsertLimit: number = 400,
+  addLiveRecursionLimit: number = 36
+): ChaintracksOptions {
+  fetch ||= new ChaintracksFetch()
+
+  const bfo: BulkFileDataManagerOptions = {
+    chain,
+    fetch,
+    maxPerFile,
+    maxRetained,
+    fromKnownSourceUrl: cdnUrl
+  }
+  const bulkFileDataManager = new BulkFileDataManager(bfo)
+
+  const so: ChaintracksStorageIdbOptions = {
+    chain,
+    bulkFileDataManager,
+    liveHeightThreshold,
+    reorgHeightThreshold,
+    bulkMigrationChunkSize,
+    batchInsertLimit
+  }
+  const storage = new ChaintracksStorageIdb(so)
+
+  const co: ChaintracksOptions = {
+    chain,
+    storage,
+    bulkIngestors: [],
+    liveIngestors: [],
+    addLiveRecursionLimit,
+    logging: (...args) => console.log(new Date().toISOString(), ...args),
+    readonly: false
+  }
+
+  const jsonResource = `${chain}NetBlockHeaders.json`
+
+  const bulkCdnOptions: BulkIngestorCDNOptions = {
+    chain,
+    jsonResource,
+    fetch,
+    cdnUrl,
+    maxPerFile
+  }
+  co.bulkIngestors.push(new BulkIngestorCDNBabbage(bulkCdnOptions))
+
+  const wocOptions: WhatsOnChainServicesOptions = {
+    chain,
+    apiKey: whatsonchainApiKey,
+    timeout: 30000,
+    userAgent: 'BabbageWhatsOnChainServices',
+    enableCache: true,
+    chainInfoMsecs: 5000
+  }
+
+  const bulkOptions: BulkIngestorWhatsOnChainOptions = {
+    ...wocOptions,
+    jsonResource,
+    idleWait: 5000
+  }
+  co.bulkIngestors.push(new BulkIngestorWhatsOnChainCdn(bulkOptions))
+
+  const liveOptions: LiveIngestorWhatsOnChainOptions = {
+    ...wocOptions,
+    idleWait: 100000
+  }
+  co.liveIngestors.push(new LiveIngestorWhatsOnChainPoll(liveOptions))
+
+  return co
+}

package/src/services/chaintracker/chaintracks/createDefaultKnexChaintracksOptions.ts

@@ -1,13 +1,16 @@
 import { Knex, knex as makeKnex } from 'knex'
 import { Chain } from '../../../sdk'
 import { ChaintracksOptions } from './Api/ChaintracksApi'
-import { Chaintracks } from './Chaintracks'
 import { ChaintracksFs } from './util/ChaintracksFs'
-import { ChaintracksStorageKnex } from './Storage/ChaintracksStorageKnex'
+import { ChaintracksStorageKnex, ChaintracksStorageKnexOptions } from './Storage/ChaintracksStorageKnex'
 import { BulkIngestorCDNBabbage } from './Ingest/BulkIngestorCDNBabbage'
 import { ChaintracksFetch } from './util/ChaintracksFetch'
-import { LiveIngestorWhatsOnChainPoll } from './Ingest/LiveIngestorWhatsOnChainPoll'
-import { BulkIngestorWhatsOnChainCdn } from './Ingest/BulkIngestorWhatsOnChainCdn'
+import { LiveIngestorWhatsOnChainOptions, LiveIngestorWhatsOnChainPoll } from './Ingest/LiveIngestorWhatsOnChainPoll'
+import { BulkIngestorWhatsOnChainCdn, BulkIngestorWhatsOnChainOptions } from './Ingest/BulkIngestorWhatsOnChainCdn'
+import { ChaintracksFetchApi } from './Api/ChaintracksFetchApi'
+import { BulkFileDataManager, BulkFileDataManagerOptions } from './util/BulkFileDataManager'
+import { BulkIngestorCDNOptions } from './Ingest/BulkIngestorCDN'
+import { WhatsOnChainServicesOptions } from './Ingest/WhatsOnChainServices'
 
 /**
  *
@@ -17,36 +20,92 @@ import { BulkIngestorWhatsOnChainCdn } from './Ingest/BulkIngestorWhatsOnChainCd
  */
 export function createDefaultKnexChaintracksOptions(
   chain: Chain,
-  rootFolder?: string,
-  knexConfig?: Knex.Config
+  rootFolder: string = './data/',
+  knexConfig?: Knex.Config,
+  whatsonchainApiKey: string = '',
+  maxPerFile: number = 100000,
+  maxRetained: number = 2,
+  fetch?: ChaintracksFetchApi,
+  cdnUrl: string = 'https://cdn.projectbabbage.com/blockheaders/',
+  liveHeightThreshold: number = 2000,
+  reorgHeightThreshold: number = 400,
+  bulkMigrationChunkSize: number = 500,
+  batchInsertLimit: number = 400,
+  addLiveRecursionLimit: number = 36
 ): ChaintracksOptions {
-  if (!rootFolder) rootFolder = './data/'
+  fetch ||= new ChaintracksFetch()
 
-  const options = Chaintracks.createOptions(chain)
+  const bfo: BulkFileDataManagerOptions = {
+    chain,
+    fetch,
+    maxPerFile,
+    maxRetained,
+    fromKnownSourceUrl: cdnUrl
+  }
+  const bulkFileDataManager = new BulkFileDataManager(bfo)
+
+  if (!knexConfig) {
+    knexConfig = {
+      client: 'sqlite3',
+      connection: { filename: ChaintracksFs.pathJoin(rootFolder, `${chain}Net_chaintracks.sqlite`) },
+      useNullAsDefault: true
+    }
+  }
+  const knexInstance = makeKnex(knexConfig)
 
-  const fs = ChaintracksFs
-  const fetch = new ChaintracksFetch()
+  const so: ChaintracksStorageKnexOptions = {
+    chain,
+    knex: knexInstance,
+    bulkFileDataManager,
+    liveHeightThreshold,
+    reorgHeightThreshold,
+    bulkMigrationChunkSize,
+    batchInsertLimit
+  }
+  const storage = new ChaintracksStorageKnex(so)
 
-  const localSqlite: Knex.Config = {
-    client: 'sqlite3',
-    connection: { filename: fs.pathJoin(rootFolder, `${chain}Net_chaintracks.sqlite`) },
-    useNullAsDefault: true
+  const co: ChaintracksOptions = {
+    chain,
+    storage,
+    bulkIngestors: [],
+    liveIngestors: [],
+    addLiveRecursionLimit,
+    logging: (...args) => console.log(new Date().toISOString(), ...args),
+    readonly: false
   }
 
-  const knexInstance = makeKnex(knexConfig || localSqlite)
+  const jsonResource = `${chain}NetBlockHeaders.json`
 
-  const knexOptions = ChaintracksStorageKnex.createStorageKnexOptions(chain)
-  knexOptions.knex = knexInstance
-  options.storage = new ChaintracksStorageKnex(knexOptions)
+  const bulkCdnOptions: BulkIngestorCDNOptions = {
+    chain,
+    jsonResource,
+    fetch,
+    cdnUrl,
+    maxPerFile
+  }
+  co.bulkIngestors.push(new BulkIngestorCDNBabbage(bulkCdnOptions))
 
-  const bulkCDNOptions = BulkIngestorCDNBabbage.createBulkIngestorCDNBabbageOptions(chain, fetch)
-  options.bulkIngestors.push(new BulkIngestorCDNBabbage(bulkCDNOptions))
+  const wocOptions: WhatsOnChainServicesOptions = {
+    chain,
+    apiKey: whatsonchainApiKey,
+    timeout: 30000,
+    userAgent: 'BabbageWhatsOnChainServices',
+    enableCache: true,
+    chainInfoMsecs: 5000
+  }
 
-  const bulkWhatsOnChainOptions = BulkIngestorWhatsOnChainCdn.createBulkIngestorWhatsOnChainOptions(chain)
-  options.bulkIngestors.push(new BulkIngestorWhatsOnChainCdn(bulkWhatsOnChainOptions))
+  const bulkOptions: BulkIngestorWhatsOnChainOptions = {
+    ...wocOptions,
+    jsonResource,
+    idleWait: 5000
+  }
+  co.bulkIngestors.push(new BulkIngestorWhatsOnChainCdn(bulkOptions))
 
-  const liveWhatsOnChainOptions = LiveIngestorWhatsOnChainPoll.createLiveIngestorWhatsOnChainOptions(chain)
-  options.liveIngestors.push(new LiveIngestorWhatsOnChainPoll(liveWhatsOnChainOptions))
+  const liveOptions: LiveIngestorWhatsOnChainOptions = {
+    ...wocOptions,
+    idleWait: 100000
+  }
+  co.liveIngestors.push(new LiveIngestorWhatsOnChainPoll(liveOptions))
 
-  return options
+  return co
 }