@bsv/wallet-toolbox 1.6.22 → 1.6.24

package/src/services/chaintracker/chaintracks/ChaintracksService.ts

@@ -4,7 +4,7 @@ import { IncomingMessage, Server, ServerResponse } from 'http'
 import express, { Request, Response } from 'express'
 import bodyParser from 'body-parser'
 import { Chain } from '../../../sdk/types'
-import { createNoDbChaintracksOptions } from './createDefaultNoDbChaintracksOptions'
+import { createDefaultNoDbChaintracksOptions } from './createDefaultNoDbChaintracksOptions'
 import { Services } from '../../Services'
 import { FiatExchangeRates, WERR_INVALID_PARAMETER } from '../../../sdk'
 import { ChaintracksInfoApi } from './Api/ChaintracksClientApi'
@@ -45,7 +45,7 @@ export class ChaintracksService {
     this.options = { ...options }
     this.port = options.port
     this.chain = options.chain
-    this.chaintracks = options.chaintracks || new Chaintracks(createNoDbChaintracksOptions(this.chain))
+    this.chaintracks = options.chaintracks || new Chaintracks(createDefaultNoDbChaintracksOptions(this.chain))
     this.services = options.services || new Services(this.chain)
     // Prevent recursion...
     this.services.updateFiatExchangeRateServices.remove('ChaintracksService')

package/src/services/chaintracker/chaintracks/Storage/ChaintracksStorageIdb.ts

@@ -263,7 +263,8 @@ export class ChaintracksStorageIdb extends ChaintracksStorageBase implements Cha
       isActiveTip: false,
       reorgDepth: 0,
       priorTip: undefined,
-      noTip: false
+      noTip: false,
+      deactivatedHeaders: []
     }
 
     // Check for duplicate
@@ -359,8 +360,10 @@ export class ChaintracksStorageIdb extends ChaintracksStorageBase implements Cha
       }
 
       if (activeAncestor.headerId !== oneBack.headerId) {
+        // Deactivate reorg'ed headers
         let headerToDeactivate = this.repairStoredLiveHeader(await activeTipIndex.get([1, 1]))!
         while (headerToDeactivate && headerToDeactivate.headerId !== activeAncestor.headerId) {
+          r.deactivatedHeaders.push(headerToDeactivate)
           await store.put(this.prepareStoredLiveHeader({ ...headerToDeactivate, isActive: false }))
           headerToDeactivate = this.repairStoredLiveHeader(await store.get(headerToDeactivate.previousHeaderId!))!
         }

package/src/services/chaintracker/chaintracks/Storage/ChaintracksStorageKnex.ts

@@ -4,25 +4,15 @@ import { InsertHeaderResult, ChaintracksStorageBaseOptions } from '../Api/Chaint
 import { ChaintracksStorageBase } from './ChaintracksStorageBase'
 import { LiveBlockHeader } from '../Api/BlockHeaderApi'
 import { BlockHeader } from '../../../../sdk/WalletServices.interfaces'
-import {
-  addWork,
-  convertBitsToWork,
-  isMoreWork,
-  serializeBaseBlockHeader,
-  serializeBaseBlockHeaders
-} from '../util/blockHeaderUtilities'
+import { addWork, convertBitsToWork, isMoreWork } from '../util/blockHeaderUtilities'
 import { verifyOneOrNone } from '../../../../utility/utilityHelpers'
 import { DBType } from '../../../../storage/StorageReader'
 import { BulkHeaderFileInfo } from '../util/BulkHeaderFile'
 import { HeightRange } from '../util/HeightRange'
-import { BulkFilesReaderStorage } from '../util/BulkFilesReader'
-import { ChaintracksFetch } from '../util/ChaintracksFetch'
 import { ChaintracksStorageBulkFileApi } from '../Api/ChaintracksStorageApi'
 import { Chain } from '../../../../sdk/types'
 import { WERR_INVALID_OPERATION, WERR_INVALID_PARAMETER } from '../../../../sdk/WERR_errors'
 import { determineDBType } from '../../../../storage/schema/KnexMigrations'
-import { BulkFileDataReader } from '../index.client'
-import { asArray } from '../../../../utility/utilityHelpers.noBuffer'
 
 export interface ChaintracksStorageKnexOptions extends ChaintracksStorageBaseOptions {
   /**
@@ -231,7 +221,8 @@ export class ChaintracksStorageKnex extends ChaintracksStorageBase implements Ch
       isActiveTip: false,
       reorgDepth: 0,
       priorTip: undefined,
-      noTip: false
+      noTip: false,
+      deactivatedHeaders: []
     }
 
     await this.knex.transaction(async trx => {
@@ -337,6 +328,7 @@ export class ChaintracksStorageKnex extends ChaintracksStorageBase implements Ch
           let [headerToDeactivate] = await trx<LiveBlockHeader>(table).where({ isChainTip: true, isActive: true })
           while (headerToDeactivate.headerId !== activeAncestor.headerId) {
             // Headers are deactivated until we reach the activeAncestor
+            r.deactivatedHeaders.push(headerToDeactivate)
             await trx<LiveBlockHeader>(table)
               .where({ headerId: headerToDeactivate.headerId })
               .update({ isActive: false })

package/src/services/chaintracker/chaintracks/Storage/ChaintracksStorageNoDb.ts

@@ -177,7 +177,8 @@ export class ChaintracksStorageNoDb extends ChaintracksStorageBase {
       isActiveTip: false,
       reorgDepth: 0,
       priorTip: undefined,
-      noTip: false
+      noTip: false,
+      deactivatedHeaders: []
     }
 
     // Check for duplicate
@@ -272,6 +273,7 @@ export class ChaintracksStorageNoDb extends ChaintracksStorageBase {
       if (activeAncestor.headerId !== oneBack.headerId) {
         let headerToDeactivate = Array.from(data.liveHeaders.values()).find(h => h.isChainTip && h.isActive)
         while (headerToDeactivate && headerToDeactivate.headerId !== activeAncestor.headerId) {
+          r.deactivatedHeaders.push(headerToDeactivate)
           data.liveHeaders.set(headerToDeactivate.headerId, { ...headerToDeactivate, isActive: false })
           headerToDeactivate = data.liveHeaders.get(headerToDeactivate.previousHeaderId!)
         }

package/src/services/chaintracker/chaintracks/__tests/Chaintracks.test.ts

@@ -2,7 +2,7 @@ import { createDefaultKnexChaintracksOptions } from '../createDefaultKnexChaintr
 import { Chaintracks } from '../Chaintracks'
 import { wait } from '../../../../utility/utilityHelpers'
 import { Chain } from '../../../../sdk'
-import { createNoDbChaintracksOptions } from '../createDefaultNoDbChaintracksOptions'
+import { createDefaultNoDbChaintracksOptions } from '../createDefaultNoDbChaintracksOptions'
 import { ChaintracksFs } from '../util/ChaintracksFs'
 import { LocalCdnServer } from './LocalCdnServer'
 
@@ -52,7 +52,7 @@ describe('Chaintracks tests', () => {
   })
 
   async function NoDbBody(chain: Chain, exportHeaders?: boolean) {
-    const o = createNoDbChaintracksOptions(chain)
+    const o = createDefaultNoDbChaintracksOptions(chain)
     const c = new Chaintracks(o)
     await c.makeAvailable()
 

package/src/services/chaintracker/chaintracks/createDefaultIdbChaintracksOptions.ts

@@ -0,0 +1,92 @@
+import { Chain } from '../../../sdk'
+import { ChaintracksOptions } from './Api/ChaintracksApi'
+import { Chaintracks } from './Chaintracks'
+import { BulkIngestorCDNBabbage } from './Ingest/BulkIngestorCDNBabbage'
+import { ChaintracksFetch } from './util/ChaintracksFetch'
+import { LiveIngestorWhatsOnChainOptions, LiveIngestorWhatsOnChainPoll } from './Ingest/LiveIngestorWhatsOnChainPoll'
+import { BulkIngestorWhatsOnChainCdn, BulkIngestorWhatsOnChainOptions } from './Ingest/BulkIngestorWhatsOnChainCdn'
+import { ChaintracksFetchApi } from './Api/ChaintracksFetchApi'
+import { ChaintracksStorageIdb, ChaintracksStorageIdbOptions } from './Storage/ChaintracksStorageIdb'
+import { BulkFileDataManager, BulkFileDataManagerOptions } from './util/BulkFileDataManager'
+import { BulkIngestorCDNOptions } from './Ingest/BulkIngestorCDN'
+import { WhatsOnChainServicesOptions } from './Ingest/WhatsOnChainServices'
+
+export function createDefaultIdbChaintracksOptions(
+  chain: Chain,
+  whatsonchainApiKey: string = '',
+  maxPerFile: number = 100000,
+  maxRetained: number = 2,
+  fetch?: ChaintracksFetchApi,
+  cdnUrl: string = 'https://cdn.projectbabbage.com/blockheaders/',
+  liveHeightThreshold: number = 2000,
+  reorgHeightThreshold: number = 400,
+  bulkMigrationChunkSize: number = 500,
+  batchInsertLimit: number = 400,
+  addLiveRecursionLimit: number = 36
+): ChaintracksOptions {
+  fetch ||= new ChaintracksFetch()
+
+  const bfo: BulkFileDataManagerOptions = {
+    chain,
+    fetch,
+    maxPerFile,
+    maxRetained,
+    fromKnownSourceUrl: cdnUrl
+  }
+  const bulkFileDataManager = new BulkFileDataManager(bfo)
+
+  const so: ChaintracksStorageIdbOptions = {
+    chain,
+    bulkFileDataManager,
+    liveHeightThreshold,
+    reorgHeightThreshold,
+    bulkMigrationChunkSize,
+    batchInsertLimit
+  }
+  const storage = new ChaintracksStorageIdb(so)
+
+  const co: ChaintracksOptions = {
+    chain,
+    storage,
+    bulkIngestors: [],
+    liveIngestors: [],
+    addLiveRecursionLimit,
+    logging: (...args) => console.log(new Date().toISOString(), ...args),
+    readonly: false
+  }
+
+  const jsonResource = `${chain}NetBlockHeaders.json`
+
+  const bulkCdnOptions: BulkIngestorCDNOptions = {
+    chain,
+    jsonResource,
+    fetch,
+    cdnUrl,
+    maxPerFile
+  }
+  co.bulkIngestors.push(new BulkIngestorCDNBabbage(bulkCdnOptions))
+
+  const wocOptions: WhatsOnChainServicesOptions = {
+    chain,
+    apiKey: whatsonchainApiKey,
+    timeout: 30000,
+    userAgent: 'BabbageWhatsOnChainServices',
+    enableCache: true,
+    chainInfoMsecs: 5000
+  }
+
+  const bulkOptions: BulkIngestorWhatsOnChainOptions = {
+    ...wocOptions,
+    jsonResource,
+    idleWait: 5000
+  }
+  co.bulkIngestors.push(new BulkIngestorWhatsOnChainCdn(bulkOptions))
+
+  const liveOptions: LiveIngestorWhatsOnChainOptions = {
+    ...wocOptions,
+    idleWait: 100000
+  }
+  co.liveIngestors.push(new LiveIngestorWhatsOnChainPoll(liveOptions))
+
+  return co
+}

package/src/services/chaintracker/chaintracks/createDefaultKnexChaintracksOptions.ts

@@ -1,13 +1,16 @@
 import { Knex, knex as makeKnex } from 'knex'
 import { Chain } from '../../../sdk'
 import { ChaintracksOptions } from './Api/ChaintracksApi'
-import { Chaintracks } from './Chaintracks'
 import { ChaintracksFs } from './util/ChaintracksFs'
-import { ChaintracksStorageKnex } from './Storage/ChaintracksStorageKnex'
+import { ChaintracksStorageKnex, ChaintracksStorageKnexOptions } from './Storage/ChaintracksStorageKnex'
 import { BulkIngestorCDNBabbage } from './Ingest/BulkIngestorCDNBabbage'
 import { ChaintracksFetch } from './util/ChaintracksFetch'
-import { LiveIngestorWhatsOnChainPoll } from './Ingest/LiveIngestorWhatsOnChainPoll'
-import { BulkIngestorWhatsOnChainCdn } from './Ingest/BulkIngestorWhatsOnChainCdn'
+import { LiveIngestorWhatsOnChainOptions, LiveIngestorWhatsOnChainPoll } from './Ingest/LiveIngestorWhatsOnChainPoll'
+import { BulkIngestorWhatsOnChainCdn, BulkIngestorWhatsOnChainOptions } from './Ingest/BulkIngestorWhatsOnChainCdn'
+import { ChaintracksFetchApi } from './Api/ChaintracksFetchApi'
+import { BulkFileDataManager, BulkFileDataManagerOptions } from './util/BulkFileDataManager'
+import { BulkIngestorCDNOptions } from './Ingest/BulkIngestorCDN'
+import { WhatsOnChainServicesOptions } from './Ingest/WhatsOnChainServices'
 
 /**
  *
@@ -17,36 +20,92 @@ import { BulkIngestorWhatsOnChainCdn } from './Ingest/BulkIngestorWhatsOnChainCd
  */
 export function createDefaultKnexChaintracksOptions(
   chain: Chain,
-  rootFolder?: string,
-  knexConfig?: Knex.Config
+  rootFolder: string = './data/',
+  knexConfig?: Knex.Config,
+  whatsonchainApiKey: string = '',
+  maxPerFile: number = 100000,
+  maxRetained: number = 2,
+  fetch?: ChaintracksFetchApi,
+  cdnUrl: string = 'https://cdn.projectbabbage.com/blockheaders/',
+  liveHeightThreshold: number = 2000,
+  reorgHeightThreshold: number = 400,
+  bulkMigrationChunkSize: number = 500,
+  batchInsertLimit: number = 400,
+  addLiveRecursionLimit: number = 36
 ): ChaintracksOptions {
-  if (!rootFolder) rootFolder = './data/'
+  fetch ||= new ChaintracksFetch()
 
-  const options = Chaintracks.createOptions(chain)
+  const bfo: BulkFileDataManagerOptions = {
+    chain,
+    fetch,
+    maxPerFile,
+    maxRetained,
+    fromKnownSourceUrl: cdnUrl
+  }
+  const bulkFileDataManager = new BulkFileDataManager(bfo)
+
+  if (!knexConfig) {
+    knexConfig = {
+      client: 'sqlite3',
+      connection: { filename: ChaintracksFs.pathJoin(rootFolder, `${chain}Net_chaintracks.sqlite`) },
+      useNullAsDefault: true
+    }
+  }
+  const knexInstance = makeKnex(knexConfig)
 
-  const fs = ChaintracksFs
-  const fetch = new ChaintracksFetch()
+  const so: ChaintracksStorageKnexOptions = {
+    chain,
+    knex: knexInstance,
+    bulkFileDataManager,
+    liveHeightThreshold,
+    reorgHeightThreshold,
+    bulkMigrationChunkSize,
+    batchInsertLimit
+  }
+  const storage = new ChaintracksStorageKnex(so)
 
-  const localSqlite: Knex.Config = {
-    client: 'sqlite3',
-    connection: { filename: fs.pathJoin(rootFolder, `${chain}Net_chaintracks.sqlite`) },
-    useNullAsDefault: true
+  const co: ChaintracksOptions = {
+    chain,
+    storage,
+    bulkIngestors: [],
+    liveIngestors: [],
+    addLiveRecursionLimit,
+    logging: (...args) => console.log(new Date().toISOString(), ...args),
+    readonly: false
   }
 
-  const knexInstance = makeKnex(knexConfig || localSqlite)
+  const jsonResource = `${chain}NetBlockHeaders.json`
 
-  const knexOptions = ChaintracksStorageKnex.createStorageKnexOptions(chain)
-  knexOptions.knex = knexInstance
-  options.storage = new ChaintracksStorageKnex(knexOptions)
+  const bulkCdnOptions: BulkIngestorCDNOptions = {
+    chain,
+    jsonResource,
+    fetch,
+    cdnUrl,
+    maxPerFile
+  }
+  co.bulkIngestors.push(new BulkIngestorCDNBabbage(bulkCdnOptions))
 
-  const bulkCDNOptions = BulkIngestorCDNBabbage.createBulkIngestorCDNBabbageOptions(chain, fetch)
-  options.bulkIngestors.push(new BulkIngestorCDNBabbage(bulkCDNOptions))
+  const wocOptions: WhatsOnChainServicesOptions = {
+    chain,
+    apiKey: whatsonchainApiKey,
+    timeout: 30000,
+    userAgent: 'BabbageWhatsOnChainServices',
+    enableCache: true,
+    chainInfoMsecs: 5000
+  }
 
-  const bulkWhatsOnChainOptions = BulkIngestorWhatsOnChainCdn.createBulkIngestorWhatsOnChainOptions(chain)
-  options.bulkIngestors.push(new BulkIngestorWhatsOnChainCdn(bulkWhatsOnChainOptions))
+  const bulkOptions: BulkIngestorWhatsOnChainOptions = {
+    ...wocOptions,
+    jsonResource,
+    idleWait: 5000
+  }
+  co.bulkIngestors.push(new BulkIngestorWhatsOnChainCdn(bulkOptions))
 
-  const liveWhatsOnChainOptions = LiveIngestorWhatsOnChainPoll.createLiveIngestorWhatsOnChainOptions(chain)
-  options.liveIngestors.push(new LiveIngestorWhatsOnChainPoll(liveWhatsOnChainOptions))
+  const liveOptions: LiveIngestorWhatsOnChainOptions = {
+    ...wocOptions,
+    idleWait: 100000
+  }
+  co.liveIngestors.push(new LiveIngestorWhatsOnChainPoll(liveOptions))
 
-  return options
+  return co
 }

package/src/services/chaintracker/chaintracks/createDefaultNoDbChaintracksOptions.ts

@@ -1,29 +1,91 @@
 import { Chain } from '../../../sdk'
 import { ChaintracksOptions } from './Api/ChaintracksApi'
-import { Chaintracks } from './Chaintracks'
 import { BulkIngestorCDNBabbage } from './Ingest/BulkIngestorCDNBabbage'
 import { ChaintracksFetch } from './util/ChaintracksFetch'
-import { LiveIngestorWhatsOnChainPoll } from './Ingest/LiveIngestorWhatsOnChainPoll'
-import { BulkIngestorWhatsOnChainCdn } from './Ingest/BulkIngestorWhatsOnChainCdn'
-import { ChaintracksStorageNoDb } from './Storage/ChaintracksStorageNoDb'
+import { LiveIngestorWhatsOnChainOptions, LiveIngestorWhatsOnChainPoll } from './Ingest/LiveIngestorWhatsOnChainPoll'
+import { BulkIngestorWhatsOnChainCdn, BulkIngestorWhatsOnChainOptions } from './Ingest/BulkIngestorWhatsOnChainCdn'
+import { ChaintracksStorageNoDb, ChaintracksStorageNoDbOptions } from './Storage/ChaintracksStorageNoDb'
+import { ChaintracksFetchApi } from './Api/ChaintracksFetchApi'
+import { BulkFileDataManager, BulkFileDataManagerOptions } from './util/BulkFileDataManager'
+import { BulkIngestorCDNOptions } from './Ingest/BulkIngestorCDN'
+import { WhatsOnChainServicesOptions } from './Ingest/WhatsOnChainServices'
 
-export function createNoDbChaintracksOptions(chain: Chain): ChaintracksOptions {
-  const options = Chaintracks.createOptions(chain)
+export function createDefaultNoDbChaintracksOptions(
+  chain: Chain,
+  whatsonchainApiKey: string = '',
+  maxPerFile: number = 100000,
+  maxRetained: number = 2,
+  fetch?: ChaintracksFetchApi,
+  cdnUrl: string = 'https://cdn.projectbabbage.com/blockheaders/',
+  liveHeightThreshold: number = 2000,
+  reorgHeightThreshold: number = 400,
+  bulkMigrationChunkSize: number = 500,
+  batchInsertLimit: number = 400,
+  addLiveRecursionLimit: number = 36
+): ChaintracksOptions {
+  fetch ||= new ChaintracksFetch()
 
-  const so = ChaintracksStorageNoDb.createStorageBaseOptions(chain)
-  const s = new ChaintracksStorageNoDb(so)
-  options.storage = s
+  const bfo: BulkFileDataManagerOptions = {
+    chain,
+    fetch,
+    maxPerFile,
+    maxRetained,
+    fromKnownSourceUrl: cdnUrl
+  }
+  const bulkFileDataManager = new BulkFileDataManager(bfo)
 
-  const fetch = new ChaintracksFetch()
+  const so: ChaintracksStorageNoDbOptions = {
+    chain,
+    bulkFileDataManager,
+    liveHeightThreshold,
+    reorgHeightThreshold,
+    bulkMigrationChunkSize,
+    batchInsertLimit
+  }
+  const storage = new ChaintracksStorageNoDb(so)
 
-  const bulkCDNOptions = BulkIngestorCDNBabbage.createBulkIngestorCDNBabbageOptions(chain, fetch)
-  options.bulkIngestors.push(new BulkIngestorCDNBabbage(bulkCDNOptions))
+  const co: ChaintracksOptions = {
+    chain,
+    storage,
+    bulkIngestors: [],
+    liveIngestors: [],
+    addLiveRecursionLimit,
+    logging: (...args) => console.log(new Date().toISOString(), ...args),
+    readonly: false
+  }
 
-  const bulkWhatsOnChainOptions = BulkIngestorWhatsOnChainCdn.createBulkIngestorWhatsOnChainOptions(chain)
-  options.bulkIngestors.push(new BulkIngestorWhatsOnChainCdn(bulkWhatsOnChainOptions))
+  const jsonResource = `${chain}NetBlockHeaders.json`
 
-  const liveWhatsOnChainOptions = LiveIngestorWhatsOnChainPoll.createLiveIngestorWhatsOnChainOptions(chain)
-  options.liveIngestors.push(new LiveIngestorWhatsOnChainPoll(liveWhatsOnChainOptions))
+  const bulkCdnOptions: BulkIngestorCDNOptions = {
+    chain,
+    jsonResource,
+    fetch,
+    cdnUrl,
+    maxPerFile
+  }
+  co.bulkIngestors.push(new BulkIngestorCDNBabbage(bulkCdnOptions))
 
-  return options
+  const wocOptions: WhatsOnChainServicesOptions = {
+    chain,
+    apiKey: whatsonchainApiKey,
+    timeout: 30000,
+    userAgent: 'BabbageWhatsOnChainServices',
+    enableCache: true,
+    chainInfoMsecs: 5000
+  }
+
+  const bulkOptions: BulkIngestorWhatsOnChainOptions = {
+    ...wocOptions,
+    jsonResource,
+    idleWait: 5000
+  }
+  co.bulkIngestors.push(new BulkIngestorWhatsOnChainCdn(bulkOptions))
+
+  const liveOptions: LiveIngestorWhatsOnChainOptions = {
+    ...wocOptions,
+    idleWait: 100000
+  }
+  co.liveIngestors.push(new LiveIngestorWhatsOnChainPoll(liveOptions))
+
+  return co
 }

package/src/services/chaintracker/chaintracks/createIdbChaintracks.ts

@@ -1,18 +1,13 @@
 import { Chain } from '../../../sdk'
-import { ChaintracksOptions } from './Api/ChaintracksApi'
 import { Chaintracks } from './Chaintracks'
-import { BulkIngestorCDNBabbage } from './Ingest/BulkIngestorCDNBabbage'
 import { ChaintracksFetch } from './util/ChaintracksFetch'
-import { LiveIngestorWhatsOnChainOptions, LiveIngestorWhatsOnChainPoll } from './Ingest/LiveIngestorWhatsOnChainPoll'
-import { BulkIngestorWhatsOnChainCdn, BulkIngestorWhatsOnChainOptions } from './Ingest/BulkIngestorWhatsOnChainCdn'
 import { ChaintracksFetchApi } from './Api/ChaintracksFetchApi'
-import { ChaintracksStorageIdb, ChaintracksStorageIdbOptions } from './Storage/ChaintracksStorageIdb'
-import { BulkFileDataManager, BulkFileDataManagerOptions } from './util/BulkFileDataManager'
-import { BulkIngestorCDNOptions, WhatsOnChainServicesOptions } from './index.client'
+import { ChaintracksStorageIdb } from './Storage/ChaintracksStorageIdb'
+import { createDefaultIdbChaintracksOptions } from './createDefaultIdbChaintracksOptions'
 
 export async function createIdbChaintracks(
   chain: Chain,
-  whatsonchainApiKey: string,
+  whatsonchainApiKey: string = '',
   maxPerFile: number = 100000,
   maxRetained: number = 2,
   fetch?: ChaintracksFetchApi,
@@ -33,67 +28,19 @@ export async function createIdbChaintracks(
   try {
     fetch ||= new ChaintracksFetch()
 
-    const bfo: BulkFileDataManagerOptions = {
+    const co = createDefaultIdbChaintracksOptions(
       chain,
-      fetch,
+      whatsonchainApiKey,
       maxPerFile,
       maxRetained,
-      fromKnownSourceUrl: cdnUrl
-    }
-    const bulkFileDataManager = new BulkFileDataManager(bfo)
-
-    const so: ChaintracksStorageIdbOptions = {
-      chain,
-      bulkFileDataManager,
+      fetch,
+      cdnUrl,
       liveHeightThreshold,
       reorgHeightThreshold,
       bulkMigrationChunkSize,
-      batchInsertLimit
-    }
-    const storage = new ChaintracksStorageIdb(so)
-
-    const co: ChaintracksOptions = {
-      chain,
-      storage,
-      bulkIngestors: [],
-      liveIngestors: [],
-      addLiveRecursionLimit,
-      logging: (...args) => console.log(new Date().toISOString(), ...args),
-      readonly: false
-    }
-
-    const jsonResource = `${chain}NetBlockHeaders.json`
-
-    const bulkCdnOptions: BulkIngestorCDNOptions = {
-      chain,
-      jsonResource,
-      fetch,
-      cdnUrl,
-      maxPerFile
-    }
-    co.bulkIngestors.push(new BulkIngestorCDNBabbage(bulkCdnOptions))
-
-    const wocOptions: WhatsOnChainServicesOptions = {
-      chain,
-      apiKey: whatsonchainApiKey,
-      timeout: 30000,
-      userAgent: 'BabbageWhatsOnChainServices',
-      enableCache: true,
-      chainInfoMsecs: 5000
-    }
-
-    const bulkOptions: BulkIngestorWhatsOnChainOptions = {
-      ...wocOptions,
-      jsonResource,
-      idleWait: 5000
-    }
-    co.bulkIngestors.push(new BulkIngestorWhatsOnChainCdn(bulkOptions))
-
-    const liveOptions: LiveIngestorWhatsOnChainOptions = {
-      ...wocOptions,
-      idleWait: 100000
-    }
-    co.liveIngestors.push(new LiveIngestorWhatsOnChainPoll(liveOptions))
+      batchInsertLimit,
+      addLiveRecursionLimit
+    )
 
     const chaintracks = new Chaintracks(co)
     const available = chaintracks.makeAvailable()
@@ -102,7 +49,7 @@ export async function createIdbChaintracks(
       chain,
       fetch,
       maxPerFile,
-      storage,
+      storage: co.storage as ChaintracksStorageIdb,
       chaintracks,
       available
     }

package/src/services/chaintracker/chaintracks/createKnexChaintracks.ts

@@ -0,0 +1,65 @@
+import { Knex, knex as makeKnex } from 'knex'
+import { Chain } from '../../../sdk'
+import { Chaintracks } from './Chaintracks'
+import { ChaintracksFetch } from './util/ChaintracksFetch'
+import { ChaintracksFetchApi } from './Api/ChaintracksFetchApi'
+import { ChaintracksStorageKnex } from './Storage/ChaintracksStorageKnex'
+import { createDefaultKnexChaintracksOptions } from './createDefaultKnexChaintracksOptions'
+
+export async function createKnexChaintracks(
+  chain: Chain,
+  rootFolder: string = './data/',
+  knexConfig?: Knex.Config,
+  whatsonchainApiKey: string = '',
+  maxPerFile: number = 100000,
+  maxRetained: number = 2,
+  fetch?: ChaintracksFetchApi,
+  cdnUrl: string = 'https://cdn.projectbabbage.com/blockheaders/',
+  liveHeightThreshold: number = 2000,
+  reorgHeightThreshold: number = 400,
+  bulkMigrationChunkSize: number = 500,
+  batchInsertLimit: number = 400,
+  addLiveRecursionLimit: number = 36
+): Promise<{
+  chain: Chain
+  maxPerFile: number
+  fetch: ChaintracksFetchApi
+  storage: ChaintracksStorageKnex
+  chaintracks: Chaintracks
+  available: Promise<void>
+}> {
+  try {
+    fetch ||= new ChaintracksFetch()
+
+    const co = createDefaultKnexChaintracksOptions(
+      chain,
+      rootFolder,
+      knexConfig,
+      whatsonchainApiKey,
+      maxPerFile,
+      maxRetained,
+      fetch,
+      cdnUrl,
+      liveHeightThreshold,
+      reorgHeightThreshold,
+      bulkMigrationChunkSize,
+      batchInsertLimit,
+      addLiveRecursionLimit
+    )
+
+    const chaintracks = new Chaintracks(co)
+    const available = chaintracks.makeAvailable()
+
+    return {
+      chain,
+      fetch,
+      maxPerFile,
+      storage: co.storage as ChaintracksStorageKnex,
+      chaintracks,
+      available
+    }
+  } catch (error) {
+    console.error('Error setting up Chaintracks with Knex Storage:', error)
+    throw error
+  }
+}