@bopen-io/wallet-toolbox 1.7.18

package/src/services/chaintracker/chaintracks/ChaintracksService.ts

@@ -0,0 +1,199 @@
+import { Chaintracks } from './Chaintracks'
+
+import { IncomingMessage, Server, ServerResponse } from 'http'
+import express, { Request, Response } from 'express'
+import bodyParser from 'body-parser'
+import { Chain } from '../../../sdk/types'
+import { createDefaultNoDbChaintracksOptions } from './createDefaultNoDbChaintracksOptions'
+import { Services } from '../../Services'
+import { FiatExchangeRates, WERR_INVALID_PARAMETER } from '../../../sdk'
+import { ChaintracksInfoApi } from './Api/ChaintracksClientApi'
+import { wait } from '../../../utility/utilityHelpers'
+import { BaseBlockHeader, BlockHeader } from './Api/BlockHeaderApi'
+
+export interface ChaintracksServiceOptions {
+  chain: Chain
+  /**
+   * prepended to the path of each registered service endpoint
+   */
+  routingPrefix: string
+  /**
+   * Defaults to default configured Chaintracks instance with NoDb storage.
+   */
+  chaintracks?: Chaintracks
+  services?: Services
+  port?: number
+}
+
+export class ChaintracksService {
+  static createChaintracksServiceOptions(chain: Chain): ChaintracksServiceOptions {
+    const options: ChaintracksServiceOptions = {
+      chain,
+      routingPrefix: ''
+    }
+    return options
+  }
+
+  chain: Chain
+  options: ChaintracksServiceOptions
+  port?: number
+  chaintracks: Chaintracks
+  services: Services
+  server?: Server<typeof IncomingMessage, typeof ServerResponse>
+
+  constructor(options: ChaintracksServiceOptions) {
+    this.options = { ...options }
+    this.port = options.port
+    this.chain = options.chain
+    this.chaintracks = options.chaintracks || new Chaintracks(createDefaultNoDbChaintracksOptions(this.chain))
+    this.services = options.services || new Services(this.chain)
+    // Prevent recursion...
+    this.services.updateFiatExchangeRateServices.remove('ChaintracksService')
+    if (this.chaintracks.chain !== this.chain || this.services.chain !== this.chain) {
+      throw new WERR_INVALID_PARAMETER(
+        'chain',
+        `All components (chaintracks and services) must be on chain ${this.chain}`
+      )
+    }
+  }
+
+  async stopJsonRpcServer(): Promise<void> {
+    this.server?.close()
+    await this.chaintracks?.destroy()
+  }
+
+  async startJsonRpcServer(port?: number): Promise<void> {
+    await this.chaintracks.makeAvailable()
+
+    port ||= this.port || 3011
+    this.port = port
+
+    const app = express()
+    app.use(bodyParser.json())
+
+    // This allows the API to be used when CORS is enforced
+    app.use((req, res, next) => {
+      res.header('Access-Control-Allow-Origin', '*')
+      res.header('Access-Control-Allow-Headers', '*')
+      res.header('Access-Control-Allow-Methods', '*')
+      res.header('Access-Control-Expose-Headers', '*')
+      res.header('Access-Control-Allow-Private-Network', 'true')
+      if (req.method === 'OPTIONS') {
+        res.sendStatus(200)
+      } else {
+        next()
+      }
+    })
+
+    app.get(`/robots.txt`, (req: Request, res: Response) => {
+      res.type('text/plain')
+      res.send(`User-agent: *\nDisallow: /`)
+    })
+
+    app.get(`/`, (req: Request, res: Response) => {
+      res.type('text/plain')
+      res.send(`Chaintracks ${this.chain}Net Block Header Service`)
+    })
+
+    const handleErr = (err: any, res: any) => {
+      res.status(500).json({
+        status: 'error',
+        code: 'ERR_INTERNAL',
+        description: err?.message || 'An internal error has occurred.'
+      })
+    }
+
+    const appGetVoid = (path: string, action: (q: any) => Promise<void>, noCache = false) => {
+      app['get'](this.options.routingPrefix + path, async (req, res) => {
+        if (noCache) {
+          res.setHeader('Cache-Control', 'no-cache, no-store, must-revalidate')
+          res.setHeader('Pragma', 'no-cache')
+          res.setHeader('Expires', '0')
+        }
+        try {
+          console.log(`request ${path}`)
+          await action(req.query)
+          res.status(200).json({ status: 'success' })
+        } catch (err) {
+          handleErr(err, res)
+        }
+      })
+    }
+
+    const appGet = <T>(path: string, action: (q: any) => Promise<T>, noCache = false) => {
+      app['get'](this.options.routingPrefix + path, async (req, res) => {
+        if (noCache) {
+          res.setHeader('Cache-Control', 'no-cache, no-store, must-revalidate')
+          res.setHeader('Pragma', 'no-cache')
+          res.setHeader('Expires', '0')
+        }
+        try {
+          const r = await action(req.query)
+          console.log('request', path, JSON.stringify(req.query), '->', JSON.stringify(r))
+          res.status(200).json({ status: 'success', value: r })
+        } catch (err) {
+          console.log(`request ${path} -> error`)
+          handleErr(err, res)
+        }
+      })
+    }
+
+    const appPostVoid = <T>(path: string, action: (p: T) => Promise<void>) => {
+      app['post'](this.options.routingPrefix + path, async (req, res) => {
+        try {
+          console.log(`request POST ${path}`)
+          await action(<T>req.body)
+          res.status(200).json({ status: 'success' })
+        } catch (err) {
+          handleErr(err, res)
+        }
+      })
+    }
+
+    appGet<Chain>('/getChain', async () => await this.chaintracks.getChain())
+    appGet<ChaintracksInfoApi>(
+      '/getInfo',
+      async q => {
+        if (q.wait) await wait(Number(q.wait))
+        const r = await this.chaintracks.getInfo()
+        if (q.wait) r['wait'] = q.wait
+        return r
+      },
+      true
+    )
+
+    appGet<FiatExchangeRates>(
+      '/getFiatExchangeRates',
+      async () => {
+        // update if needed
+        await this.services.getFiatExchangeRate('GBP')
+        // return current values
+        return this.services.options.fiatExchangeRates
+      },
+      true
+    )
+
+    appPostVoid('/addHeaderHex', async (header: BaseBlockHeader) => {
+      await this.chaintracks.addHeader(header)
+    })
+
+    appGet<number>('/getPresentHeight', async () => await this.chaintracks.getPresentHeight(), true)
+    appGet<string>('/findChainTipHashHex', async () => (await this.chaintracks.findChainTipHash()) || '', true)
+    appGet<BlockHeader>('/findChainTipHeaderHex', async () => await this.chaintracks.findChainTipHeader(), true)
+
+    appGet<BlockHeader | undefined>('/findHeaderHexForHeight', async q => {
+      return await this.chaintracks.findHeaderForHeight(Number(q.height))
+    })
+    appGet<BlockHeader | undefined>('/findHeaderHexForBlockHash', async q => {
+      return await this.chaintracks.findLiveHeaderForBlockHash(q.hash)
+    })
+
+    appGet<string>('/getHeaders', async q => {
+      return await this.chaintracks.getHeaders(Number(q.height), Number(q.count))
+    })
+
+    this.server = app.listen(this.port, () => {
+      console.log(`ChaintracksService listening on port ${this.port}`)
+    })
+  }
+}

package/src/services/chaintracker/chaintracks/ChaintracksServiceClient.ts

@@ -0,0 +1,154 @@
+import { Chain } from '../../../sdk/types'
+import { asString } from '../../../utility/utilityHelpers.noBuffer'
+import { BaseBlockHeader, BlockHeader } from './Api/BlockHeaderApi'
+import { ChaintracksClientApi, ChaintracksInfoApi, HeaderListener, ReorgListener } from './Api/ChaintracksClientApi'
+
+interface FetchStatus<T> {
+  status: 'success' | 'error'
+  code?: string
+  description?: string
+  value?: T
+}
+
+export interface ChaintracksServiceClientOptions {}
+
+/**
+ * Connects to a ChaintracksService to implement 'ChaintracksClientApi'
+ *
+ */
+export class ChaintracksServiceClient implements ChaintracksClientApi {
+  static createChaintracksServiceClientOptions(): ChaintracksServiceClientOptions {
+    const options: ChaintracksServiceClientOptions = {
+      useAuthrite: false
+    }
+    return options
+  }
+
+  options: ChaintracksServiceClientOptions
+
+  constructor(
+    public chain: Chain,
+    public serviceUrl: string,
+    options?: ChaintracksServiceClientOptions
+  ) {
+    this.options = options || ChaintracksServiceClient.createChaintracksServiceClientOptions()
+  }
+
+  subscribeHeaders(listener: HeaderListener): Promise<string> {
+    throw new Error('Method not implemented.')
+  }
+  subscribeReorgs(listener: ReorgListener): Promise<string> {
+    throw new Error('Method not implemented.')
+  }
+  unsubscribe(subscriptionId: string): Promise<boolean> {
+    throw new Error('Method not implemented.')
+  }
+
+  async currentHeight(): Promise<number> {
+    return await this.getPresentHeight()
+  }
+
+  async isValidRootForHeight(root: string, height: number): Promise<boolean> {
+    const r = await this.findHeaderForHeight(height)
+    if (!r) return false
+    const isValid = root === asString(r.merkleRoot)
+    return isValid
+  }
+
+  async getJsonOrUndefined<T>(path: string): Promise<T | undefined> {
+    let e: Error | undefined = undefined
+    for (let retry = 0; retry < 3; retry++) {
+      try {
+        const r = await fetch(`${this.serviceUrl}${path}`)
+        const v = <FetchStatus<T>>await r.json()
+        if (v.status === 'success') return v.value
+        else e = new Error(JSON.stringify(v))
+      } catch (eu: unknown) {
+        e = eu as Error
+      }
+      if (e && e.name !== 'ECONNRESET') break
+    }
+    if (e) throw e
+  }
+
+  async getJson<T>(path: string): Promise<T> {
+    const r = await this.getJsonOrUndefined<T>(path)
+    if (r === undefined) throw new Error('Value was undefined. Requested object may not exist.')
+    return r
+  }
+
+  async postJsonVoid<T>(path: string, params: T): Promise<void> {
+    const headers = {}
+    headers['Content-Type'] = 'application/json'
+    const r = await fetch(`${this.serviceUrl}${path}`, {
+      body: JSON.stringify(params),
+      method: 'POST',
+      headers
+      //cache: 'no-cache',
+    })
+    try {
+      const s = <FetchStatus<void>>await r.json()
+      if (s.status === 'success') return
+      throw new Error(JSON.stringify(s))
+    } catch (e) {
+      console.log(`Exception: ${JSON.stringify(e)}`)
+      throw new Error(JSON.stringify(e))
+    }
+  }
+
+  //
+  // HTTP API FUNCTIONS
+  //
+
+  async addHeader(header: BaseBlockHeader): Promise<void> {
+    const r = await this.postJsonVoid('/addHeaderHex', header)
+    if (typeof r === 'string') throw new Error(r)
+  }
+
+  async startListening(): Promise<void> {
+    await this.getPresentHeight()
+  }
+  async listening(): Promise<void> {
+    await this.getPresentHeight()
+  }
+  async getChain(): Promise<Chain> {
+    return this.chain
+    //return await this.getJson('/getChain')
+  }
+
+  async isListening(): Promise<boolean> {
+    try {
+      await this.getPresentHeight()
+      return true
+    } catch {
+      return false
+    }
+  }
+  async isSynchronized(): Promise<boolean> {
+    return await this.isListening()
+  }
+  async getPresentHeight(): Promise<number> {
+    return await this.getJson('/getPresentHeight')
+  }
+  async getInfo(): Promise<ChaintracksInfoApi> {
+    return await this.getJson('/getInfo')
+  }
+  async findChainTipHeader(): Promise<BlockHeader> {
+    return await this.getJson('/findChainTipHeaderHex')
+  }
+  async findChainTipHash(): Promise<string> {
+    return await this.getJson('/findChainTipHashHex')
+  }
+
+  async getHeaders(height: number, count: number): Promise<string> {
+    return await this.getJson<string>(`/getHeaders?height=${height}&count=${count}`)
+  }
+
+  async findHeaderForHeight(height: number): Promise<BlockHeader | undefined> {
+    return await this.getJsonOrUndefined(`/findHeaderHexForHeight?height=${height}`)
+  }
+
+  async findHeaderForBlockHash(hash: string): Promise<BlockHeader | undefined> {
+    return await this.getJsonOrUndefined(`/findHeaderHexForBlockHash?hash=${hash}`)
+  }
+}

package/src/services/chaintracker/chaintracks/Ingest/BulkIngestorBase.ts

@@ -0,0 +1,176 @@
+import { BulkIngestorApi, BulkIngestorBaseOptions, BulkSyncResult } from '../Api/BulkIngestorApi'
+import { ChaintracksStorageApi } from '../Api/ChaintracksStorageApi'
+
+import { BulkHeaderFilesInfo } from '../util/BulkHeaderFile'
+import { HeightRange, HeightRanges } from '../util/HeightRange'
+import { Chain } from '../../../../sdk/types'
+import { BlockHeader } from '../Api/BlockHeaderApi'
+import { ChaintracksStorageBase } from '../Storage/ChaintracksStorageBase'
+
+export abstract class BulkIngestorBase implements BulkIngestorApi {
+  /**
+   *
+   * @param chain
+   * @param localCachePath defaults to './data/ingest_headers/'
+   * @returns
+   */
+  static createBulkIngestorBaseOptions(chain: Chain) {
+    const options: BulkIngestorBaseOptions = {
+      chain,
+      jsonResource: `${chain}NetBlockHeaders.json`
+    }
+    return options
+  }
+
+  chain: Chain
+  jsonFilename: string
+  log: (...args: any[]) => void = () => {}
+
+  constructor(options: BulkIngestorBaseOptions) {
+    if (!options.jsonResource) throw new Error('The jsonFilename options property is required.')
+    this.chain = options.chain
+    this.jsonFilename = options.jsonResource
+  }
+
+  private storageEngine: ChaintracksStorageBase | undefined
+
+  async setStorage(storage: ChaintracksStorageBase, log: (...args: any[]) => void): Promise<void> {
+    this.storageEngine = storage
+    this.log = log
+  }
+
+  async shutdown(): Promise<void> {}
+
+  storageOrUndefined(): ChaintracksStorageApi | undefined {
+    return this.storageEngine
+  }
+
+  storage(): ChaintracksStorageBase {
+    if (!this.storageEngine) throw new Error('storageEngine must be set.')
+    return this.storageEngine
+  }
+
+  /**
+   * information about locally cached bulk header files managed by this bulk ingestor
+   */
+  filesInfo: BulkHeaderFilesInfo | undefined
+
+  /**
+   * At least one derived BulkIngestor must override this method to provide the current height of the active chain tip.
+   * @returns undefined unless overridden
+   */
+  async getPresentHeight(): Promise<number | undefined> {
+    return undefined
+  }
+
+  /**
+   * A BulkIngestor fetches and updates storage with bulk headers in bulkRange.
+   *
+   * If it can, it must also fetch live headers in fetch range that are not in bulkRange and return them as an array.
+   *
+   * The storage methods `insertBulkFile`, `updateBulkFile`, and `addBulkHeaders` should be used to add bulk headers to storage.
+   *
+   * @param before bulk and live range of headers before ingesting any new headers.
+   * @param fetchRange range of headers still needed, includes both missing bulk and live headers.
+   * @param bulkRange range of bulk headers still needed
+   * @param priorLiveHeaders any headers accumulated by prior bulk ingestor(s) that are too recent for bulk storage.
+   * @returns new live headers: headers in fetchRange but not in bulkRange
+   */
+  abstract fetchHeaders(
+    before: HeightRanges,
+    fetchRange: HeightRange,
+    bulkRange: HeightRange,
+    priorLiveHeaders: BlockHeader[]
+  ): Promise<BlockHeader[]>
+
+  /**
+   * A BulkIngestor has two potential goals:
+   * 1. To source missing bulk headers and include them in bulk storage.
+   * 2. To source missing live headers to be forwarded to live storage.
+   *
+   * @param presentHeight current height of the active chain tip, may lag the true value.
+   * @param before current bulk and live storage height ranges, either may be empty.
+   * @param priorLiveHeaders any headers accumulated by prior bulk ingestor(s) that are too recent for bulk storage.
+   * @returns updated priorLiveHeaders including any accumulated by this ingestor
+   */
+  async synchronize(
+    presentHeight: number,
+    before: HeightRanges,
+    priorLiveHeaders: BlockHeader[]
+  ): Promise<BulkSyncResult> {
+    const storage = this.storage()
+
+    const r: BulkSyncResult = {
+      liveHeaders: priorLiveHeaders,
+      liveRange: HeightRange.from(priorLiveHeaders),
+      done: false,
+      log: ''
+    }
+
+    // Decisions to be made:
+    // Q1. Are we already done?
+    // Q2. Are there live headers that should be migrated to bulk?
+    // Q3. What range of headers do we still need to retrieve?
+
+    // Q1: We are done if we have enough live headers and they include presentHeight.
+    const currentFullRange = before.bulk.union(before.live)
+    if (currentFullRange.maxHeight >= presentHeight) {
+      r.done = true
+      return r
+    }
+
+    const targetBulkRange = new HeightRange(0, Math.max(0, presentHeight - storage.liveHeightThreshold))
+    let missingBulkRange = targetBulkRange.subtract(before.bulk)
+
+    const updateMissingBulkRange = async () => {
+      before = await storage.getAvailableHeightRanges()
+      missingBulkRange = targetBulkRange.subtract(before.bulk)
+    }
+
+    // Q2: If missingBulkRange isn't empty and there are live headers in storage,
+    // migrate from existing live headers in excess of reorgHeightThreshold.
+    if (!missingBulkRange.isEmpty && !before.live.isEmpty) {
+      const countToMigrate = Math.min(
+        missingBulkRange.length,
+        Math.max(0, before.live.length - storage.reorgHeightThreshold)
+      )
+      r.log += `Migrating ${countToMigrate} live headers to bulk storage.\n`
+      await storage.migrateLiveToBulk(countToMigrate)
+      await updateMissingBulkRange()
+      if (!missingBulkRange.isEmpty) {
+        // If there are still missing bulk headers, MUST flush live storage.
+        const countToFlush = before.live.length
+        r.log += `Flushing ${countToFlush} live headers from live storage.\n`
+        await storage.deleteLiveBlockHeaders()
+        await updateMissingBulkRange()
+      }
+    }
+
+    const targetFullRange = new HeightRange(0, presentHeight)
+    // Q3: What to fetch...
+    let rangeToFetch: HeightRange
+    if (missingBulkRange.isEmpty) {
+      // If there are no missing bulk headers, we don't need existing bulk range.
+      rangeToFetch = targetFullRange.subtract(before.bulk)
+      // And if there are live headers in excess of reorgHeightThreshold, they can be skipped as well.
+      if (before.live.length > storage.reorgHeightThreshold) {
+        rangeToFetch = rangeToFetch.subtract(
+          new HeightRange(before.live.minHeight, before.live.maxHeight - storage.reorgHeightThreshold)
+        )
+      }
+    } else {
+      // If there are missing bulk headers, ingest from start of missing through present height.
+      rangeToFetch = new HeightRange(missingBulkRange.minHeight, presentHeight)
+    }
+
+    const newLiveHeaders = await this.fetchHeaders(before, rangeToFetch, missingBulkRange, priorLiveHeaders)
+
+    await updateMissingBulkRange()
+
+    r.liveHeaders = newLiveHeaders
+    r.liveRange = HeightRange.from(r.liveHeaders)
+    r.done = missingBulkRange.isEmpty && r.liveRange.maxHeight >= presentHeight
+
+    return r
+  }
+}

package/src/services/chaintracker/chaintracks/Ingest/BulkIngestorCDN.ts

@@ -0,0 +1,174 @@
+import { Chain } from '../../../../sdk/types'
+import { BlockHeader } from '../Api/BlockHeaderApi'
+import { BulkIngestorBaseOptions } from '../Api/BulkIngestorApi'
+import { BulkIngestorBase } from './BulkIngestorBase'
+import { BulkHeaderFileInfo } from '../util/BulkHeaderFile'
+import { BulkHeaderFilesInfo } from '../util/BulkHeaderFile'
+import { HeightRange, HeightRanges } from '../util/HeightRange'
+import { ChaintracksFetchApi } from '../Api/ChaintracksFetchApi'
+import { WalletError, WERR_INVALID_PARAMETER } from '../../../../sdk'
+import { validateBulkFileData } from '../util/blockHeaderUtilities'
+import { selectBulkHeaderFiles } from '../util/BulkFileDataManager'
+
+export interface BulkIngestorCDNOptions extends BulkIngestorBaseOptions {
+  /**
+   * Required.
+   *
+   * The name of the JSON resource to request from CDN which describes currently
+   * available bulk block header resources.
+   */
+  jsonResource: string | undefined
+
+  /**
+   * Required.
+   *
+   * URL to CDN implementing the bulk ingestor CDN service protocol
+   */
+  cdnUrl: string | undefined
+
+  maxPerFile: number | undefined
+
+  fetch: ChaintracksFetchApi
+}
+
+export class BulkIngestorCDN extends BulkIngestorBase {
+  /**
+   *
+   * @param chain
+   * @param localCachePath defaults to './data/bulk_cdn_headers/'
+   * @returns
+   */
+  static createBulkIngestorCDNOptions(
+    chain: Chain,
+    cdnUrl: string,
+    fetch: ChaintracksFetchApi,
+    maxPerFile?: number
+  ): BulkIngestorCDNOptions {
+    const options: BulkIngestorCDNOptions = {
+      ...BulkIngestorBase.createBulkIngestorBaseOptions(chain),
+      fetch,
+      jsonResource: `${chain}NetBlockHeaders.json`,
+      cdnUrl,
+      maxPerFile
+    }
+    return options
+  }
+
+  fetch: ChaintracksFetchApi
+  jsonResource: string
+  cdnUrl: string
+  maxPerFile: number | undefined
+
+  availableBulkFiles: BulkHeaderFilesInfo | undefined
+  selectedFiles: BulkHeaderFileInfo[] | undefined
+  currentRange: HeightRange | undefined
+
+  constructor(options: BulkIngestorCDNOptions) {
+    super(options)
+    if (!options.jsonResource) throw new Error('The jsonResource options property is required.')
+    if (!options.cdnUrl) throw new Error('The cdnUrl options property is required.')
+
+    this.fetch = options.fetch
+    this.jsonResource = options.jsonResource
+    this.cdnUrl = options.cdnUrl
+    this.maxPerFile = options.maxPerFile
+  }
+
+  override async getPresentHeight(): Promise<number | undefined> {
+    return undefined
+  }
+
+  getJsonHttpHeaders(): Record<string, string> {
+    const headers: Record<string, string> = {
+      Accept: 'application/json'
+    }
+    return headers
+  }
+
+  /**
+   * A BulkFile CDN serves a JSON BulkHeaderFilesInfo resource which lists all the available binary bulk header files available and associated metadata.
+   *
+   * The term "CDN file" is used for a local bulk file that has a sourceUrl. (Not undefined)
+   * The term "incremental file" is used for the local bulk file that holds all the non-CDN bulk headers and must chain to the live headers if there are any.
+   *
+   * Bulk ingesting from a CDN happens in one of three contexts:
+   *
+   * 1. Cold Start: No local bulk or live headers.
+   * 2. Incremental: Available CDN files extend into an existing incremental file but not into the live headers.
+   * 3. Replace: Available CDN files extend into live headers.
+   *
+   * Context Cold Start:
+   * - The CDN files are selected in height order, starting at zero, always choosing the largest count less than the local maximum (maxPerFile).
+   *
+   * Context Incremental:
+   * - Last existing CDN file is updated if CDN now has a higher count.
+   * - Additional CDN files are added as in Cold Start.
+   * - The existing incremental file is truncated or deleted.
+   *
+   * Context Replace:
+   * - Existing live headers are truncated or deleted.
+   * - Proceed as context Incremental.
+   *
+   * @param before bulk and live range of headers before ingesting any new headers.
+   * @param fetchRange total range of header heights needed including live headers
+   * @param bulkRange range of missing bulk header heights required.
+   * @param priorLiveHeaders
+   * @returns
+   */
+  async fetchHeaders(
+    before: HeightRanges,
+    fetchRange: HeightRange,
+    bulkRange: HeightRange,
+    priorLiveHeaders: BlockHeader[]
+  ): Promise<BlockHeader[]> {
+    const storage = this.storage()
+
+    const toUrl = (file: string) => this.fetch.pathJoin(this.cdnUrl, file)
+
+    const url = toUrl(this.jsonResource)
+    this.availableBulkFiles = await this.fetch.fetchJson(url)
+    if (!this.availableBulkFiles) {
+      throw new WERR_INVALID_PARAMETER(
+        `${this.jsonResource}`,
+        `a valid BulkHeaderFilesInfo JSON resource available from ${url}`
+      )
+    }
+    this.selectedFiles = selectBulkHeaderFiles(
+      this.availableBulkFiles.files,
+      this.chain,
+      this.maxPerFile || this.availableBulkFiles.headersPerFile
+    )
+    for (const bf of this.selectedFiles) {
+      if (!bf.fileHash) {
+        throw new WERR_INVALID_PARAMETER(`fileHash`, `valid for alll files in ${this.jsonResource} from ${url}`)
+      }
+      if (!bf.chain || bf.chain !== this.chain) {
+        throw new WERR_INVALID_PARAMETER(`chain`, `"${this.chain}" for all files in ${this.jsonResource} from ${url}`)
+      }
+      if (!bf.sourceUrl || bf.sourceUrl !== this.cdnUrl) bf.sourceUrl = this.cdnUrl
+    }
+
+    let log = 'BulkIngestorCDN fetchHeaders log:\n'
+    log += `  url: ${url}\n`
+
+    this.currentRange = await storage.bulkManager.getHeightRange()
+    log += `  bulk range before: ${this.currentRange}\n`
+
+    const r = await storage.bulkManager.merge(this.selectedFiles)
+
+    this.currentRange = await storage.bulkManager.getHeightRange()
+    log += `  bulk range after:  ${this.currentRange}\n`
+    for (const u of r.unchanged) {
+      log += `  unchanged: ${u.fileName}, fileId=${u.fileId}\n`
+    }
+    for (const i of r.inserted) {
+      log += `  inserted: ${i.fileName}, fileId=${i.fileId}\n`
+    }
+    for (const u of r.updated) {
+      log += `  updated: ${u.fileName}, fileId=${u.fileId}\n`
+    }
+    this.log(log)
+
+    return priorLiveHeaders
+  }
+}