@bsv/wallet-toolbox 1.5.21 → 1.6.2

package/src/services/chaintracker/chaintracks/Ingest/BulkIngestorWhatsOnChainCdn.ts

@@ -0,0 +1,114 @@
+import { Chain } from '../../../../sdk'
+import { logger } from '../../../../utility/utilityHelpers'
+import { BlockHeader } from '../Api/BlockHeaderApi'
+import { BulkIngestorBaseOptions } from '../Api/BulkIngestorApi'
+import { ChaintracksFetchApi } from '../Api/ChaintracksFetchApi'
+import { BulkIngestorBase } from './BulkIngestorBase'
+import { deserializeBlockHeader } from '../util/blockHeaderUtilities'
+import { ChaintracksFetch } from '../util/ChaintracksFetch'
+import { HeightRange, HeightRanges } from '../util/HeightRange'
+import { StopListenerToken } from './WhatsOnChainIngestorWs'
+import { WhatsOnChainServices, WhatsOnChainServicesOptions } from './WhatsOnChainServices'
+
+export interface BulkIngestorWhatsOnChainOptions extends BulkIngestorBaseOptions, WhatsOnChainServicesOptions {
+  /**
+   * Maximum msces of "normal" pause with no new data arriving.
+   */
+  idleWait: number | undefined
+  /**
+   * Which chain is being tracked: main, test, or stn.
+   */
+  chain: Chain
+  /**
+   * WhatsOnChain.com API Key
+   * https://docs.taal.com/introduction/get-an-api-key
+   * If unknown or empty, maximum request rate is limited.
+   * https://developers.whatsonchain.com/#rate-limits
+   */
+  apiKey?: string
+  /**
+   * Request timeout for GETs to https://api.whatsonchain.com/v1/bsv
+   */
+  timeout: number
+  /**
+   * User-Agent header value for requests to https://api.whatsonchain.com/v1/bsv
+   */
+  userAgent: string
+  /**
+   * Enable WhatsOnChain client cache option.
+   */
+  enableCache: boolean
+  /**
+   * How long chainInfo is considered still valid before updating (msecs).
+   */
+  chainInfoMsecs: number
+  /**
+   *
+   */
+  fetch?: ChaintracksFetchApi
+}
+
+export class BulkIngestorWhatsOnChainCdn extends BulkIngestorBase {
+  /**
+   *
+   * @param chain
+   * @param localCachePath defaults to './data/ingest_whatsonchain_headers'
+   * @returns
+   */
+  static createBulkIngestorWhatsOnChainOptions(chain: Chain): BulkIngestorWhatsOnChainOptions {
+    const options: BulkIngestorWhatsOnChainOptions = {
+      ...WhatsOnChainServices.createWhatsOnChainServicesOptions(chain),
+      ...BulkIngestorBase.createBulkIngestorBaseOptions(chain),
+      idleWait: 5000
+    }
+    return options
+  }
+
+  fetch: ChaintracksFetchApi
+  idleWait: number
+  woc: WhatsOnChainServices
+  stopOldListenersToken: StopListenerToken = { stop: undefined }
+
+  constructor(options: BulkIngestorWhatsOnChainOptions) {
+    super(options)
+    this.idleWait = options.idleWait || 5000
+    this.woc = new WhatsOnChainServices(options)
+    this.fetch = options.fetch || new ChaintracksFetch()
+  }
+
+  override async getPresentHeight(): Promise<number | undefined> {
+    const presentHeight = await this.woc.getChainTipHeight()
+    logger(`presentHeight=${presentHeight}`)
+    return presentHeight
+  }
+
+  async fetchHeaders(
+    before: HeightRanges,
+    fetchRange: HeightRange,
+    bulkRange: HeightRange,
+    priorLiveHeaders: BlockHeader[]
+  ): Promise<BlockHeader[]> {
+    const oldHeaders: BlockHeader[] = []
+
+    try {
+      const ranges = await this.woc.getHeaderByteFileLinks(fetchRange, this.fetch)
+      const headers: BlockHeader[] = []
+      for (const range of ranges) {
+        for (let height = range.range.minHeight; height <= range.range.maxHeight; height++) {
+          if (fetchRange.contains(height)) {
+            if (!range.data)
+              range.data = await this.fetch.download(this.fetch.pathJoin(range.sourceUrl, range.fileName))
+            const h = deserializeBlockHeader(range.data, (height - range.range.minHeight) * 80, height)
+            oldHeaders.push(h)
+          }
+        }
+      }
+    } catch (e) {
+      logger(`Errors during WhatsOnChain ingestion:\n${e}`)
+    }
+
+    const liveHeaders = await this.storage().addBulkHeaders(oldHeaders, bulkRange, priorLiveHeaders)
+
+    return liveHeaders
+  }
+}

package/src/services/chaintracker/chaintracks/Ingest/BulkIngestorWhatsOnChainWs.ts

@@ -0,0 +1,80 @@
+import { Chain } from '../../../../sdk'
+import { logger } from '../../../../utility/utilityHelpers'
+import { BlockHeader } from '../Api/BlockHeaderApi'
+import { BulkIngestorBase } from './BulkIngestorBase'
+import { HeightRange, HeightRanges } from '../util/HeightRange'
+import { BulkIngestorWhatsOnChainOptions } from './BulkIngestorWhatsOnChainCdn'
+import { StopListenerToken, WocHeadersBulkListener } from './WhatsOnChainIngestorWs'
+import { EnqueueHandler, ErrorHandler, WhatsOnChainServices } from './WhatsOnChainServices'
+
+export class BulkIngestorWhatsOnChainWs extends BulkIngestorBase {
+  /**
+   *
+   * @param chain
+   * @param localCachePath defaults to './data/ingest_whatsonchain_headers'
+   * @returns
+   */
+  static createBulkIngestorWhatsOnChainOptions(chain: Chain): BulkIngestorWhatsOnChainOptions {
+    const options: BulkIngestorWhatsOnChainOptions = {
+      ...WhatsOnChainServices.createWhatsOnChainServicesOptions(chain),
+      ...BulkIngestorBase.createBulkIngestorBaseOptions(chain),
+      idleWait: 5000
+    }
+    return options
+  }
+
+  idleWait: number
+  woc: WhatsOnChainServices
+  stopOldListenersToken: StopListenerToken = { stop: undefined }
+
+  constructor(options: BulkIngestorWhatsOnChainOptions) {
+    super(options)
+    this.idleWait = options.idleWait || 5000
+    this.woc = new WhatsOnChainServices(options)
+  }
+
+  override async getPresentHeight(): Promise<number | undefined> {
+    const presentHeight = await this.woc.getChainTipHeight()
+    logger(`presentHeight=${presentHeight}`)
+    return presentHeight
+  }
+
+  async fetchHeaders(
+    before: HeightRanges,
+    fetchRange: HeightRange,
+    bulkRange: HeightRange,
+    priorLiveHeaders: BlockHeader[]
+  ): Promise<BlockHeader[]> {
+    const oldHeaders: BlockHeader[] = []
+    const errors: { code: number; message: string; count: number }[] = []
+    const enqueue: EnqueueHandler = header => {
+      oldHeaders.push(header)
+    }
+    const error: ErrorHandler = (code, message) => {
+      errors.push({ code, message, count: errors.length })
+      return false
+    }
+
+    const ok = await WocHeadersBulkListener(
+      fetchRange.minHeight,
+      fetchRange.maxHeight,
+      enqueue,
+      error,
+      this.stopOldListenersToken,
+      this.chain,
+      this.idleWait
+    )
+
+    let liveHeaders: BlockHeader[] = []
+    if (ok) {
+      liveHeaders = await this.storage().addBulkHeaders(oldHeaders, bulkRange, priorLiveHeaders)
+    }
+
+    if (errors.length > 0) {
+      const errorMessages = errors.map(e => `(${e.code}) ${e.message} (${e.count})`).join('\n')
+      logger(`Errors during WhatsOnChain ingestion:\n${errorMessages}`)
+    }
+
+    return liveHeaders
+  }
+}

package/src/services/chaintracker/chaintracks/Ingest/LiveIngestorBase.ts

@@ -0,0 +1,84 @@
+/* eslint-disable @typescript-eslint/no-empty-function */
+import { Chain } from '../../../../sdk/types'
+import { LiveIngestorApi } from '../Api/LiveIngestorApi'
+import { ChaintracksStorageApi } from '../Api/ChaintracksStorageApi'
+import { BlockHeader } from '../Api/BlockHeaderApi'
+
+export interface LiveIngestorBaseOptions {
+  /**
+   * The target chain: "main" or "test"
+   */
+  chain: Chain
+}
+
+/**
+ *
+ */
+export abstract class LiveIngestorBase implements LiveIngestorApi {
+  static createLiveIngestorBaseOptions(chain: Chain) {
+    const options: LiveIngestorBaseOptions = {
+      chain
+    }
+    return options
+  }
+
+  chain: Chain
+
+  constructor(options: LiveIngestorBaseOptions) {
+    this.chain = options.chain
+  }
+
+  /**
+   * Release resources.
+   * Override if required.
+   */
+  async shutdown(): Promise<void> {}
+
+  private storageEngine?: ChaintracksStorageApi
+
+  /**
+   * Allocate resources.
+   * @param storage coordinating storage engine.
+   */
+  async setStorage(storage: ChaintracksStorageApi): Promise<void> {
+    this.storageEngine = storage
+  }
+
+  /**
+   *
+   * @returns coordinating storage engine.
+   */
+  storage(): ChaintracksStorageApi {
+    if (!this.storageEngine) throw new Error('storageEngine must be set.')
+    return this.storageEngine
+  }
+
+  /**
+   * Called to retrieve a missing block header,
+   * when the previousHash of a new header is unknown.
+   *
+   * @param hash block hash of missing header
+   */
+  abstract getHeaderByHash(hash: string): Promise<BlockHeader | undefined>
+
+  /**
+   * Begin retrieving new block headers.
+   *
+   * New headers are pushed onto the liveHeaders array.
+   *
+   * Continue waiting for new headers.
+   *
+   * Return only when either `stopListening` or `shutdown` are called.
+   *
+   * Be prepared to resume listening after `stopListening` but not
+   * after `shutdown`.
+   *
+   * @param liveHeaders
+   */
+  abstract startListening(liveHeaders: BlockHeader[]): Promise<void>
+
+  /**
+   * Causes `startListening` to stop listening for new block headers and return.
+   */
+  abstract stopListening(): void
+}

package/src/services/chaintracker/chaintracks/Ingest/LiveIngestorTeranodeP2P.ts

@@ -0,0 +1,59 @@
+import { BlockHeader, Chain } from '../../../../sdk'
+import { LiveIngestorBase, LiveIngestorBaseOptions } from './LiveIngestorBase'
+import { EnqueueHandler, ErrorHandler, WhatsOnChainServices, WhatsOnChainServicesOptions } from './WhatsOnChainServices'
+
+export interface LiveIngestorTeranodeP2POptions extends LiveIngestorBaseOptions {
+  /**
+   *
+   */
+  apiKey?: string
+  /**
+   * User-Agent header value for requests
+   */
+  userAgent?: string
+}
+
+export class LiveIngestorTeranodeP2P extends LiveIngestorBase {
+  static createLiveIngestorTeranodeP2POptions(chain: Chain): LiveIngestorTeranodeP2POptions {
+    const options: LiveIngestorTeranodeP2POptions = {
+      ...LiveIngestorBase.createLiveIngestorBaseOptions(chain)
+    }
+    return options
+  }
+
+  constructor(options: LiveIngestorTeranodeP2POptions) {
+    super(options)
+  }
+
+  async getHeaderByHash(hash: string): Promise<BlockHeader | undefined> {
+    return undefined
+  }
+
+  async startListening(liveHeaders: BlockHeader[]): Promise<void> {
+    const errors: { code: number; message: string; count: number }[] = []
+    const enqueue: EnqueueHandler = header => {
+      liveHeaders.push(header)
+    }
+    const error: ErrorHandler = (code, message) => {
+      errors.push({ code, message, count: errors.length })
+      return false
+    }
+
+    for (;;) {
+      const ok = true // await this.woc.listenForNewBlockHeaders(enqueue, error, this.idleWait)
+
+      if (!ok || errors.length > 0) {
+        console.log(`TeranodeP2P live ingestor ok=${ok} error count=${errors.length}`)
+        for (const e of errors) console.log(`TeranodeP2P error code=${e.code} count=${e.count} message=${e.message}`)
+      }
+
+      if (ok) break
+
+      errors.length = 0
+    }
+  }
+
+  stopListening(): void {
+    //this.woc?.stopNewListener()
+  }
+}

package/src/services/chaintracker/chaintracks/Ingest/LiveIngestorWhatsOnChainPoll.ts

@@ -0,0 +1,104 @@
+import { BlockHeader, Chain } from '../../../../sdk'
+import { wait } from '../../../../utility/utilityHelpers'
+import { LiveIngestorBase, LiveIngestorBaseOptions } from './LiveIngestorBase'
+import {
+  WhatsOnChainServices,
+  WhatsOnChainServicesOptions,
+  WocGetHeadersHeader,
+  wocGetHeadersHeaderToBlockHeader
+} from './WhatsOnChainServices'
+
+export interface LiveIngestorWhatsOnChainOptions extends LiveIngestorBaseOptions, WhatsOnChainServicesOptions {
+  /**
+   * Maximum msces of "normal" time with no ping received from connected WoC service.
+   */
+  idleWait: number | undefined
+  /**
+   * Which chain is being tracked: main, test, or stn.
+   */
+  chain: Chain
+  /**
+   * WhatsOnChain.com API Key
+   * https://docs.taal.com/introduction/get-an-api-key
+   * If unknown or empty, maximum request rate is limited.
+   * https://developers.whatsonchain.com/#rate-limits
+   */
+  apiKey?: string
+  /**
+   * Request timeout for GETs to https://api.whatsonchain.com/v1/bsv
+   */
+  timeout: number
+  /**
+   * User-Agent header value for requests to https://api.whatsonchain.com/v1/bsv
+   */
+  userAgent: string
+  /**
+   * Enable WhatsOnChain client cache option.
+   */
+  enableCache: boolean
+  /**
+   * How long chainInfo is considered still valid before updating (msecs).
+   */
+  chainInfoMsecs: number
+}
+
+/**
+ * Reports new headers by polling periodically.
+ */
+export class LiveIngestorWhatsOnChainPoll extends LiveIngestorBase {
+  static createLiveIngestorWhatsOnChainOptions(chain: Chain): LiveIngestorWhatsOnChainOptions {
+    const options: LiveIngestorWhatsOnChainOptions = {
+      ...WhatsOnChainServices.createWhatsOnChainServicesOptions(chain),
+      ...LiveIngestorBase.createLiveIngestorBaseOptions(chain),
+      idleWait: 100000
+    }
+    return options
+  }
+
+  idleWait: number
+  woc: WhatsOnChainServices
+  done: boolean = false
+
+  constructor(options: LiveIngestorWhatsOnChainOptions) {
+    super(options)
+    this.idleWait = options.idleWait || 100000
+    this.woc = new WhatsOnChainServices(options)
+  }
+
+  async getHeaderByHash(hash: string): Promise<BlockHeader | undefined> {
+    const header = await this.woc.getHeaderByHash(hash)
+    return header
+  }
+
+  async startListening(liveHeaders: BlockHeader[]): Promise<void> {
+    this.done = false
+    let lastHeaders: WocGetHeadersHeader[] = []
+
+    for (; !this.done; ) {
+      const headers = await this.woc.getHeaders()
+
+      const newHeaders = headers.filter(h => !lastHeaders.some(lh => lh.hash === h.hash))
+
+      for (const h of newHeaders) {
+        const bh = wocGetHeadersHeaderToBlockHeader(h)
+        liveHeaders.unshift(bh)
+      }
+
+      lastHeaders = headers
+
+      for (let sec = 0; sec < 60 && !this.done; sec++) {
+        // Only wait a second at a time so we notice `done` sooner...
+        await wait(1000)
+      }
+    }
+    console.log(`LiveIngestorWhatsOnChainPoll stopped`)
+  }
+
+  stopListening(): void {
+    this.done = true
+  }
+
+  override async shutdown(): Promise<void> {
+    this.stopListening()
+  }
+}

package/src/services/chaintracker/chaintracks/Ingest/LiveIngestorWhatsOnChainWs.ts

@@ -0,0 +1,59 @@
+import { BlockHeader, Chain } from '../../../../sdk'
+import { LiveIngestorBase } from './LiveIngestorBase'
+import { LiveIngestorWhatsOnChainOptions } from './LiveIngestorWhatsOnChainPoll'
+import { StopListenerToken, WocHeadersLiveListener } from './WhatsOnChainIngestorWs'
+import { EnqueueHandler, ErrorHandler, WhatsOnChainServices } from './WhatsOnChainServices'
+
+export class LiveIngestorWhatsOnChainWs extends LiveIngestorBase {
+  static createLiveIngestorWhatsOnChainOptions(chain: Chain): LiveIngestorWhatsOnChainOptions {
+    const options: LiveIngestorWhatsOnChainOptions = {
+      ...WhatsOnChainServices.createWhatsOnChainServicesOptions(chain),
+      ...LiveIngestorBase.createLiveIngestorBaseOptions(chain),
+      idleWait: 100000
+    }
+    return options
+  }
+
+  idleWait: number
+  woc: WhatsOnChainServices
+  stopNewListenersToken: StopListenerToken = { stop: undefined }
+
+  constructor(options: LiveIngestorWhatsOnChainOptions) {
+    super(options)
+    this.idleWait = options.idleWait || 100000
+    this.woc = new WhatsOnChainServices(options)
+  }
+
+  async getHeaderByHash(hash: string): Promise<BlockHeader | undefined> {
+    const header = await this.woc.getHeaderByHash(hash)
+    return header
+  }
+
+  async startListening(liveHeaders: BlockHeader[]): Promise<void> {
+    const errors: { code: number; message: string; count: number }[] = []
+    const enqueue: EnqueueHandler = header => {
+      liveHeaders.push(header)
+    }
+    const error: ErrorHandler = (code, message) => {
+      errors.push({ code, message, count: errors.length })
+      return false
+    }
+
+    for (;;) {
+      const ok = await WocHeadersLiveListener(enqueue, error, this.stopNewListenersToken, this.chain, this.idleWait)
+
+      if (!ok || errors.length > 0) {
+        console.log(`WhatsOnChain live ingestor ok=${ok} error count=${errors.length}`)
+        for (const e of errors) console.log(`WhatsOnChain error code=${e.code} count=${e.count} message=${e.message}`)
+      }
+
+      if (ok) break
+
+      errors.length = 0
+    }
+  }
+
+  stopListening(): void {
+    this.stopNewListenersToken.stop?.()
+  }
+}