@bopen-io/wallet-toolbox 1.7.18

package/src/services/chaintracker/chaintracks/Storage/ChaintracksStorageNoDb.ts

@@ -0,0 +1,304 @@
+import { ChaintracksStorageBaseOptions, InsertHeaderResult } from '../Api/ChaintracksStorageApi'
+import { ChaintracksStorageBase } from '../Storage/ChaintracksStorageBase'
+import { LiveBlockHeader } from '../Api/BlockHeaderApi'
+import { addWork, convertBitsToWork, isMoreWork, serializeBaseBlockHeader } from '../util/blockHeaderUtilities'
+import { HeightRange } from '../util/HeightRange'
+import { BulkFilesReaderStorage } from '../util/BulkFilesReader'
+import { ChaintracksFetch } from '../util/ChaintracksFetch'
+import { Chain } from '../../../../sdk/types'
+import { WERR_INVALID_OPERATION, WERR_INVALID_PARAMETER } from '../../../../sdk/WERR_errors'
+import { BlockHeader } from '../../../../sdk/WalletServices.interfaces'
+
+interface ChaintracksNoDbData {
+  chain: Chain
+  liveHeaders: Map<number, LiveBlockHeader>
+  maxHeaderId: number
+  tipHeaderId: number
+  hashToHeaderId: Map<string, number>
+}
+
+export interface ChaintracksStorageNoDbOptions extends ChaintracksStorageBaseOptions {}
+
+export class ChaintracksStorageNoDb extends ChaintracksStorageBase {
+  static mainData: ChaintracksNoDbData = {
+    chain: 'main',
+    liveHeaders: new Map<number, LiveBlockHeader>(),
+    maxHeaderId: 0,
+    tipHeaderId: 0,
+    hashToHeaderId: new Map<string, number>()
+  }
+  static testData: ChaintracksNoDbData = {
+    chain: 'test',
+    liveHeaders: new Map<number, LiveBlockHeader>(),
+    maxHeaderId: 0,
+    tipHeaderId: 0,
+    hashToHeaderId: new Map<string, number>()
+  }
+
+  constructor(options: ChaintracksStorageNoDbOptions) {
+    super(options)
+  }
+
+  override async destroy(): Promise<void> {}
+
+  async getData(): Promise<ChaintracksNoDbData> {
+    if (this.chain === 'main') {
+      return ChaintracksStorageNoDb.mainData
+    } else if (this.chain === 'test') {
+      return ChaintracksStorageNoDb.testData
+    } else {
+      throw new WERR_INVALID_PARAMETER('chain', `either 'main' or 'test. '${this.chain}' is unsupported.`)
+    }
+  }
+
+  override async deleteLiveBlockHeaders(): Promise<void> {
+    const data = await this.getData()
+    data.liveHeaders.clear()
+    data.maxHeaderId = 0
+    data.tipHeaderId = 0
+    data.hashToHeaderId.clear()
+  }
+
+  override async deleteOlderLiveBlockHeaders(maxHeight: number): Promise<number> {
+    const data = await this.getData()
+    let deletedCount = 0
+
+    // Clear previousHeaderId references
+    for (const [headerId, header] of data.liveHeaders) {
+      if (header.previousHeaderId) {
+        const prevHeader = data.liveHeaders.get(header.previousHeaderId)
+        if (prevHeader && prevHeader.height <= maxHeight) {
+          data.liveHeaders.set(headerId, { ...header, previousHeaderId: null })
+        }
+      }
+    }
+
+    // Delete headers up to maxHeight
+    const headersToDelete = new Set<number>()
+    for (const [headerId, header] of data.liveHeaders) {
+      if (header.height <= maxHeight) {
+        headersToDelete.add(headerId)
+        data.hashToHeaderId.delete(header.hash)
+      }
+    }
+    deletedCount = headersToDelete.size
+    for (const headerId of headersToDelete) {
+      data.liveHeaders.delete(headerId)
+    }
+
+    // Update tipHeaderId if necessary
+    if (data.liveHeaders.size > 0) {
+      const tip = Array.from(data.liveHeaders.values()).find(h => h.isActive && h.isChainTip)
+      data.tipHeaderId = tip ? tip.headerId : 0
+    } else {
+      data.tipHeaderId = 0
+    }
+
+    return deletedCount
+  }
+
+  override async findChainTipHeader(): Promise<LiveBlockHeader> {
+    const data = await this.getData()
+    const tip = Array.from(data.liveHeaders.values()).find(h => h.isActive && h.isChainTip)
+    if (!tip) throw new Error('Database contains no active chain tip header.')
+    return tip
+  }
+
+  override async findChainTipHeaderOrUndefined(): Promise<LiveBlockHeader | undefined> {
+    const data = await this.getData()
+    return Array.from(data.liveHeaders.values()).find(h => h.isActive && h.isChainTip)
+  }
+
+  override async findLiveHeaderForBlockHash(hash: string): Promise<LiveBlockHeader | null> {
+    const data = await this.getData()
+    const headerId = data.hashToHeaderId.get(hash)
+    return headerId ? data.liveHeaders.get(headerId) || null : null
+  }
+
+  override async findLiveHeaderForHeaderId(headerId: number): Promise<LiveBlockHeader> {
+    const data = await this.getData()
+    const header = data.liveHeaders.get(headerId)
+    if (!header) throw new Error(`HeaderId ${headerId} not found in live header database.`)
+    return header
+  }
+
+  override async findLiveHeaderForHeight(height: number): Promise<LiveBlockHeader | null> {
+    const data = await this.getData()
+    return Array.from(data.liveHeaders.values()).find(h => h.height === height && h.isActive) || null
+  }
+
+  override async findLiveHeaderForMerkleRoot(merkleRoot: string): Promise<LiveBlockHeader | null> {
+    const data = await this.getData()
+    return Array.from(data.liveHeaders.values()).find(h => h.merkleRoot === merkleRoot) || null
+  }
+
+  override async findLiveHeightRange(): Promise<HeightRange> {
+    const data = await this.getData()
+    const activeHeaders = Array.from(data.liveHeaders.values()).filter(h => h.isActive)
+    if (activeHeaders.length === 0) {
+      return HeightRange.empty
+    }
+    const minHeight = Math.min(...activeHeaders.map(h => h.height))
+    const maxHeight = Math.max(...activeHeaders.map(h => h.height))
+    return new HeightRange(minHeight, maxHeight)
+  }
+
+  override async findMaxHeaderId(): Promise<number> {
+    const data = await this.getData()
+    return data.maxHeaderId
+  }
+
+  override async liveHeadersForBulk(count: number): Promise<LiveBlockHeader[]> {
+    const data = await this.getData()
+    return Array.from(data.liveHeaders.values())
+      .filter(h => h.isActive)
+      .sort((a, b) => a.height - b.height)
+      .slice(0, count)
+  }
+
+  override async getLiveHeaders(range: HeightRange): Promise<LiveBlockHeader[]> {
+    if (range.isEmpty) return []
+    const data = await this.getData()
+    const headers = Array.from(data.liveHeaders.values())
+      .filter(h => h.isActive && h.height >= range.minHeight && h.height <= range.maxHeight)
+      .sort((a, b) => a.height - b.height)
+    return headers
+  }
+
+  override async insertHeader(header: BlockHeader): Promise<InsertHeaderResult> {
+    const data = await this.getData()
+
+    const r: InsertHeaderResult = {
+      added: false,
+      dupe: false,
+      noPrev: false,
+      badPrev: false,
+      noActiveAncestor: false,
+      isActiveTip: false,
+      reorgDepth: 0,
+      priorTip: undefined,
+      noTip: false,
+      deactivatedHeaders: []
+    }
+
+    // Check for duplicate
+    if (data.hashToHeaderId.has(header.hash)) {
+      r.dupe = true
+      return r
+    }
+
+    // Find previous header
+    let oneBack = Array.from(data.liveHeaders.values()).find(h => h.hash === header.previousHash)
+
+    if (!oneBack) {
+      // Check if this is first live header
+      const count = data.liveHeaders.size
+      if (count === 0) {
+        // If this is the first live header, the last bulk header (if there is one) is the previous header.
+        const lbf = await this.bulkManager.getLastFile()
+        if (!lbf) throw new WERR_INVALID_OPERATION('bulk headers must exist before first live header can be added')
+        if (header.previousHash === lbf.lastHash && header.height === lbf.firstHeight + lbf.count) {
+          // Valid first live header. Add it.
+          const chainWork = addWork(lbf.lastChainWork, convertBitsToWork(header.bits))
+          r.isActiveTip = true
+          const newHeader = {
+            ...header,
+            headerId: ++data.maxHeaderId,
+            previousHeaderId: null,
+            chainWork,
+            isChainTip: r.isActiveTip,
+            isActive: r.isActiveTip
+          }
+          data.liveHeaders.set(newHeader.headerId, newHeader)
+          data.hashToHeaderId.set(header.hash, newHeader.headerId)
+          data.tipHeaderId = newHeader.headerId
+          r.added = true
+          return r
+        }
+      }
+      // Failure without a oneBack
+      // First live header that does not follow last bulk header or
+      // Not the first live header and live headers doesn't include a previousHash header.
+      r.noPrev = true
+      return r
+    }
+
+    // This header's previousHash matches an existing live header's hash, if height isn't +1, reject it.
+    if (oneBack.height + 1 !== header.height) {
+      r.badPrev = true
+      return r
+    }
+
+    r.priorTip =
+      oneBack.isActive && oneBack.isChainTip
+        ? oneBack
+        : Array.from(data.liveHeaders.values()).find(h => h.isActive && h.isChainTip)
+
+    if (!r.priorTip) {
+      // No active chain tip found. This is a logic error in state of live headers.
+      r.noTip = true
+      return r
+    }
+
+    // We have an acceptable new live header...and live headers has an active chain tip.
+
+    const chainWork = addWork(oneBack.chainWork, convertBitsToWork(header.bits))
+
+    r.isActiveTip = isMoreWork(chainWork, r.priorTip.chainWork)
+
+    const newHeader = {
+      ...header,
+      headerId: ++data.maxHeaderId,
+      previousHeaderId: oneBack.headerId,
+      chainWork,
+      isChainTip: r.isActiveTip,
+      isActive: r.isActiveTip
+    }
+
+    if (r.isActiveTip) {
+      let activeAncestor = oneBack
+      while (!activeAncestor.isActive) {
+        const previousHeader = data.liveHeaders.get(activeAncestor.previousHeaderId!)
+        if (!previousHeader) {
+          r.noActiveAncestor = true
+          return r
+        }
+        activeAncestor = previousHeader
+      }
+
+      if (!(oneBack.isActive && oneBack.isChainTip)) {
+        r.reorgDepth = Math.min(r.priorTip.height, header.height) - activeAncestor.height
+      }
+
+      if (activeAncestor.headerId !== oneBack.headerId) {
+        let headerToDeactivate = Array.from(data.liveHeaders.values()).find(h => h.isChainTip && h.isActive)
+        while (headerToDeactivate && headerToDeactivate.headerId !== activeAncestor.headerId) {
+          r.deactivatedHeaders.push(headerToDeactivate)
+          data.liveHeaders.set(headerToDeactivate.headerId, { ...headerToDeactivate, isActive: false })
+          headerToDeactivate = data.liveHeaders.get(headerToDeactivate.previousHeaderId!)
+        }
+
+        let headerToActivate = oneBack
+        while (headerToActivate.headerId !== activeAncestor.headerId) {
+          data.liveHeaders.set(headerToActivate.headerId, { ...headerToActivate, isActive: true })
+          headerToActivate = data.liveHeaders.get(headerToActivate.previousHeaderId!)!
+        }
+      }
+    }
+
+    if (oneBack.isChainTip) {
+      data.liveHeaders.set(oneBack.headerId, { ...oneBack, isChainTip: false })
+    }
+
+    data.liveHeaders.set(newHeader.headerId, newHeader)
+    data.hashToHeaderId.set(newHeader.hash, newHeader.headerId)
+    r.added = true
+
+    if (r.added && r.isActiveTip) {
+      data.tipHeaderId = newHeader.headerId
+      this.pruneLiveBlockHeaders(newHeader.height)
+    }
+
+    return r
+  }
+}

package/src/services/chaintracker/chaintracks/Storage/__tests/ChaintracksStorageIdb.test.ts

@@ -0,0 +1,102 @@
+import 'fake-indexeddb/auto'
+import { ChaintracksFs } from '../../util/ChaintracksFs'
+import { Chain } from '../../../../../sdk'
+import { ChaintracksStorageKnex } from '../ChaintracksStorageKnex'
+import { deserializeBaseBlockHeader, genesisHeader } from '../../util/blockHeaderUtilities'
+import { ChaintracksStorageIdb, ChaintracksStorageIdbOptions } from '../ChaintracksStorageIdb'
+import { ChaintracksStorageBase } from '../ChaintracksStorageBase'
+import { LiveBlockHeader } from '../../Api/BlockHeaderApi'
+
+describe('ChaintracksStorageIdb tests', () => {
+  jest.setTimeout(99999999)
+
+  test('0', async () => {
+    const options: ChaintracksStorageIdbOptions = ChaintracksStorageBase.createStorageBaseOptions('main')
+    const storage = new ChaintracksStorageIdb(options)
+    const r = await storage.migrateLatest()
+    const db = storage.db!
+    expect(db).toBeTruthy()
+
+    const tip = await storage.findChainTipHeaderOrUndefined()
+    expect(tip).toBeUndefined()
+
+    const ranges = await storage.getAvailableHeightRanges()
+
+    const lh: LiveBlockHeader = {
+      headerId: 0,
+      chainWork: '00'.repeat(32),
+      isChainTip: false,
+      isActive: true,
+      previousHeaderId: null,
+      height: ranges.bulk.maxHeight + 1,
+      hash: '1111',
+      version: 0,
+      previousHash: '01'.repeat(32),
+      merkleRoot: '11'.repeat(32),
+      time: 0,
+      bits: 0,
+      nonce: 0
+    }
+    await storage.insertLiveHeader(lh)
+    lh.previousHeaderId = lh.headerId
+    lh.hash = '2222'
+    lh.merkleRoot = '22'.repeat(32)
+    lh.height++
+    await storage.insertLiveHeader(lh)
+    lh.previousHeaderId = lh.headerId
+    lh.hash = '3333'
+    lh.merkleRoot = '33'.repeat(32)
+    lh.height++
+    await storage.insertLiveHeader(lh)
+    lh.previousHeaderId = lh.headerId
+    lh.hash = '4444'
+    lh.height++
+    await storage.insertLiveHeader(lh)
+    lh.previousHeaderId = lh.headerId
+    lh.hash = '5555'
+    lh.height++
+    lh.isChainTip = true
+    await storage.insertLiveHeader(lh)
+
+    const h1 = await storage.findLiveHeaderForBlockHash('1111')
+    expect(h1!.headerId).toBe(1)
+
+    const h2 = await storage.findLiveHeaderForMerkleRoot('22'.repeat(32))
+    expect(h2!.headerId).toBe(2)
+
+    const h3 = await storage.findLiveHeaderForHeaderId(3)
+    expect(h3!.headerId).toBe(3)
+
+    const h4 = await storage.findLiveHeaderForHeight(4 + ranges.bulk.maxHeight)
+    expect(h4!.headerId).toBe(4)
+
+    const h5 = await storage.findChainTipHeader()
+    expect(h5.headerId).toBe(5)
+
+    const range = await storage.findLiveHeightRange()
+    expect(range).toEqual({ minHeight: 1 + ranges.bulk.maxHeight, maxHeight: 5 + ranges.bulk.maxHeight })
+
+    const maxId = await storage.findMaxHeaderId()
+    expect(maxId).toBe(5)
+
+    const hfbs = await storage.liveHeadersForBulk(3)
+    expect(hfbs.length).toBe(3)
+
+    const lhs = await storage.getHeaders(0, 10)
+    expect(lhs.length).toBe(10)
+
+    const lhs2 = await storage.getHeaders(0 + ranges.bulk.maxHeight + 1, 10)
+    expect(lhs2.length).toBe(5)
+
+    const lhs3 = await storage.getHeaders(0 + ranges.bulk.maxHeight - 2, 10)
+    expect(lhs3.length).toBe(8)
+
+    const data = await storage.getHeadersUint8Array(0, 10)
+    expect(data.length).toBe(10 * 80)
+
+    const deleteCount = await storage.deleteOlderLiveBlockHeaders(3 + ranges.bulk.maxHeight)
+    expect(deleteCount).toBe(3)
+
+    await storage.deleteLiveBlockHeaders()
+  })
+})

package/src/services/chaintracker/chaintracks/Storage/__tests/ChaintracksStorageKnex.test.ts

@@ -0,0 +1,45 @@
+import { Knex, knex as makeKnex } from 'knex'
+import { ChaintracksFs } from '../../util/ChaintracksFs'
+import { Chain } from '../../../../../sdk'
+import { ChaintracksStorageKnex } from '../ChaintracksStorageKnex'
+import { deserializeBaseBlockHeader, genesisHeader } from '../../util/blockHeaderUtilities'
+
+describe('ChaintracksStorageKnex tests', () => {
+  jest.setTimeout(99999999)
+
+  test('0', async () => {
+    const chain: Chain = 'main'
+    const fs = ChaintracksFs
+    const rootFolder = './src/services/chaintracker/chaintracks/__tests/data'
+    const localSqlite: Knex.Config = {
+      client: 'sqlite3',
+      connection: { filename: fs.pathJoin(rootFolder, `${chain}Net_chaintracks.sqlite`) },
+      useNullAsDefault: true
+    }
+
+    const knexInstance = makeKnex(localSqlite)
+
+    const knexOptions = ChaintracksStorageKnex.createStorageKnexOptions(chain)
+    knexOptions.knex = knexInstance
+    const storage = new ChaintracksStorageKnex(knexOptions)
+    await storage.makeAvailable()
+
+    const bfs = await storage.bulkManager.getBulkFiles()
+    // Test assumes synchronization has occurred and bulk files are available.
+    if (bfs?.length === 0) return
+
+    expect(bfs.length).toBeGreaterThan(7)
+
+    const gh = await storage.getBulkFileData(bfs[0].fileId!, 0, 80)
+    const dgh = deserializeBaseBlockHeader(gh!)
+    const rgh = genesisHeader(chain)
+    expect(dgh.merkleRoot).toEqual(rgh.merkleRoot)
+    expect(dgh.bits).toEqual(rgh.bits)
+    expect(dgh.nonce).toEqual(rgh.nonce)
+
+    const header = await storage.findHeaderForHeight(101010)
+    expect(header.hash).toEqual('000000000001af33247fff33aae7c31baee4148d5a189e7353bf13bcee618202')
+
+    await storage.shutdown()
+  })
+})

package/src/services/chaintracker/chaintracks/__tests/Chaintracks.test.ts

@@ -0,0 +1,77 @@
+import { createDefaultKnexChaintracksOptions } from '../createDefaultKnexChaintracksOptions'
+import { Chaintracks } from '../Chaintracks'
+import { wait } from '../../../../utility/utilityHelpers'
+import { Chain } from '../../../../sdk'
+import { createDefaultNoDbChaintracksOptions } from '../createDefaultNoDbChaintracksOptions'
+import { ChaintracksFs } from '../util/ChaintracksFs'
+import { LocalCdnServer } from './LocalCdnServer'
+import { _tu } from '../../../../../test/utils/TestUtilsWalletStorage'
+
+const rootFolder = './src/services/chaintracker/chaintracks/__tests/data'
+
+describe('Chaintracks tests', () => {
+  jest.setTimeout(99999999)
+
+  test.skip('0 basic operation mainnet', async () => {
+    const o = createDefaultKnexChaintracksOptions('main', rootFolder)
+    const c = new Chaintracks(o)
+    await c.makeAvailable()
+
+    let done = false
+    for (; !done; ) {
+      await wait(10000)
+    }
+
+    await c.destroy()
+  })
+
+  test('1 NoDb mainnet', async () => {
+    if (_tu.noEnv('main')) return
+    await NoDbBody('main')
+  })
+
+  test('2 NoDb testnet', async () => {
+    if (_tu.noEnv('main')) return
+    await NoDbBody('test')
+  })
+
+  test.skip('3 NoDb export mainnet', async () => {
+    await NoDbBody('main', true)
+  })
+
+  test.skip('4 NoDb export testnet', async () => {
+    await NoDbBody('test', true)
+  })
+
+  test.skip('5 run local CDN on port 8300', async () => {
+    const fs = ChaintracksFs
+    const server = new LocalCdnServer(8300, fs.pathJoin(rootFolder, 'export'))
+    await server.start()
+    let done = false
+    for (; !done; ) {
+      await wait(10000)
+    }
+    await server.stop()
+  })
+
+  async function NoDbBody(chain: Chain, exportHeaders?: boolean) {
+    const o = createDefaultNoDbChaintracksOptions(chain)
+    const c = new Chaintracks(o)
+    await c.makeAvailable()
+
+    c.subscribeHeaders(header => {
+      console.log(`Header received: ${header.height} ${header.hash}`)
+    })
+
+    if (exportHeaders) {
+      const rootFolder = './src/services/chaintracker/chaintracks/__tests/data/export'
+      await c.exportBulkHeaders(rootFolder, ChaintracksFs, `https://cdn.projectbabbage.com/blockheaders`, 100000)
+    }
+    //let done = false
+    //for (; !done; ) {
+    await wait(1000)
+    //}
+
+    await c.destroy()
+  }
+})