@bsv/wallet-toolbox 1.5.21 → 1.6.1

package/src/services/chaintracker/chaintracks/util/ChaintracksFs.ts

@@ -0,0 +1,137 @@
+import {
+  ChaintracksAppendableFileApi,
+  ChaintracksFsApi,
+  ChaintracksReadableFileApi,
+  ChaintracksWritableFileApi
+} from '../Api/ChaintracksFsApi'
+import { promises as fs } from 'fs'
+import Path from 'path'
+
+export abstract class ChaintracksFsStatics {
+  static async delete(path: string): Promise<void> {
+    await fs.unlink(path)
+  }
+  static async writeFile(path: string, data: Uint8Array): Promise<void> {
+    await this.ensureFoldersExist(path)
+    await fs.writeFile(path, Buffer.from(data))
+  }
+  static async readFile(path: string): Promise<Uint8Array> {
+    const buffer = await fs.readFile(path)
+    return Uint8Array.from(buffer)
+  }
+  static async openReadableFile(path: string): Promise<ChaintracksReadableFileApi> {
+    return await ChaintracksReadableFile.openAsReadable(path)
+  }
+  static async openWritableFile(path: string): Promise<ChaintracksWritableFileApi> {
+    return await ChaintracksWritableFile.openAsWritable(path)
+  }
+
+  static async openAppendableFile(path: string): Promise<ChaintracksAppendableFileApi> {
+    return await ChaintracksAppendableFile.openAsAppendable(path)
+  }
+  static async ensureFoldersExist(path: string): Promise<void> {
+    const parsedPath = Path.parse(path)
+    await fs.mkdir(parsedPath.dir, { recursive: true })
+  }
+  static pathJoin(...parts: string[]): string {
+    return Path.join(...parts)
+  }
+}
+
+export const ChaintracksFs: ChaintracksFsApi = ChaintracksFsStatics
+
+export class ChaintracksReadableFile implements ChaintracksReadableFileApi {
+  path: string
+  parsedPath: Path.ParsedPath
+  f: fs.FileHandle
+
+  protected constructor(path: string, f: fs.FileHandle) {
+    this.path = path
+    this.f = f
+    this.parsedPath = Path.parse(path)
+  }
+
+  async close(): Promise<void> {
+    await this.f.close()
+  }
+
+  async getLength(): Promise<number> {
+    const stats = await this.f.stat()
+    return stats.size
+  }
+
+  async read(length?: number, offset?: number): Promise<Uint8Array> {
+    length ||= 80 * 1024 // Default to 80KB if no length is specified
+    const buffer = Buffer.alloc(length)
+    const rr = await this.f.read(buffer, 0, length, offset || 0)
+    const rb = rr.bytesRead < length ? buffer.subarray(0, rr.bytesRead) : buffer
+    return Uint8Array.from(rb)
+  }
+
+  static async openAsReadable(path: string): Promise<ChaintracksReadableFile> {
+    const f = await fs.open(path, 'r')
+    const file = new ChaintracksReadableFile(path, f)
+    return file
+  }
+}
+
+export class ChaintracksWritableFile implements ChaintracksWritableFileApi {
+  path: string
+  parsedPath: Path.ParsedPath
+  f: fs.FileHandle
+  foldersEnsured: boolean = false
+
+  private constructor(path: string, f: fs.FileHandle) {
+    this.path = path
+    this.f = f
+    this.parsedPath = Path.parse(path)
+  }
+
+  static async openAsWritable(path: string): Promise<ChaintracksWritableFile> {
+    const f = await fs.open(path, 'w')
+    const file = new ChaintracksWritableFile(path, f)
+    return file
+  }
+
+  async close(): Promise<void> {
+    await this.f.close()
+  }
+
+  async ensureFoldersExist(): Promise<void> {
+    if (!this.foldersEnsured) {
+      await ChaintracksFsStatics.ensureFoldersExist(this.path)
+      this.foldersEnsured = true
+    }
+  }
+
+  async append(data: Uint8Array): Promise<void> {
+    await this.ensureFoldersExist()
+    throw new Error('Method not implemented.')
+  }
+}
+
+export class ChaintracksAppendableFile extends ChaintracksReadableFile implements ChaintracksAppendableFileApi {
+  foldersEnsured: boolean = false
+
+  private constructor(path: string, f: fs.FileHandle) {
+    super(path, f)
+  }
+
+  static async openAsAppendable(path: string): Promise<ChaintracksAppendableFile> {
+    const f = await fs.open(path, 'a+')
+    const file = new ChaintracksAppendableFile(path, f)
+    return file
+  }
+
+  async ensureFoldersExist(): Promise<void> {
+    if (!this.foldersEnsured) {
+      await ChaintracksFsStatics.ensureFoldersExist(this.path)
+      this.foldersEnsured = true
+    }
+  }
+
+  async append(data: Uint8Array): Promise<void> {
+    await this.ensureFoldersExist()
+    await this.f.write(Buffer.from(data))
+  }
+}

package/src/services/chaintracker/chaintracks/util/HeightRange.ts

@@ -0,0 +1,132 @@
+import { BlockHeader } from '../../../../sdk/WalletServices.interfaces'
+
+export interface HeightRangeApi {
+  minHeight: number
+  maxHeight: number
+}
+
+export interface HeightRanges {
+  bulk: HeightRange
+  live: HeightRange
+}
+
+export class HeightRange implements HeightRangeApi {
+  constructor(
+    public minHeight: number,
+    public maxHeight: number
+  ) {}
+
+  static readonly empty = new HeightRange(0, -1)
+
+  /**
+   * @param headers
+   * @returns range of height values from the given headers, or the empty range if there are no headers.
+   */
+  static from(headers: BlockHeader[]): HeightRange {
+    if (headers.length === 0) return HeightRange.empty
+    const minHeight = headers.reduce((min, h) => Math.min(min, h.height), headers[0].height)
+    const maxHeight = headers.reduce((max, h) => Math.max(max, h.height), headers[0].height)
+    return new HeightRange(minHeight, maxHeight)
+  }
+
+  get length() {
+    return Math.max(0, this.maxHeight - this.minHeight + 1)
+  }
+
+  get isEmpty() {
+    return this.minHeight > this.maxHeight
+  }
+
+  toString(): string {
+    return this.isEmpty ? '<empty>' : `${this.minHeight}-${this.maxHeight}`
+  }
+
+  /**
+   * @param range HeightRange or single height value.
+   * @returns true if `range` is entirely within this range.
+   */
+  contains(range: HeightRange | number) {
+    if (typeof range === 'number') {
+      return this.minHeight <= range && this.maxHeight >= range
+    }
+    return this.minHeight <= range.minHeight && this.maxHeight >= range.maxHeight
+  }
+
+  /**
+   * Return the intersection with another height range.
+   *
+   * Intersection with an empty range is always empty.
+   *
+   * The result is always a single, possibly empty, range.
+   * @param range
+   * @returns
+   */
+  intersect(range: HeightRange) {
+    return new HeightRange(Math.max(this.minHeight, range.minHeight), Math.min(this.maxHeight, range.maxHeight))
+  }
+
+  /**
+   * Return the union with another height range.
+   *
+   * Only valid if the two ranges overlap or touch, or one is empty.
+   *
+   * Throws an error if the union would create two disjoint ranges.
+   *
+   * @param range
+   * @returns
+   */
+  union(range: HeightRange) {
+    if (this.isEmpty) return range.copy()
+    if (range.isEmpty) return this.copy()
+    if (this.maxHeight + 1 < range.minHeight || range.maxHeight + 1 < this.minHeight)
+      throw new Error('Union of ranges with a gap between them is not supported.')
+    return new HeightRange(Math.min(this.minHeight, range.minHeight), Math.max(this.maxHeight, range.maxHeight))
+  }
+
+  /**
+   * Returns `range` subtracted from this range.
+   *
+   * Throws an error if the subtraction would create two disjoint ranges.
+   *
+   * @param range
+   * @returns
+   */
+  subtract(range: HeightRange) {
+    if (this.isEmpty || range.isEmpty) return this.copy()
+    if (this.minHeight < range.minHeight && this.maxHeight > range.maxHeight)
+      throw new Error('Subtraction of range that creates two disjoint ranges is not supported.')
+    if (range.maxHeight < this.minHeight || range.minHeight > this.maxHeight)
+      // Leave untouched. Subtracted is either all lower or all higher.
+      return this.copy()
+    if (range.minHeight <= this.minHeight && range.maxHeight < this.maxHeight)
+      // Remove a chunk on the low side.
+      return new HeightRange(range.maxHeight + 1, this.maxHeight)
+    if (range.minHeight <= this.minHeight && range.maxHeight >= this.maxHeight)
+      // Remove the whole thing
+      return new HeightRange(this.minHeight, this.minHeight - 1) // empty
+    if (range.minHeight <= this.maxHeight && range.maxHeight >= this.maxHeight)
+      // Remove a chunk on the high side.
+      return new HeightRange(this.minHeight, range.minHeight - 1)
+    throw new Error('All cases should have been handled :-) .')
+  }
+
+  /**
+   * If `range` is not empty and this is not empty, returns a new range minHeight
+   * replaced by to range.maxHeight + 1.
+   *
+   * Otherwise returns a copy of this range.
+   *
+   * This returns the portion of this range that is strictly above `range`.
+   */
+  above(range: HeightRange) {
+    if (range.isEmpty || this.isEmpty) return this.copy()
+    return new HeightRange(range.maxHeight + 1, this.maxHeight)
+  }
+
+  /**
+   * Return a copy of this range.
+   */
+  copy(): HeightRange {
+    return new HeightRange(this.minHeight, this.maxHeight)
+  }
+}

package/src/services/chaintracker/chaintracks/util/SingleWriterMultiReaderLock.ts

@@ -0,0 +1,76 @@
+/**
+ * A reader-writer lock to manage concurrent access.
+ * Allows multiple readers or one writer at a time.
+ */
+export class SingleWriterMultiReaderLock {
+  private readers: number = 0
+  private writerActive: boolean = false
+  private readerQueue: Array<() => void> = []
+  private writerQueue: Array<() => void> = []
+
+  private checkQueues(): void {
+    if (this.writerActive || this.readers > 0) return
+    if (this.writerQueue.length > 0) {
+      // If there are waiting writers and no active readers or writers, start the next writer
+      const resolve = this.writerQueue.shift()!
+      resolve()
+    } else if (this.readerQueue.length > 0) {
+      // If there are waiting readers and no waiting writers, start all readers
+      const readers = this.readerQueue.splice(0)
+      for (const resolve of readers) {
+        resolve()
+      }
+    }
+  }
+
+  async withReadLock<T>(fn: () => Promise<T>): Promise<T> {
+    if (!this.writerActive && this.writerQueue.length === 0) {
+      // Fast path: no active writer or waiting writers, proceed immediately
+      this.readers++
+      try {
+        return await fn()
+      } finally {
+        this.readers--
+        this.checkQueues()
+      }
+    } else {
+      // Queue the reader until writers are done
+      const promise = new Promise<void>(resolve => {
+        this.readerQueue.push(resolve)
+      })
+      await promise
+      this.readers++
+      try {
+        return await fn()
+      } finally {
+        this.readers--
+        this.checkQueues()
+      }
+    }
+  }
+
+  async withWriteLock<T>(fn: () => Promise<T>): Promise<T> {
+    if (!this.writerActive && this.readers === 0) {
+      // Fast path: no active writer or readers, proceed immediately
+      this.writerActive = true
+      try {
+        return await fn()
+      } finally {
+        this.writerActive = false
+        this.checkQueues()
+      }
+    } else {
+      const promise = new Promise<void>(resolve => {
+        this.writerQueue.push(resolve)
+      })
+      await promise
+      this.writerActive = true
+      try {
+        return await fn()
+      } finally {
+        this.writerActive = false
+        this.checkQueues()
+      }
+    }
+  }
+}

package/src/services/chaintracker/chaintracks/util/__tests/BulkFileDataManager.test.ts

@@ -0,0 +1,304 @@
+import { Knex, knex as makeKnex } from 'knex'
+import { ChaintracksStorageKnex } from '../../Storage/ChaintracksStorageKnex'
+import { deserializeBlockHeaders } from '../blockHeaderUtilities'
+import { BulkFileDataManager } from '../BulkFileDataManager'
+import { BulkHeaderFileInfo } from '../BulkHeaderFile'
+import { ChaintracksFs } from '../ChaintracksFs'
+import { LocalCdnServer } from '../../__tests/LocalCdnServer'
+import { Chain } from '../../../../../sdk/types'
+import { BlockHeader } from '../../../../../sdk/WalletServices.interfaces'
+
+const runSlowTests = false
+
+describe('BulkFileDataManager tests', () => {
+  jest.setTimeout(99999999)
+
+  const chain: Chain = 'main'
+  const fs = ChaintracksFs
+  const rootFolder = './src/services/chaintracker/chaintracks/__tests/data'
+  let headers300_399: BlockHeader[] = []
+  let headers400_499: BlockHeader[] = []
+  let server349: LocalCdnServer | undefined
+  let server379: LocalCdnServer | undefined
+  let server399: LocalCdnServer | undefined
+  let server402: LocalCdnServer | undefined
+  let server499: LocalCdnServer | undefined
+
+  beforeAll(async () => {
+    const data300_399 = await ChaintracksFs.readFile(fs.pathJoin(rootFolder, 'cdnTest499/mainNet_3.headers'))
+    const data400_499 = await ChaintracksFs.readFile(fs.pathJoin(rootFolder, 'cdnTest499/mainNet_4.headers'))
+    headers300_399 = deserializeBlockHeaders(300, data300_399)
+    headers400_499 = deserializeBlockHeaders(400, data400_499)
+
+    // Start the local CDN servers
+    server349 = new LocalCdnServer(8349, fs.pathJoin(rootFolder, 'cdnTest349'))
+    await server349.start()
+    server379 = new LocalCdnServer(8379, fs.pathJoin(rootFolder, 'cdnTest379'))
+    await server379.start()
+    server399 = new LocalCdnServer(8399, fs.pathJoin(rootFolder, 'cdnTest399'))
+    await server399.start()
+    server402 = new LocalCdnServer(8402, fs.pathJoin(rootFolder, 'cdnTest402'))
+    await server402.start()
+    server499 = new LocalCdnServer(8499, fs.pathJoin(rootFolder, 'cdnTest499'))
+    await server499.start()
+  })
+
+  afterAll(async () => {
+    if (server349) {
+      await server349.stop()
+    }
+    if (server379) {
+      await server379.stop()
+    }
+    if (server399) {
+      await server399.stop()
+    }
+    if (server402) {
+      await server402.stop()
+    }
+    if (server499) {
+      await server499.stop()
+    }
+  })
+
+  async function test0Body(manager: BulkFileDataManager) {
+    // Verify the default options and minimum expected files from default CDN
+    expect(manager.chain).toBe(chain)
+    expect(manager.maxPerFile).toBe(100000)
+    expect(manager.maxRetained).toBe(2)
+    expect(manager.fromKnownSourceUrl).toBe('https://cdn.projectbabbage.com/blockheaders')
+    const files = await manager.getBulkFiles()
+    expect(files.length).toBeGreaterThan(7)
+    const range = await manager.getHeightRange()
+    expect(range.minHeight).toBe(0)
+    expect(range.maxHeight).toBeGreaterThan(800000)
+  }
+
+  test('0 default options CDN files', async () => {
+    const options = BulkFileDataManager.createDefaultOptions(chain)
+    const manager = new BulkFileDataManager(options)
+
+    await test0Body(manager)
+  })
+
+  test('0a default options CDN files noDropAll', async () => {
+    if (!runSlowTests) return
+    const options = BulkFileDataManager.createDefaultOptions(chain)
+    const manager = new BulkFileDataManager(options)
+    const storage = await setupStorageKnex(manager, `BulkFileDataManager.test_0a`, false)
+
+    await test0Body(manager)
+
+    await storage.destroy()
+  })
+
+  test('0b default options CDN files dropAll', async () => {
+    if (!runSlowTests) return
+    const options = BulkFileDataManager.createDefaultOptions(chain)
+    const manager = new BulkFileDataManager(options)
+    const storage = await setupStorageKnex(manager, `BulkFileDataManager.test_0b`, true)
+
+    await test0Body(manager)
+
+    await storage.destroy()
+  })
+
+  async function test1Body(manager: BulkFileDataManager, minCount: number) {
+    // Verify header retrieval from different heights and data caching
+    expect(countDatas(manager)).toBe(minCount)
+    let h0 = await manager.findHeaderForHeightOrUndefined(0)
+    expect(h0?.hash).toBe('000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f')
+    expect(countDatas(manager)).toBe(minCount === 0 ? 1 : minCount)
+    const h101010 = await manager.findHeaderForHeightOrUndefined(101010)
+    expect(h101010?.hash).toBe('000000000001af33247fff33aae7c31baee4148d5a189e7353bf13bcee618202')
+    expect(countDatas(manager)).toBe(2)
+
+    h0 = await manager.findHeaderForHeightOrUndefined(0)
+    expect(h0?.hash).toBe('000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f')
+    expect(countDatas(manager)).toBe(2)
+
+    const h202020 = await manager.findHeaderForHeightOrUndefined(202020)
+    expect(h202020?.hash).toBe('00000000000003a40858736f3788edcbca3aa89ac5723a8c6b42f0227084f949')
+    expect(countDatas(manager)).toBe(2)
+    const h303030 = await manager.findHeaderForHeightOrUndefined(303030)
+    expect(h303030?.hash).toBe('00000000000000002f66589be500afbf212eabf7b10e12fe4639684df808c83b')
+    expect(countDatas(manager)).toBe(2)
+
+    h0 = await manager.findHeaderForHeightOrUndefined(0)
+    expect(h0?.hash).toBe('000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f')
+    expect(countDatas(manager)).toBe(2)
+
+    const h808080 = await manager.findHeaderForHeightOrUndefined(808080)
+    expect(h808080?.hash).toBe('00000000000000000a7287950fae52dac3098ba43011fd1e1315974a419b0110')
+    expect(countDatas(manager)).toBe(2)
+
+    // Verify a height request that is out of range returns undefined and does not affect cached data
+    const h909090 = await manager.findHeaderForHeightOrUndefined(909090)
+    expect(h909090?.hash).toBeUndefined()
+    expect(countDatas(manager)).toBe(2)
+
+    // Verify retrieval from cached data.
+    h0 = await manager.findHeaderForHeightOrUndefined(0)
+    expect(h0?.hash).toBe('000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f')
+  }
+
+  test('1 headers from heights maxRetained 2', async () => {
+    if (!runSlowTests) return
+    const options = BulkFileDataManager.createDefaultOptions(chain)
+    const manager = new BulkFileDataManager(options)
+
+    await test1Body(manager, 0)
+  })
+
+  test('1a headers from heights maxRetained 2', async () => {
+    if (!runSlowTests) return
+    const options = BulkFileDataManager.createDefaultOptions(chain)
+    const manager = new BulkFileDataManager(options)
+    const storage = await setupStorageKnex(manager, `BulkFileDataManager.test_1a`, true)
+
+    await test1Body(manager, 2)
+
+    await storage.destroy()
+  })
+
+  test('2 ReValidate', async () => {
+    if (!runSlowTests) return
+    const options = BulkFileDataManager.createDefaultOptions(chain)
+    const manager = new BulkFileDataManager(options)
+
+    // Verify full data re-validation
+    await manager.ReValidate()
+    expect(countDatas(manager)).toBe(2)
+  })
+
+  test('3 exportHeadersToFs', async () => {
+    if (!runSlowTests) return
+    const options = BulkFileDataManager.createDefaultOptions(chain)
+    const manager = new BulkFileDataManager(options)
+
+    for (const i of [349, 379, 399, 402, 499]) {
+      const folder = fs.pathJoin(rootFolder, `cdnTest${i}`)
+      await manager.exportHeadersToFs(ChaintracksFs, 100, folder, `http://localhost:8${i}/blockheaders`, i)
+    }
+  })
+
+  async function test4Body(manager: BulkFileDataManager, maxCount: number) {
+    expect(countDatas(manager)).toBe(2)
+    const range = await manager.getHeightRange()
+    expect(range.maxHeight).toBe(349)
+
+    await manager.mergeIncrementalBlockHeaders(headers300_399.slice(50))
+    await manager.ReValidate()
+    expect(countDatas(manager)).toBe(maxCount)
+    await manager.mergeIncrementalBlockHeaders(headers400_499)
+    await manager.ReValidate()
+    expect(countDatas(manager)).toBe(maxCount)
+
+    await updateFromLocalServer(manager, server379!)
+    await manager.ReValidate()
+    expect(countDatas(manager)).toBe(maxCount)
+    await updateFromLocalServer(manager, server399!)
+    await manager.ReValidate()
+    expect(countDatas(manager)).toBe(maxCount)
+    await updateFromLocalServer(manager, server402!)
+    await manager.ReValidate()
+    expect(countDatas(manager)).toBe(maxCount)
+    await updateFromLocalServer(manager, server499!)
+    await manager.ReValidate()
+    expect(countDatas(manager)).toBe(2)
+  }
+
+  test('4 add two incremental chunks overwrite by CDN', async () => {
+    const manager = await setupManagerOnLocalServer(server349!)
+
+    await test4Body(manager, 3)
+  })
+
+  test('4a add two incremental chunks overwrite by CDN', async () => {
+    const manager = await setupManagerOnLocalServer(server349!)
+    const storage = await setupStorageKnex(manager, `BulkFileDataManager.test_4a`, true)
+
+    await test4Body(manager, 2)
+
+    await storage.destroy()
+  })
+
+  async function test5Body(manager: BulkFileDataManager, maxCount: number) {
+    expect(countDatas(manager)).toBe(2)
+    await updateFromLocalServer(manager, server379!)
+    await manager.ReValidate()
+    expect(countDatas(manager)).toBe(2)
+
+    await manager.mergeIncrementalBlockHeaders(headers300_399.slice(50))
+    await manager.ReValidate()
+    expect(countDatas(manager)).toBe(maxCount)
+
+    await updateFromLocalServer(manager, server499!)
+    await manager.ReValidate()
+    expect(countDatas(manager)).toBe(2)
+
+    await manager.mergeIncrementalBlockHeaders(headers400_499)
+    await manager.ReValidate()
+    expect(countDatas(manager)).toBe(2)
+  }
+
+  test('5 add CDN incremental CDN incremental', async () => {
+    const manager = await setupManagerOnLocalServer(server349!)
+
+    await test5Body(manager, 3)
+  })
+
+  test('5a add CDN incremental CDN incremental', async () => {
+    const manager = await setupManagerOnLocalServer(server349!)
+    const storage = await setupStorageKnex(manager, `BulkFileDataManager.test_5a`, true)
+
+    await test5Body(manager, 2)
+
+    await storage.destroy()
+  })
+
+  async function setupStorageKnex(
+    manager: BulkFileDataManager,
+    filename: string,
+    deleteSqliteFile: boolean
+  ): Promise<ChaintracksStorageKnex> {
+    const path = fs.pathJoin(rootFolder, `${filename}.sqlite`)
+    const localSqlite: Knex.Config = {
+      client: 'sqlite3',
+      connection: { filename: path },
+      useNullAsDefault: true
+    }
+    if (deleteSqliteFile) {
+      const fs = ChaintracksFs
+      try {
+        await fs.delete(path)
+      } catch {}
+    }
+    const knexOptions = ChaintracksStorageKnex.createStorageKnexOptions(chain, makeKnex(localSqlite))
+    knexOptions.bulkFileDataManager = manager
+    const storage = new ChaintracksStorageKnex(knexOptions)
+    await storage.makeAvailable()
+
+    return storage
+  }
+
+  async function setupManagerOnLocalServer(server: LocalCdnServer) {
+    const options = BulkFileDataManager.createDefaultOptions(chain)
+    options.fromKnownSourceUrl = undefined
+    const manager = new BulkFileDataManager(options)
+    await updateFromLocalServer(manager, server)
+    return manager
+  }
+})
+
+async function updateFromLocalServer(manager: BulkFileDataManager, server: LocalCdnServer) {
+  await manager.updateFromUrl(`http://localhost:${server.port}/blockheaders`)
+}
+
+function countDatas(manager: BulkFileDataManager): number {
+  let count = 0
+  for (const file of manager['bfds'] as BulkHeaderFileInfo[]) {
+    if (file.data) count += 1
+  }
+  return count
+}