@bopen-io/wallet-toolbox 1.7.18

package/src/services/chaintracker/chaintracks/__tests/ChaintracksClientApi.test.ts

@@ -0,0 +1,192 @@
+import { Chain } from '../../../../sdk/types'
+import { BaseBlockHeader, BlockHeader } from '../../../../sdk/WalletServices.interfaces'
+import { asUint8Array } from '../../../../utility/utilityHelpers.noBuffer'
+import { ChaintracksClientApi } from '../Api/ChaintracksClientApi'
+import { ChaintracksStorageApi } from '../Api/ChaintracksStorageApi'
+import { Chaintracks } from '../Chaintracks'
+import { ChaintracksService } from '../ChaintracksService'
+import { ChaintracksServiceClient } from '../ChaintracksServiceClient'
+import {
+  blockHash,
+  deserializeBaseBlockHeaders,
+  genesisBuffer,
+  serializeBaseBlockHeader
+} from '../util/blockHeaderUtilities'
+
+type ClientClass = 'ChaintracksSingletonClient' | 'Chaintracks' | 'ChaintracksServiceClient' | undefined
+let clientClass: ClientClass = undefined
+
+clientClass = 'Chaintracks'
+//clientClass = "ChaintracksSingletonClient"
+//clientClass = "ChaintracksServiceClient"
+const includeLocalServiceClient = true
+const includeLocalServiceChaintracks = true
+const includeNpmRegistryClient = false
+const includeGcrTestClient = false
+describe(`ChaintracksClientApi tests`, () => {
+  jest.setTimeout(999999999)
+
+  const chain: Chain = 'main'
+
+  const clients: { client: ChaintracksClientApi; chain: Chain }[] = []
+
+  let localService: ChaintracksService
+  let localServiceStorage: ChaintracksStorageApi
+  let localServiceClient: ChaintracksClientApi
+  let firstTip: BlockHeader
+
+  beforeAll(async () => {
+    if (includeLocalServiceChaintracks || includeLocalServiceClient) {
+      localService = new ChaintracksService(ChaintracksService.createChaintracksServiceOptions(chain))
+      localServiceStorage = localService.chaintracks['storageEngine'] as ChaintracksStorageApi
+      await localService.startJsonRpcServer()
+
+      if (includeLocalServiceClient) {
+        localServiceClient = new ChaintracksServiceClient(chain, `http://localhost:${localService.port}`, {})
+        clients.push({ client: localServiceClient, chain })
+      }
+
+      if (includeLocalServiceChaintracks) {
+        clients.push({ client: localService.chaintracks, chain })
+      }
+    }
+
+    if (includeGcrTestClient) {
+      const gcr = new ChaintracksServiceClient('test', `https://testnet-chaintracks.babbage.systems`, {})
+      clients.push({ client: gcr, chain: 'test' })
+    }
+    if (includeGcrTestClient) {
+      const gcr = new ChaintracksServiceClient('main', `https://mainnet-chaintracks.babbage.systems`, {})
+      clients.push({ client: gcr, chain: 'main' })
+    }
+
+    if (includeNpmRegistryClient) {
+      clients.push({ client: makeNpmRegistryClient(chain), chain })
+    }
+
+    const ft = await clients[0].client.findChainTipHeader()
+    if (!ft) throw new Error('No chain tip found')
+    firstTip = ft
+  })
+
+  afterAll(async () => {
+    await localService?.stopJsonRpcServer()
+  })
+
+  test('0 getChain', async () => {
+    for (const { client, chain } of clients) {
+      const gotChain = await client.getChain()
+      expect(gotChain).toBe(chain)
+    }
+  })
+
+  test('1 getInfo', async () => {
+    for (const { client, chain } of clients) {
+      const gotInfo = await client.getInfo()
+      expect(gotInfo.chain).toBe(chain)
+      expect(gotInfo.heightBulk).toBeGreaterThan(700000)
+      expect(gotInfo.heightLive).toBeGreaterThanOrEqual(firstTip.height - 2)
+    }
+  })
+
+  test('2 getPresentHeight', async () => {
+    for (const { client, chain } of clients) {
+      const presentHeight = await client.getPresentHeight()
+      expect(presentHeight).toBeGreaterThanOrEqual(firstTip.height - 2)
+    }
+  })
+
+  test('3 getHeaders', async () => {
+    for (const { client, chain } of clients) {
+      const info = await client.getInfo()
+      const h0 = info.heightBulk + 1
+      const h1 = info.heightLive || 10
+      const bulkHeaders = await getHeaders(h0 - 2, 2)
+      expect(bulkHeaders.length).toBe(2)
+      expect(bulkHeaders[1].previousHash === blockHash(bulkHeaders[0])).toBe(true)
+      const bothHeaders = await getHeaders(h0 - 1, 2)
+      expect(bothHeaders.length).toBe(2)
+      expect(bothHeaders[1].previousHash === blockHash(bothHeaders[0])).toBe(true)
+      const liveHeaders = await getHeaders(h0 - 0, 2)
+      expect(liveHeaders.length).toBe(2)
+      expect(liveHeaders[1].previousHash === blockHash(liveHeaders[0])).toBe(true)
+      const partHeaders = await getHeaders(h1, 2)
+      expect(partHeaders.length).toBe(1)
+
+      async function getHeaders(h: number, c: number): Promise<BaseBlockHeader[]> {
+        const data = asUint8Array(await client.getHeaders(h, c))
+        const headers = deserializeBaseBlockHeaders(data)
+        return headers
+      }
+    }
+  })
+
+  test('4 findChainTipHeader', async () => {
+    for (const { client, chain } of clients) {
+      const tipHeader = await client.findChainTipHeader()
+      expect(tipHeader.height >= firstTip.height).toBe(true)
+    }
+  })
+
+  test('5 findChainTipHash', async () => {
+    for (const { client, chain } of clients) {
+      const hash = await client.findChainTipHash()
+      expect(hash.length === 64).toBe(true)
+    }
+  })
+
+  test('6 findHeaderForHeight', async () => {
+    for (const { client, chain } of clients) {
+      const header0 = await client.findHeaderForHeight(0)
+      expect(header0 !== undefined).toBe(true)
+      if (header0) {
+        expect(genesisBuffer(chain)).toEqual(serializeBaseBlockHeader(header0))
+      }
+
+      const header = await client.findHeaderForHeight(firstTip.height)
+      expect(header && header.height === firstTip.height).toBe(true)
+
+      const missing = await client.findHeaderForHeight(99999999)
+      expect(missing === undefined).toBe(true)
+    }
+  })
+
+  test('7 addHeader', async () => {
+    for (const { client, chain } of clients) {
+      const t = await client.findChainTipHeader()
+      const h: BaseBlockHeader = {
+        version: t.version,
+        previousHash: t.previousHash,
+        merkleRoot: t.merkleRoot,
+        time: t.time,
+        bits: t.bits,
+        nonce: t.nonce
+      }
+      await client.addHeader(h)
+    }
+  })
+
+  /*
+  const headers: BlockHeader[] = []
+  const headerListener: HeaderListener = (header) => { headers.push(header) }
+
+  test("subscribeHeaders", async () => {
+      const id = await client.subscribeHeaders(headerListener)
+      expect(typeof id === 'string').toBe(true)
+      expect(await client.unsubscribe(id)).toBe(true)
+  })
+
+  const reorgs: ({ depth: number, oldTip: BlockHeader, newTip: BlockHeader })[] = []
+  const reorgListener: ReorgListener = (depth, oldTip, newTip) => { reorgs.push({ depth, oldTip, newTip }) }
+
+  test("subscribeReorgs", async () => {
+      const id = await client.subscribeReorgs(reorgListener)
+      expect(typeof id === 'string').toBe(true)
+      expect(await client.unsubscribe(id)).toBe(true)
+  })
+  */
+})
+
+function makeNpmRegistryClient(chain: Chain) {
+  return new ChaintracksServiceClient(chain, `https://npm-registry.babbage.systems:${chain === 'main' ? 8084 : 8083}`)
+}

package/src/services/chaintracker/chaintracks/__tests/LocalCdnServer.ts

@@ -0,0 +1,75 @@
+import express from 'express'
+import path from 'path'
+import fs from 'fs'
+import cors from 'cors'
+import http from 'http'
+
+export class LocalCdnServer {
+  app: express.Application
+  server?: http.Server
+
+  constructor(
+    public port: number,
+    public folder: string
+  ) {
+    // Ensure the files directory exists
+    if (!fs.existsSync(this.folder)) {
+      fs.mkdirSync(this.folder)
+    }
+  }
+
+  async start() {
+    this.app = express()
+    this.server = http.createServer(this.app)
+
+    // Enable CORS for all routes
+    this.app.use(cors())
+
+    // Serve static files from the files directory
+    this.app.use('/blockheaders', express.static(this.folder))
+
+    // List all files in the directory
+    this.app.get('/files', (req, res) => {
+      fs.readdir(this.folder, (err, files) => {
+        if (err) {
+          return res.status(500).json({ error: 'Unable to read directory' })
+        }
+        res.json({ files })
+      })
+    })
+
+    // Download specific file
+    this.app.get('/download/:filename', (req, res) => {
+      const filePath = path.join(this.folder, req.params.filename)
+
+      // Ensure filePath stays within the allowed folder (prevent directory traversal)
+      const resolvedFolder = path.resolve(this.folder)
+      const resolvedFilePath = path.resolve(filePath)
+      if (!resolvedFilePath.startsWith(resolvedFolder + path.sep)) {
+        // Prevent access to files outside the allowed directory
+        return res.status(403).json({ error: 'Access denied' })
+      }
+
+      if (!fs.existsSync(resolvedFilePath)) {
+        return res.status(404).json({ error: 'File not found' })
+      }
+
+      res.download(resolvedFilePath, err => {
+        if (err) {
+          res.status(500).json({ error: 'Error downloading file' })
+        }
+      })
+    })
+
+    this.server.listen(this.port, () => {
+      //      console.log(`Server running at http://localhost:${this.port}`)
+      //      console.log(`Serving files from: ${this.folder}`)
+      //      console.log(`Access files at: http://localhost:${this.port}/blockheaders`)
+      //      console.log(`List files at: http://localhost:${this.port}/files`)
+    })
+  }
+
+  async stop() {
+    await this.server?.close()
+  }
+}

package/src/services/chaintracker/chaintracks/__tests/createIdbChaintracks.test.ts

@@ -0,0 +1,62 @@
+import { _tu } from '../../../../../test/utils/TestUtilsWalletStorage'
+import { wait } from '../../../../utility/utilityHelpers'
+import { createIdbChaintracks } from '../createIdbChaintracks'
+import { BulkFileDataManager } from '../util/BulkFileDataManager'
+import { BulkHeaderFileInfo } from '../util/BulkHeaderFile'
+import { HeaderListener } from '../Api/ChaintracksClientApi'
+import { Chain } from '../../../../sdk/types'
+import { BlockHeader } from '../Api/BlockHeaderApi'
+
+import 'fake-indexeddb/auto'
+
+describe('createIdbChaintracks tests', () => {
+  jest.setTimeout(99999999)
+
+  test('0', async () => {
+    const target: Chain = 'main'
+    if (_tu.noEnv(target)) return
+    const env = _tu.getEnv(target)
+    const { chain, chaintracks, storage, available } = await createIdbChaintracks(env.chain, env.whatsonchainApiKey)
+    await available
+    const headerListener: HeaderListener = (header: BlockHeader) => {
+      console.log(`headerListener: height: ${header.height} hash: ${header.hash} ${new Date().toISOString()}`)
+    }
+    chaintracks.subscribeHeaders(headerListener)
+    expect(countDatas(storage.bulkManager)).toBeLessThanOrEqual(3)
+
+    const tipHash = await chaintracks.findChainTipHash()
+    const tip = await chaintracks.findChainTipHeader()
+    chaintracks.log(`tip: height: ${tip.height} hash: ${tip.hash}`)
+    expect(tip.hash).toBe(tipHash)
+
+    const forHash = await chaintracks.findHeaderForBlockHash(tip.hash)
+    expect(forHash?.hash).toBe(tip.hash)
+
+    const forHashLive = await chaintracks.findLiveHeaderForBlockHash(tip.hash)
+    expect(forHashLive?.hash).toBe(tip.hash)
+
+    const chainwork = await chaintracks.findChainWorkForBlockHash(tip.hash)
+    expect(forHashLive?.chainWork).toBe(chainwork)
+
+    const forHeight = await chaintracks.findHeaderForHeight(tip.height)
+    expect(forHeight?.hash).toBe(tip.hash)
+
+    const listening = await chaintracks.isListening()
+    expect(listening).toBe(true)
+
+    //    console.log('validating...')
+    //    const validated = await chaintracks.validate()
+    //    expect(validated).toBe(true)
+    //    console.log('validated')
+
+    for (;;) await wait(120000)
+  })
+})
+
+function countDatas(manager: BulkFileDataManager): number {
+  let count = 0
+  for (const file of manager['bfds'] as BulkHeaderFileInfo[]) {
+    if (file.data) count += 1
+  }
+  return count
+}

package/src/services/chaintracker/chaintracks/__tests/data/cdnTest349/mainNetBlockHeaders.json

@@ -0,0 +1 @@
+{"rootFolder":"http://localhost:8349/blockheaders","jsonFilename":"mainNetBlockHeaders.json","headersPerFile":100,"files":[{"chain":"main","count":100,"fileHash":"wtQ5sUgMxVOzg8+30WLkOgyW7mbf6vC1qfincLMycpg=","fileName":"mainNet_0.headers","firstHeight":0,"lastChainWork":"0000000000000000000000000000000000000000000000000000006400640064","lastHash":"00000000cd9b12643e6854cb25939b39cd7a1ad0af31a9bd8b2efe67854b1995","prevChainWork":"0000000000000000000000000000000000000000000000000000000000000000","prevHash":"0000000000000000000000000000000000000000000000000000000000000000","sourceUrl":"http://localhost:8349/blockheaders"},{"chain":"main","count":100,"fileHash":"xpWhru0ArBs2k//SG7BY1nq5Jifo8VJF5l51uoaOslI=","fileName":"mainNet_1.headers","firstHeight":100,"lastChainWork":"000000000000000000000000000000000000000000000000000000c800c800c8","lastHash":"00000000b7691ccc084542565697eca256e56bb7f67e560b48789db27f0468eb","prevChainWork":"0000000000000000000000000000000000000000000000000000006400640064","prevHash":"00000000cd9b12643e6854cb25939b39cd7a1ad0af31a9bd8b2efe67854b1995","sourceUrl":"http://localhost:8349/blockheaders"},{"chain":"main","count":100,"fileHash":"wtshdbmZyP3aJMKClO2b7Byg8n/MXZwehlXUWFsqRoA=","fileName":"mainNet_2.headers","firstHeight":200,"lastChainWork":"0000000000000000000000000000000000000000000000000000012c012c012c","lastHash":"00000000b9b003c04b2fe21b4f7a2fa809eb643852573c3a1748653e2019f01c","prevChainWork":"000000000000000000000000000000000000000000000000000000c800c800c8","prevHash":"00000000b7691ccc084542565697eca256e56bb7f67e560b48789db27f0468eb","sourceUrl":"http://localhost:8349/blockheaders"},{"chain":"main","count":50,"fileHash":"FbE8Z/I08DD6/HZvJlqlZme3f8PDFB6vDH5i3buzAYY=","fileName":"mainNet_3.headers","firstHeight":300,"lastChainWork":"0000000000000000000000000000000000000000000000000000015e015e015e","lastHash":"00000000c4f49e3bffbd95679dc5684e388bf65b0c5160a64b34ab9a22ebc6d3","prevChainWork":"0000000000000000000000000000000000000000000000000000012c012c012c","prevHash":"00000000b9b003c04b2fe21b4f7a2fa809eb643852573c3a1748653e2019f01c","sourceUrl":"http://localhost:8349/blockheaders"}]}

package/src/services/chaintracker/chaintracks/__tests/data/cdnTest379/mainNetBlockHeaders.json

@@ -0,0 +1 @@
+{"rootFolder":"http://localhost:8379/blockheaders","jsonFilename":"mainNetBlockHeaders.json","headersPerFile":100,"files":[{"chain":"main","count":100,"fileHash":"wtQ5sUgMxVOzg8+30WLkOgyW7mbf6vC1qfincLMycpg=","fileName":"mainNet_0.headers","firstHeight":0,"lastChainWork":"0000000000000000000000000000000000000000000000000000006400640064","lastHash":"00000000cd9b12643e6854cb25939b39cd7a1ad0af31a9bd8b2efe67854b1995","prevChainWork":"0000000000000000000000000000000000000000000000000000000000000000","prevHash":"0000000000000000000000000000000000000000000000000000000000000000","sourceUrl":"http://localhost:8379/blockheaders"},{"chain":"main","count":100,"fileHash":"xpWhru0ArBs2k//SG7BY1nq5Jifo8VJF5l51uoaOslI=","fileName":"mainNet_1.headers","firstHeight":100,"lastChainWork":"000000000000000000000000000000000000000000000000000000c800c800c8","lastHash":"00000000b7691ccc084542565697eca256e56bb7f67e560b48789db27f0468eb","prevChainWork":"0000000000000000000000000000000000000000000000000000006400640064","prevHash":"00000000cd9b12643e6854cb25939b39cd7a1ad0af31a9bd8b2efe67854b1995","sourceUrl":"http://localhost:8379/blockheaders"},{"chain":"main","count":100,"fileHash":"wtshdbmZyP3aJMKClO2b7Byg8n/MXZwehlXUWFsqRoA=","fileName":"mainNet_2.headers","firstHeight":200,"lastChainWork":"0000000000000000000000000000000000000000000000000000012c012c012c","lastHash":"00000000b9b003c04b2fe21b4f7a2fa809eb643852573c3a1748653e2019f01c","prevChainWork":"000000000000000000000000000000000000000000000000000000c800c800c8","prevHash":"00000000b7691ccc084542565697eca256e56bb7f67e560b48789db27f0468eb","sourceUrl":"http://localhost:8379/blockheaders"},{"chain":"main","count":80,"fileHash":"wmUh++Q9rJXr3FJA1c8FSZv5QryD+3e2gT2kxbDmZn8=","fileName":"mainNet_3.headers","firstHeight":300,"lastChainWork":"0000000000000000000000000000000000000000000000000000017c017c017c","lastHash":"00000000ca826763c172f28976c008fcf73f4f2cbac96335d49918162cf3b018","prevChainWork":"0000000000000000000000000000000000000000000000000000012c012c012c","prevHash":"00000000b9b003c04b2fe21b4f7a2fa809eb643852573c3a1748653e2019f01c","sourceUrl":"http://localhost:8379/blockheaders"}]}

package/src/services/chaintracker/chaintracks/__tests/data/cdnTest399/mainNetBlockHeaders.json

@@ -0,0 +1 @@
+{"rootFolder":"http://localhost:8399/blockheaders","jsonFilename":"mainNetBlockHeaders.json","headersPerFile":100,"files":[{"chain":"main","count":100,"fileHash":"wtQ5sUgMxVOzg8+30WLkOgyW7mbf6vC1qfincLMycpg=","fileName":"mainNet_0.headers","firstHeight":0,"lastChainWork":"0000000000000000000000000000000000000000000000000000006400640064","lastHash":"00000000cd9b12643e6854cb25939b39cd7a1ad0af31a9bd8b2efe67854b1995","prevChainWork":"0000000000000000000000000000000000000000000000000000000000000000","prevHash":"0000000000000000000000000000000000000000000000000000000000000000","sourceUrl":"http://localhost:8399/blockheaders"},{"chain":"main","count":100,"fileHash":"xpWhru0ArBs2k//SG7BY1nq5Jifo8VJF5l51uoaOslI=","fileName":"mainNet_1.headers","firstHeight":100,"lastChainWork":"000000000000000000000000000000000000000000000000000000c800c800c8","lastHash":"00000000b7691ccc084542565697eca256e56bb7f67e560b48789db27f0468eb","prevChainWork":"0000000000000000000000000000000000000000000000000000006400640064","prevHash":"00000000cd9b12643e6854cb25939b39cd7a1ad0af31a9bd8b2efe67854b1995","sourceUrl":"http://localhost:8399/blockheaders"},{"chain":"main","count":100,"fileHash":"wtshdbmZyP3aJMKClO2b7Byg8n/MXZwehlXUWFsqRoA=","fileName":"mainNet_2.headers","firstHeight":200,"lastChainWork":"0000000000000000000000000000000000000000000000000000012c012c012c","lastHash":"00000000b9b003c04b2fe21b4f7a2fa809eb643852573c3a1748653e2019f01c","prevChainWork":"000000000000000000000000000000000000000000000000000000c800c800c8","prevHash":"00000000b7691ccc084542565697eca256e56bb7f67e560b48789db27f0468eb","sourceUrl":"http://localhost:8399/blockheaders"},{"chain":"main","count":100,"fileHash":"e9psHUbYviyz/5i45ZN1wrGVkbMr37x3UOp6+nA5g1w=","fileName":"mainNet_3.headers","firstHeight":300,"lastChainWork":"0000000000000000000000000000000000000000000000000000019001900190","lastHash":"0000000006a774e00b730eeba018fbca6673c32753fce367a316f5c9be4332bd","prevChainWork":"0000000000000000000000000000000000000000000000000000012c012c012c","prevHash":"00000000b9b003c04b2fe21b4f7a2fa809eb643852573c3a1748653e2019f01c","sourceUrl":"http://localhost:8399/blockheaders"}]}

package/src/services/chaintracker/chaintracks/__tests/data/cdnTest402/mainNetBlockHeaders.json

@@ -0,0 +1 @@
+{"rootFolder":"http://localhost:8402/blockheaders","jsonFilename":"mainNetBlockHeaders.json","headersPerFile":100,"files":[{"chain":"main","count":100,"fileHash":"wtQ5sUgMxVOzg8+30WLkOgyW7mbf6vC1qfincLMycpg=","fileName":"mainNet_0.headers","firstHeight":0,"lastChainWork":"0000000000000000000000000000000000000000000000000000006400640064","lastHash":"00000000cd9b12643e6854cb25939b39cd7a1ad0af31a9bd8b2efe67854b1995","prevChainWork":"0000000000000000000000000000000000000000000000000000000000000000","prevHash":"0000000000000000000000000000000000000000000000000000000000000000","sourceUrl":"http://localhost:8402/blockheaders"},{"chain":"main","count":100,"fileHash":"xpWhru0ArBs2k//SG7BY1nq5Jifo8VJF5l51uoaOslI=","fileName":"mainNet_1.headers","firstHeight":100,"lastChainWork":"000000000000000000000000000000000000000000000000000000c800c800c8","lastHash":"00000000b7691ccc084542565697eca256e56bb7f67e560b48789db27f0468eb","prevChainWork":"0000000000000000000000000000000000000000000000000000006400640064","prevHash":"00000000cd9b12643e6854cb25939b39cd7a1ad0af31a9bd8b2efe67854b1995","sourceUrl":"http://localhost:8402/blockheaders"},{"chain":"main","count":100,"fileHash":"wtshdbmZyP3aJMKClO2b7Byg8n/MXZwehlXUWFsqRoA=","fileName":"mainNet_2.headers","firstHeight":200,"lastChainWork":"0000000000000000000000000000000000000000000000000000012c012c012c","lastHash":"00000000b9b003c04b2fe21b4f7a2fa809eb643852573c3a1748653e2019f01c","prevChainWork":"000000000000000000000000000000000000000000000000000000c800c800c8","prevHash":"00000000b7691ccc084542565697eca256e56bb7f67e560b48789db27f0468eb","sourceUrl":"http://localhost:8402/blockheaders"},{"chain":"main","count":100,"fileHash":"e9psHUbYviyz/5i45ZN1wrGVkbMr37x3UOp6+nA5g1w=","fileName":"mainNet_3.headers","firstHeight":300,"lastChainWork":"0000000000000000000000000000000000000000000000000000019001900190","lastHash":"0000000006a774e00b730eeba018fbca6673c32753fce367a316f5c9be4332bd","prevChainWork":"0000000000000000000000000000000000000000000000000000012c012c012c","prevHash":"00000000b9b003c04b2fe21b4f7a2fa809eb643852573c3a1748653e2019f01c","sourceUrl":"http://localhost:8402/blockheaders"},{"chain":"main","count":3,"fileHash":"HC0yHVInsxaJqmS2ds53D/Yw4MinlIUhceBOI+qX3cc=","fileName":"mainNet_4.headers","firstHeight":400,"lastChainWork":"0000000000000000000000000000000000000000000000000000019301930193","lastHash":"000000005265c5afb97e91a7961ea9861612d965e7714df0c9d38da02b59d8f8","prevChainWork":"0000000000000000000000000000000000000000000000000000019001900190","prevHash":"0000000006a774e00b730eeba018fbca6673c32753fce367a316f5c9be4332bd","sourceUrl":"http://localhost:8402/blockheaders"}]}

package/src/services/chaintracker/chaintracks/__tests/data/cdnTest499/mainNetBlockHeaders.json

@@ -0,0 +1 @@
+{"rootFolder":"http://localhost:8499/blockheaders","jsonFilename":"mainNetBlockHeaders.json","headersPerFile":100,"files":[{"chain":"main","count":100,"fileHash":"wtQ5sUgMxVOzg8+30WLkOgyW7mbf6vC1qfincLMycpg=","fileName":"mainNet_0.headers","firstHeight":0,"lastChainWork":"0000000000000000000000000000000000000000000000000000006400640064","lastHash":"00000000cd9b12643e6854cb25939b39cd7a1ad0af31a9bd8b2efe67854b1995","prevChainWork":"0000000000000000000000000000000000000000000000000000000000000000","prevHash":"0000000000000000000000000000000000000000000000000000000000000000","sourceUrl":"http://localhost:8499/blockheaders"},{"chain":"main","count":100,"fileHash":"xpWhru0ArBs2k//SG7BY1nq5Jifo8VJF5l51uoaOslI=","fileName":"mainNet_1.headers","firstHeight":100,"lastChainWork":"000000000000000000000000000000000000000000000000000000c800c800c8","lastHash":"00000000b7691ccc084542565697eca256e56bb7f67e560b48789db27f0468eb","prevChainWork":"0000000000000000000000000000000000000000000000000000006400640064","prevHash":"00000000cd9b12643e6854cb25939b39cd7a1ad0af31a9bd8b2efe67854b1995","sourceUrl":"http://localhost:8499/blockheaders"},{"chain":"main","count":100,"fileHash":"wtshdbmZyP3aJMKClO2b7Byg8n/MXZwehlXUWFsqRoA=","fileName":"mainNet_2.headers","firstHeight":200,"lastChainWork":"0000000000000000000000000000000000000000000000000000012c012c012c","lastHash":"00000000b9b003c04b2fe21b4f7a2fa809eb643852573c3a1748653e2019f01c","prevChainWork":"000000000000000000000000000000000000000000000000000000c800c800c8","prevHash":"00000000b7691ccc084542565697eca256e56bb7f67e560b48789db27f0468eb","sourceUrl":"http://localhost:8499/blockheaders"},{"chain":"main","count":100,"fileHash":"e9psHUbYviyz/5i45ZN1wrGVkbMr37x3UOp6+nA5g1w=","fileName":"mainNet_3.headers","firstHeight":300,"lastChainWork":"0000000000000000000000000000000000000000000000000000019001900190","lastHash":"0000000006a774e00b730eeba018fbca6673c32753fce367a316f5c9be4332bd","prevChainWork":"0000000000000000000000000000000000000000000000000000012c012c012c","prevHash":"00000000b9b003c04b2fe21b4f7a2fa809eb643852573c3a1748653e2019f01c","sourceUrl":"http://localhost:8499/blockheaders"},{"chain":"main","count":100,"fileHash":"VW0lh8bl2Wtvn5Uqzh7rt1ty/nBn9XJa0AASpRnV4rQ=","fileName":"mainNet_4.headers","firstHeight":400,"lastChainWork":"000000000000000000000000000000000000000000000000000001f401f401f4","lastHash":"00000000806df68baab17e49e567d4211177fef4849ffd8242d095c6a1169f45","prevChainWork":"0000000000000000000000000000000000000000000000000000019001900190","prevHash":"0000000006a774e00b730eeba018fbca6673c32753fce367a316f5c9be4332bd","sourceUrl":"http://localhost:8499/blockheaders"}]}

package/src/services/chaintracker/chaintracks/createDefaultIdbChaintracksOptions.ts

@@ -0,0 +1,92 @@
+import { Chain } from '../../../sdk'
+import { ChaintracksOptions } from './Api/ChaintracksApi'
+import { Chaintracks } from './Chaintracks'
+import { BulkIngestorCDNBabbage } from './Ingest/BulkIngestorCDNBabbage'
+import { ChaintracksFetch } from './util/ChaintracksFetch'
+import { LiveIngestorWhatsOnChainOptions, LiveIngestorWhatsOnChainPoll } from './Ingest/LiveIngestorWhatsOnChainPoll'
+import { BulkIngestorWhatsOnChainCdn, BulkIngestorWhatsOnChainOptions } from './Ingest/BulkIngestorWhatsOnChainCdn'
+import { ChaintracksFetchApi } from './Api/ChaintracksFetchApi'
+import { ChaintracksStorageIdb, ChaintracksStorageIdbOptions } from './Storage/ChaintracksStorageIdb'
+import { BulkFileDataManager, BulkFileDataManagerOptions } from './util/BulkFileDataManager'
+import { BulkIngestorCDNOptions } from './Ingest/BulkIngestorCDN'
+import { WhatsOnChainServicesOptions } from './Ingest/WhatsOnChainServices'
+
+export function createDefaultIdbChaintracksOptions(
+  chain: Chain,
+  whatsonchainApiKey: string = '',
+  maxPerFile: number = 100000,
+  maxRetained: number = 2,
+  fetch?: ChaintracksFetchApi,
+  cdnUrl: string = 'https://cdn.projectbabbage.com/blockheaders/',
+  liveHeightThreshold: number = 2000,
+  reorgHeightThreshold: number = 400,
+  bulkMigrationChunkSize: number = 500,
+  batchInsertLimit: number = 400,
+  addLiveRecursionLimit: number = 36
+): ChaintracksOptions {
+  fetch ||= new ChaintracksFetch()
+
+  const bfo: BulkFileDataManagerOptions = {
+    chain,
+    fetch,
+    maxPerFile,
+    maxRetained,
+    fromKnownSourceUrl: cdnUrl
+  }
+  const bulkFileDataManager = new BulkFileDataManager(bfo)
+
+  const so: ChaintracksStorageIdbOptions = {
+    chain,
+    bulkFileDataManager,
+    liveHeightThreshold,
+    reorgHeightThreshold,
+    bulkMigrationChunkSize,
+    batchInsertLimit
+  }
+  const storage = new ChaintracksStorageIdb(so)
+
+  const co: ChaintracksOptions = {
+    chain,
+    storage,
+    bulkIngestors: [],
+    liveIngestors: [],
+    addLiveRecursionLimit,
+    logging: (...args) => console.log(new Date().toISOString(), ...args),
+    readonly: false
+  }
+
+  const jsonResource = `${chain}NetBlockHeaders.json`
+
+  const bulkCdnOptions: BulkIngestorCDNOptions = {
+    chain,
+    jsonResource,
+    fetch,
+    cdnUrl,
+    maxPerFile
+  }
+  co.bulkIngestors.push(new BulkIngestorCDNBabbage(bulkCdnOptions))
+
+  const wocOptions: WhatsOnChainServicesOptions = {
+    chain,
+    apiKey: whatsonchainApiKey,
+    timeout: 30000,
+    userAgent: 'BabbageWhatsOnChainServices',
+    enableCache: true,
+    chainInfoMsecs: 5000
+  }
+
+  const bulkOptions: BulkIngestorWhatsOnChainOptions = {
+    ...wocOptions,
+    jsonResource,
+    idleWait: 5000
+  }
+  co.bulkIngestors.push(new BulkIngestorWhatsOnChainCdn(bulkOptions))
+
+  const liveOptions: LiveIngestorWhatsOnChainOptions = {
+    ...wocOptions,
+    idleWait: 100000
+  }
+  co.liveIngestors.push(new LiveIngestorWhatsOnChainPoll(liveOptions))
+
+  return co
+}

package/src/services/chaintracker/chaintracks/createDefaultKnexChaintracksOptions.ts

@@ -0,0 +1,111 @@
+import { Knex, knex as makeKnex } from 'knex'
+import { Chain } from '../../../sdk'
+import { ChaintracksOptions } from './Api/ChaintracksApi'
+import { ChaintracksFs } from './util/ChaintracksFs'
+import { ChaintracksStorageKnex, ChaintracksStorageKnexOptions } from './Storage/ChaintracksStorageKnex'
+import { BulkIngestorCDNBabbage } from './Ingest/BulkIngestorCDNBabbage'
+import { ChaintracksFetch } from './util/ChaintracksFetch'
+import { LiveIngestorWhatsOnChainOptions, LiveIngestorWhatsOnChainPoll } from './Ingest/LiveIngestorWhatsOnChainPoll'
+import { BulkIngestorWhatsOnChainCdn, BulkIngestorWhatsOnChainOptions } from './Ingest/BulkIngestorWhatsOnChainCdn'
+import { ChaintracksFetchApi } from './Api/ChaintracksFetchApi'
+import { BulkFileDataManager, BulkFileDataManagerOptions } from './util/BulkFileDataManager'
+import { BulkIngestorCDNOptions } from './Ingest/BulkIngestorCDN'
+import { WhatsOnChainServicesOptions } from './Ingest/WhatsOnChainServices'
+
+/**
+ *
+ * @param chain
+ * @param rootFolder defaults to "./data/"
+ * @returns
+ */
+export function createDefaultKnexChaintracksOptions(
+  chain: Chain,
+  rootFolder: string = './data/',
+  knexConfig?: Knex.Config,
+  whatsonchainApiKey: string = '',
+  maxPerFile: number = 100000,
+  maxRetained: number = 2,
+  fetch?: ChaintracksFetchApi,
+  cdnUrl: string = 'https://cdn.projectbabbage.com/blockheaders/',
+  liveHeightThreshold: number = 2000,
+  reorgHeightThreshold: number = 400,
+  bulkMigrationChunkSize: number = 500,
+  batchInsertLimit: number = 400,
+  addLiveRecursionLimit: number = 36
+): ChaintracksOptions {
+  fetch ||= new ChaintracksFetch()
+
+  const bfo: BulkFileDataManagerOptions = {
+    chain,
+    fetch,
+    maxPerFile,
+    maxRetained,
+    fromKnownSourceUrl: cdnUrl
+  }
+  const bulkFileDataManager = new BulkFileDataManager(bfo)
+
+  if (!knexConfig) {
+    knexConfig = {
+      client: 'sqlite3',
+      connection: { filename: ChaintracksFs.pathJoin(rootFolder, `${chain}Net_chaintracks.sqlite`) },
+      useNullAsDefault: true
+    }
+  }
+  const knexInstance = makeKnex(knexConfig)
+
+  const so: ChaintracksStorageKnexOptions = {
+    chain,
+    knex: knexInstance,
+    bulkFileDataManager,
+    liveHeightThreshold,
+    reorgHeightThreshold,
+    bulkMigrationChunkSize,
+    batchInsertLimit
+  }
+  const storage = new ChaintracksStorageKnex(so)
+
+  const co: ChaintracksOptions = {
+    chain,
+    storage,
+    bulkIngestors: [],
+    liveIngestors: [],
+    addLiveRecursionLimit,
+    logging: (...args) => console.log(new Date().toISOString(), ...args),
+    readonly: false
+  }
+
+  const jsonResource = `${chain}NetBlockHeaders.json`
+
+  const bulkCdnOptions: BulkIngestorCDNOptions = {
+    chain,
+    jsonResource,
+    fetch,
+    cdnUrl,
+    maxPerFile
+  }
+  co.bulkIngestors.push(new BulkIngestorCDNBabbage(bulkCdnOptions))
+
+  const wocOptions: WhatsOnChainServicesOptions = {
+    chain,
+    apiKey: whatsonchainApiKey,
+    timeout: 30000,
+    userAgent: 'BabbageWhatsOnChainServices',
+    enableCache: true,
+    chainInfoMsecs: 5000
+  }
+
+  const bulkOptions: BulkIngestorWhatsOnChainOptions = {
+    ...wocOptions,
+    jsonResource,
+    idleWait: 5000
+  }
+  co.bulkIngestors.push(new BulkIngestorWhatsOnChainCdn(bulkOptions))
+
+  const liveOptions: LiveIngestorWhatsOnChainOptions = {
+    ...wocOptions,
+    idleWait: 100000
+  }
+  co.liveIngestors.push(new LiveIngestorWhatsOnChainPoll(liveOptions))
+
+  return co
+}