@exodus/bitcoin-api 1.0.0-alpha.0

package/src/insight-api-client/util.js

@@ -0,0 +1,110 @@
+import assert from 'minimalistic-assert'
+import { groupBy, sortBy } from 'lodash'
+
+// return first to last (ALSO FILTERS)
+// we need to do this because Insight often returns the blocktime as time and many
+// txs may fall inside the same block, but if one is an input to other, it should preceed
+/*
+  A -> B (tx A is an input to tx B)
+  B -> C
+  A -> C
+
+  start:
+
+  C B A  (same time, ordered)... look at C, check if C's inputs are ahead, look for one furthest, then set + 1
+
+  B A C  check B, A is in input
+
+  A B C  check A, no inputs
+*/
+export function orderTxs(txs: Array) {
+  if (txs.length === 1) return [...txs]
+
+  // filter duplicates
+  const txMap = {}
+  txs.forEach((tx) => {
+    txMap[tx.txid] = tx
+  })
+
+  // group by time
+  const txTimesMap = groupBy(Object.values(txMap), (tx) => tx.time || '')
+
+  // sort unique time keys
+  const txTimes = Object.keys(txTimesMap)
+  txTimes.sort((a, b) => ~~a - ~~b)
+
+  let rettxs = []
+
+  txTimes.forEach((txTime) => {
+    let txsAtTime = txTimesMap[txTime] // array of txs
+    // should always be an array, but just incase
+    if (!Array.isArray(txsAtTime)) return
+    if (txsAtTime.length === 1) return rettxs.push(txsAtTime[0])
+
+    // existing order { txid: orderNum }
+    let txOrderCol = []
+    for (let i = 0; i < txsAtTime.length; ++i) {
+      txOrderCol.push({ txid: txsAtTime[i].txid, order: i })
+    }
+
+    // determine new orders, checks each vin and looks for max order,
+    // want current tx greater than any vin
+    const txsAtTimeClone = txsAtTime.slice(0)
+    while (txsAtTime.length > 0) {
+      let tx = txsAtTime.shift()
+      let maxOrder = 0
+      tx.vin.forEach((vin) => {
+        let txOrderMatch = txOrderCol.find((opair) => opair.txid === vin.txid)
+        if (txOrderMatch && txOrderMatch.order > maxOrder) maxOrder = txOrderMatch.order
+      })
+      txOrderCol.find((opair) => opair.txid === tx.txid).order = maxOrder + 1
+    }
+
+    let baseTime = ~~txTime
+    const newWorldOrder = sortBy(txOrderCol, ['order'])
+    newWorldOrder.forEach((opair, i) => {
+      let txid = opair.txid
+
+      let tx = txsAtTimeClone.find((tx) => tx.txid === txid)
+      assert(tx, 'InsightAPIClient.orderTxs() tx undefined.')
+
+      // we must do this to preserve order outside of this algorithm, difference is negligible
+      tx.time = txTime ? baseTime + i : baseTime
+      rettxs.push(tx)
+    })
+  })
+
+  return rettxs
+}
+
+/**
+ * It concerts an api URL to a WS service URL.
+ *
+ * https://somebtc.a.exodus.io/insight/ => https://somebtc.a.exodus.io
+ *
+ * @param {string} apiUrl the original apiUrl
+ * @returns {string} a WS url without the paths.
+ */
+export function toWSUrl(apiUrl) {
+  if (!apiUrl) {
+    return apiUrl
+  }
+  try {
+    // Note, we are not using URL because URL is different between mobile, desktop and BE
+    // Using hydra's networking modules is an overkill for this function.
+    const firstSplit = apiUrl.split('://')
+    if (firstSplit.length > 1) {
+      return `${firstSplit[0]}://${firstSplit[1].split('/')[0]}`
+    } else {
+      return apiUrl
+    }
+  } catch (e) {
+    return apiUrl
+  }
+}
+
+export function normalizeInsightConfig(config) {
+  const apiUrl = config?.insightServers?.[0]
+  const wsUrl = config?.insightServersWS?.[0] || toWSUrl(apiUrl)
+  return { apiUrl, wsUrl }
+}

package/src/insight-api-client/ws.js

@@ -0,0 +1,54 @@
+import { EventEmitter } from 'events'
+import io from 'socket.io-client'
+
+export default class InsightWSClient extends EventEmitter {
+  constructor(url, assetName) {
+    super()
+    this.url = url
+    this.assetName = assetName
+    this.io = null
+  }
+
+  connect(addresses, opts) {
+    const options = Object.assign(
+      { transports: ['websocket'], reconnectionDelayMax: 30000, reconnectionDelay: 10000 },
+      opts
+    )
+
+    this.io = io(this.url, options)
+
+    this.io.on('connect', () => {
+      this.emit('connect')
+
+      addresses.forEach((address) => {
+        this.io.emit('subscribe', address)
+      })
+
+      if (['bitcoin', 'bitcoinregtest', 'bitcointestnet'].includes(this.assetName)) {
+        this.io.emit('subscribe', 'inv_blocks')
+      }
+    })
+
+    addresses.forEach((address) => {
+      this.io.on(address, (data) => {
+        this.emit('message', { address, data })
+      })
+    })
+
+    this.io.on('block', (data) => {
+      this.emit('block', data)
+    })
+
+    this.io.on('disconnect', () => {
+      this.emit('disconnect')
+    })
+
+    this.io.on('reconnect', () => {
+      this.emit('reconnect')
+    })
+  }
+
+  close() {
+    if (this.io) this.io.disconnect()
+  }
+}

package/src/key-identifier.js

@@ -0,0 +1,19 @@
+import { buildBip32Path } from '@exodus/keychain'
+
+export default ({ asset }) => ({ purpose, accountIndex, chainIndex, addressIndex }) => {
+  // TODO, move key-utils.js hydra/keychain to asset libs.
+  // https://github.com/ExodusMovement/assets/pull/410#discussion_r1026331735
+  const derivationPath = buildBip32Path({
+    asset: asset,
+    purpose,
+    accountIndex,
+    chainIndex,
+    addressIndex,
+  })
+
+  return {
+    derivationAlgorithm: 'BIP32',
+    derivationPath,
+    keyType: 'secp256k1',
+  }
+}

package/src/tx-log/bitcoin-monitor-scanner.js

@@ -0,0 +1,475 @@
+/* @flow */
+import { orderTxs } from '../insight-api-client/util'
+import { Address, UtxoCollection } from '@exodus/models'
+import { isEqual, compact } from 'lodash'
+import { isRVNAssetScript } from '@exodus/ravencoin-lib'
+import ms from 'ms'
+
+import assert from 'minimalistic-assert'
+import { getUtxos } from '../utxos-utils'
+
+// Time to check whether to drop a sent tx
+const SENT_TIME_TO_DROP = ms('2m')
+
+function isReceiveAddress(addr: Address): boolean {
+  return parsePath(addr.meta.path)[0] === 0
+}
+function isChangeAddress(addr: Address): boolean {
+  return parsePath(addr.meta.path)[0] === 1
+}
+
+function parsePath(path) {
+  let p1 = path ? path.replace('m/', '').split('/') : ['0', '0']
+  p1 = p1.map((i) => parseInt(i, 10))
+  return p1
+}
+
+export async function rescanBlockchainInsight({
+  asset,
+  walletAccount,
+  refresh,
+  assetClientInterface,
+  insightClient,
+  yieldToUI,
+  isIos,
+}) {
+  const assetName = asset.name
+  const purposes = await assetClientInterface.getSupportedPurposes({ assetName, walletAccount })
+  const accountState = await assetClientInterface.getAccountState({ assetName, walletAccount })
+  const currency = asset.currency
+  const currentTxs = await assetClientInterface.getTxLog({ assetName, walletAccount })
+  const currentUtxos = getUtxos({ asset, accountState })
+
+  const currentTime = new Date().getTime()
+  const unconfirmedTxsToCheck = Array.from(currentTxs).reduce((txs, tx) => {
+    if (tx.pending) {
+      // Don't check sent tx that is younger than 2 minutes, because server might not have indexed it yet
+      if (!tx.sent || !tx.date || currentTime - tx.date.getTime() > SENT_TIME_TO_DROP) {
+        txs[tx.txId] = tx
+      }
+    }
+    return txs
+  }, {})
+
+  // this works because txs come back descending
+  // if we receive a batch that is completely in our txLog, we'll assume we reached old territory
+  // we also only consider confirmed txs because unconfirmed txs' order is not guaranteed
+  const shouldStop = refresh
+    ? () => false
+    : (txs) =>
+        txs.filter((tx) => {
+          const txItem = currentTxs.get(tx.txid)
+          return txItem && txItem.confirmed
+        }).length >= txs.length
+
+  const unusedAddressIndexes = await assetClientInterface.getUnusedAddressIndexes({
+    assetName,
+    walletAccount,
+  })
+  /*
+   * The chain fields/variables are number arrays of size 2
+   *
+   * The index of the array represents a chainIndex. Possible values are [0,1]
+   * A value of the array represents the largest unused addressIndex that chainIndex. Possible values are 0+
+   *
+   * The chains and newChains structures have the following format:
+   *
+   * [{ purpose: purposeNumber, chain: [receiveAddressIndexNumber, changeAddressIndexNumber] }, ...]
+   *
+   *  - purpose is the available purpose for the given chain. BTC allows 44, 84, 89 atm.
+   *  - chain is always an array of 2. Each value represent the higher unused addressIndex
+   *  - in multi-address mode on, the wallet would show the highest unused addressIndex address
+   *  - in multi-address mode off, the wallet would show the 0 addressIndex address
+   *
+   * Example:
+   * [ { purpose: 44,  chain: [5, 7] } , { purpose: 84,  chain: [1, 10] }, { purpose: 86,  chain: [0, 0] } ]
+   *
+   * Means:
+   * - for bip/purpose 44 and chainIndex 0 (receive), 5 is the greatest unused addressIndex
+   * - for bip/purpose 44 and chainIndex 1 (change), 7 is the greatest unused addressIndex
+   * - for bip/purpose 84 and chainIndex 0 (receive), 1 is the greatest unused addressIndex
+   * - for bip/purpose 84 and chainIndex 1 (change), 10 is the greatest unused addressIndex
+   */
+
+  const chains = purposes.map((purpose) => ({
+    purpose,
+    chain: unusedAddressIndexes.find((indexes) => indexes.purpose === purpose)?.chain || [0, 0],
+  }))
+
+  // cloning
+  const newChains = chains.map(({ purpose, chain }) => ({ purpose, chain: [...chain] }))
+
+  const addrMap = {}
+  const purposeMap = {}
+
+  // NOTE: the hope is that this gets replaced with a fetch by xpub method and then following the chain here won't be necessary
+
+  const aggregateAddresses = async (chainObjects) => {
+    const promises = []
+    for (let chainObject of chainObjects) {
+      const { purpose, chainIndex, startAddressIndex, endAddressIndex } = chainObject
+      for (let addressIndex = startAddressIndex; addressIndex < endAddressIndex; addressIndex++) {
+        promises.push(
+          assetClientInterface
+            .getAddress({
+              assetName,
+              walletAccount,
+              purpose,
+              chainIndex,
+              addressIndex,
+            })
+            .then((address) => ({ address, purpose }))
+        )
+      }
+    }
+    const addresses = []
+    const result = await Promise.all(promises)
+    result.forEach(({ address, purpose }) => {
+      addrMap[String(address)] = address
+      purposeMap[String(address)] = purpose
+      addresses.push(String(address))
+    })
+    return addresses
+  }
+
+  const fetchAllTxs = async (addresses) => {
+    if (addresses.length === 0) return []
+
+    const promises = []
+    const fetchTimeout = 250
+    const fetchLimit = refresh && isIos && assetName !== 'decred' ? 50 : 10
+    const fetchAddressesSize = 25
+    for (let i = 0; i < addresses.length; i += fetchAddressesSize) {
+      const promise = insightClient.fetchAllTxData(
+        addresses.slice(i, i + fetchAddressesSize),
+        fetchLimit,
+        fetchTimeout,
+        shouldStop
+      )
+      promises.push(promise)
+    }
+
+    const txArrays = await Promise.all(promises)
+    return txArrays.reduce((total, some) => total.concat(some), [])
+  }
+
+  const receiveGapLimit = 10
+  const restoreGapLimit = 10
+  const changeGapLimit = 10
+  const gapLimits = [receiveGapLimit, changeGapLimit]
+  const gapSearchParameters = newChains.map(({ purpose, chain }) => {
+    return {
+      purpose,
+      chain,
+      startAddressIndexes: chain.map(() => 0),
+      endAddressIndexes: chain.map(
+        (addressIndex, chainIndex) =>
+          refresh ? restoreGapLimit : addressIndex + (gapLimits[chainIndex] || receiveGapLimit) // right of the || should never happen
+      ),
+    }
+  })
+
+  let allTxs = []
+
+  for (let fetchCount = 0; ; fetchCount++) {
+    const chainObjects = gapSearchParameters
+      .map(({ purpose, chain, startAddressIndexes, endAddressIndexes }) => {
+        return chain.map((_, chainIndex) => {
+          return {
+            purpose,
+            chainIndex,
+            startAddressIndex: startAddressIndexes[chainIndex],
+            endAddressIndex: endAddressIndexes[chainIndex],
+          }
+        })
+      })
+      .flat()
+
+    const addresses = await aggregateAddresses(chainObjects)
+
+    const txs = await fetchAllTxs(addresses)
+    if (txs.length === 0) break
+    allTxs = allTxs.concat(txs)
+
+    // Update chains to see if we need to fetch more
+    txs.forEach((tx) =>
+      tx.vout.forEach((vout) => {
+        if (!vout.scriptPubKey) return
+        // this is an array because legacy multisig has multiple addresses
+        if (!Array.isArray(vout.scriptPubKey.addresses)) return
+        if (vout.scriptPubKey.addresses.length === 0) return
+        if (!addrMap[vout.scriptPubKey.addresses[0]]) return
+
+        const address = addrMap[vout.scriptPubKey.addresses[0]]
+        // this is a used address, we need to update the chain
+        const pd = address.meta.path.split('/')
+        const metaChainIndex = parseInt(pd[1])
+        const metaAddressIndex = parseInt(pd[2])
+        const addressString = String(address)
+        const purposeToUpdate = purposeMap[addressString]
+        if (!purposeToUpdate) {
+          console.warn(`${assetName}: Cannot resolve purpose from address ${addressString}`)
+          return
+        }
+        const chainToUpgrade = newChains.find((newChain) => newChain.purpose === purposeToUpdate)
+        if (!chainToUpgrade) {
+          console.log(
+            `${assetName}: There is no chain info for purpose ${purposeToUpdate} and address ${addressString}`
+          )
+          return
+        }
+        if (chainToUpgrade.chain[metaChainIndex] === undefined) {
+          console.log(
+            `${assetName}: There is no chain info for purpose ${purposeToUpdate}, address ${addressString} and chain index ${metaChainIndex}`
+          )
+          return
+        }
+        chainToUpgrade.chain[metaChainIndex] = Math.max(
+          metaAddressIndex + 1,
+          chainToUpgrade.chain[metaChainIndex]
+        )
+      })
+    )
+
+    gapSearchParameters.forEach((indexData) => {
+      indexData.startAddressIndexes = [...indexData.endAddressIndexes]
+      indexData.endAddressIndexes = indexData.chain.map((addressIndex, chainIndex) => {
+        return (
+          addressIndex +
+          (refresh && addressIndex === indexData.endAddressIndexes[chainIndex]
+            ? restoreGapLimit
+            : gapLimits[chainIndex] || receiveGapLimit)
+        )
+      })
+    })
+
+    // Safety check. Slow down if after 100 iterations txs are still being fetched.
+    if (fetchCount > 100) await yieldToUI(1000)
+  }
+
+  allTxs = orderTxs(allTxs)
+
+  // post process TX data
+  // NOTE: this can be optimized
+  let vinTxids = {}
+  let utxos = []
+  const utxosToRemove = []
+  let newTxs = []
+  let existingTxs = []
+
+  allTxs.forEach((txItem) => {
+    let txLogItem = {
+      txId: txItem.txid,
+      coinAmount: currency.ZERO,
+      date: txItem.time ? new Date(txItem.time * 1000) : new Date(),
+      coinName: assetName,
+      confirmations: txItem.confirmations ? 1 : 0, // we don't care about a count - this should really be block height based if we want a count
+      addresses: [],
+      error: null,
+      dropped: false,
+      selfSend: false,
+      data: {
+        // feePerKB in satoshis, so multiply by 1e8
+        feePerKB: txItem.fees ? (txItem.fees / txItem.vsize) * 1000 * 1e8 : null,
+        rbfEnabled: txItem.rbf,
+        blocksSeen: 0,
+      },
+    }
+
+    let from = []
+
+    // if txItem.vin has an address that matches ours, means we've spent this tx
+    let isSent = false
+    txItem.vin.forEach((vin) => {
+      // It's an coinbase vin
+      if (Object.keys(vin).length === 0) return
+
+      vinTxids[`${vin.txid}-${vin.vout}`] = true
+      from.push(vin.addr)
+
+      if (!addrMap[vin.addr]) return
+      // it came from us...
+      txLogItem.coinAmount = txLogItem.coinAmount.sub(currency.defaultUnit(vin.value))
+      isSent = true
+      txLogItem.data.sent = []
+
+      // this is only used to exclude the utxos in the reducer which is why we don't care about the other fields
+      utxosToRemove.push({
+        address: addrMap[vin.addr],
+        txId: vin.txid,
+        vout: vin.vout,
+        value: currency.defaultUnit(vin.value || 0),
+      })
+    })
+
+    // Add the inputs for unconfirmed spends in case they drop
+    if (
+      isSent &&
+      txItem.confirmations < 1 &&
+      ['bitcoin', 'bitcoinregtest', 'bitcointestnet'].includes(asset.name)
+    ) {
+      txLogItem.data.inputs = UtxoCollection.fromArray(
+        txItem.vin.map((vin) => ({
+          address: addrMap[vin.addr],
+          script: asset.address.toScriptPubKey(vin.addr).toString('hex'),
+          txId: vin.txid,
+          vout: vin.vout,
+          value: currency.defaultUnit(vin.value || 0),
+        }))
+      ).toJSON()
+    }
+
+    // Fix self send tx
+    const voutAddresses = compact(
+      txItem.vout.map((vout) => {
+        if (!vout.scriptPubKey) return
+        // this is an array because legacy multisig has multiple addresses
+        if (!Array.isArray(vout.scriptPubKey.addresses)) return
+        if (vout.scriptPubKey.addresses.length === 0) return
+        return vout.scriptPubKey.addresses[0]
+      })
+    )
+    const isSelfSent = isSent && voutAddresses.every((v) => addrMap[v])
+    if (isSelfSent) {
+      txLogItem.to = voutAddresses[0]
+    }
+
+    txItem.vout.forEach((vout) => {
+      if (!vout.scriptPubKey) return
+      // this is an array because legacy multisig has multiple addresses
+      if (!Array.isArray(vout.scriptPubKey.addresses)) return
+      if (vout.scriptPubKey.addresses.length === 0) return
+      if (!addrMap[vout.scriptPubKey.addresses[0]]) {
+        if (isSent && !txLogItem.to) {
+          const val = currency.defaultUnit(vout.value)
+          txLogItem.data.sent.push({ address: vout.scriptPubKey.addresses[0], amount: val })
+        }
+        return
+      }
+
+      const address = addrMap[vout.scriptPubKey.addresses[0]]
+      if (isReceiveAddress(address)) {
+        txLogItem.addresses.push(address)
+      }
+
+      if (isSent && isChangeAddress(address)) {
+        txLogItem.data.changeAddress = address
+      }
+
+      // it was sent to us...
+      let val = currency.defaultUnit(vout.value)
+      txLogItem.coinAmount = txLogItem.coinAmount.add(val)
+
+      const output = {
+        address,
+        txId: txItem.txid,
+        vout: vout.n,
+        confirmations: txLogItem.confirmations,
+        script: vout.scriptPubKey.hex,
+        value: val,
+        rbfEnabled: txItem.rbf,
+      }
+
+      if (assetName === 'ravencoin') {
+        try {
+          if (isRVNAssetScript(Buffer.from(output.script, 'hex'))) return
+        } catch (err) {
+          console.warn(err)
+          return
+        }
+      }
+      if (vout.spentTxId) return
+      utxos.push(output) // but save the unspent ones for state.utxos
+    })
+
+    // BCH: convert to cash format
+    if (assetName === 'bcash') {
+      if (txLogItem.to) {
+        txLogItem.to = asset.address.toCashAddress(txLogItem.to)
+      }
+
+      from = from.map(asset.address.toCashAddress)
+    }
+
+    if (isSent) {
+      txLogItem.feeAmount = currency.defaultUnit(txItem.fees || 0)
+      if (isSelfSent) {
+        txLogItem.selfSend = true
+        txLogItem.coinAmount = currency.ZERO
+      } else {
+        // we want coinAmount to be without the fee
+        // so far coinAmount = -vins + vouts (to us) = - (sent amount + fees)
+        // add fees to get only sent amount
+        txLogItem.coinAmount = txLogItem.coinAmount.add(txLogItem.feeAmount)
+      }
+    } else {
+      txLogItem.from = from
+    }
+
+    if (currentTxs.has(txLogItem.txId)) {
+      const existingItem = currentTxs.get(txLogItem.txId)
+      // we only want to detect changes when it matters - when a tx confirms
+      if (
+        existingItem.confirmations !== txLogItem.confirmations ||
+        existingItem.selfSend !== txLogItem.selfSend ||
+        existingItem.dropped
+      ) {
+        existingTxs.push(txLogItem)
+      }
+
+      delete unconfirmedTxsToCheck[txLogItem.txId] // we remove what we find, so we are left with what wasn't found
+    } else {
+      newTxs.push(txLogItem)
+    }
+  })
+
+  // this protects from the server returning bad spentTxId
+  utxos = utxos.filter((utxo) => {
+    if (!vinTxids[`${utxo.txId}-${utxo.vout}`]) return utxo
+  })
+
+  let utxoCol = UtxoCollection.fromArray(utxos, { currency })
+
+  const utxosToRemoveCol = UtxoCollection.fromArray(utxosToRemove, { currency })
+  // if something else can update currentUtxos in the meantime, we probably need to grab it from state again
+  utxoCol = utxoCol.union(currentUtxos.difference(utxosToRemoveCol))
+
+  for (let tx of Object.values(unconfirmedTxsToCheck)) {
+    existingTxs.push({ ...tx, dropped: true }) // TODO: this will decrease the chain index, it shouldn't be an issue considering the gap limit
+    utxoCol = utxoCol.difference(utxoCol.getTxIdUtxos(tx.txId))
+    const utxosToAdd = []
+    if (tx.data.inputs) {
+      const prevUtxos = UtxoCollection.fromJSON(tx.data.inputs, { currency })
+      for (let utxo of prevUtxos.toArray()) {
+        if (vinTxids[`${utxo.txId}-${utxo.vout}`]) {
+          // This utxo was already spent in another tx
+          continue
+        }
+        const tx = await insightClient.fetchTx(utxo.txId)
+        if (tx) {
+          // previously spent tx still exists, readd utxo
+          utxosToAdd.push(utxo)
+        }
+      }
+    }
+    utxoCol = utxoCol.union(UtxoCollection.fromArray(utxosToAdd, { currency }))
+  }
+
+  // no changes, ignore
+  if (utxoCol.equals(currentUtxos)) {
+    utxoCol = null
+  }
+
+  const changedUnusedAddressIndexes = newChains.filter(({ purpose, chain }, index) => {
+    const originalChain = chains[index]
+    assert(originalChain, `originalChain with purpose ${purpose} not found!`)
+    return !isEqual(chain, originalChain.chain)
+  })
+
+  return {
+    txsToUpdate: existingTxs,
+    txsToAdd: newTxs,
+    utxos: utxoCol,
+    changedUnusedAddressIndexes,
+  }
+}