edge-currency-plugins 1.0.0-rc.1

package/lib/common/utxobased/engine/makeUtxoEngineState.js

@@ -0,0 +1,1443 @@
+"use strict";Object.defineProperty(exports, "__esModule", {value: true}); function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } function _nullishCoalesce(lhs, rhsFn) { if (lhs != null) { return lhs; } else { return rhsFn(); } } async function _asyncNullishCoalesce(lhs, rhsFn) { if (lhs != null) { return lhs; } else { return await rhsFn(); } } function _optionalChain(ops) { let lastAccessLHS = undefined; let value = ops[0]; let i = 1; while (i < ops.length) { const op = ops[i]; const fn = ops[i + 1]; i += 2; if ((op === 'optionalAccess' || op === 'optionalCall') && value == null) { return undefined; } if (op === 'access' || op === 'optionalAccess') { lastAccessLHS = value; value = fn(value); } else if (op === 'call' || op === 'optionalCall') { value = fn((...args) => value.call(lastAccessLHS, ...args)); lastAccessLHS = undefined; } } return value; }var _biggystring = require('biggystring');
+
+
+
+
+
+
+
+var _makeEngineEmitter = require('../../plugin/makeEngineEmitter');
+
+
+
+
+
+
+
+
+var _utils = require('../../plugin/utils');
+
+var _ProcessorTransaction = require('../db/Models/ProcessorTransaction');
+
+
+
+
+
+var _types5 = require('../db/types');
+
+
+
+
+
+var _keymanager = require('../keymanager/keymanager');
+
+
+
+
+
+
+
+
+
+
+var _BlockBookAPI = require('../network/BlockBookAPI');
+var _Deferred = require('../network/Deferred'); var _Deferred2 = _interopRequireDefault(_Deferred);
+
+var _awaitlock = require('./await-lock'); var _awaitlock2 = _interopRequireDefault(_awaitlock);
+var _constants = require('./constants');
+var _makeServerStates = require('./makeServerStates');
+
+
+
+
+
+
+var _utils3 = require('./utils');
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ function makeUtxoEngineState(
+  config
+) {
+  const {
+    pluginInfo,
+    walletInfo,
+    walletTools,
+    options: { emitter, log },
+    processor,
+    pluginState
+  } = config
+
+  const { walletFormats } = walletInfo.keys
+
+  const taskCache = {
+    addressWatching: false,
+    blockWatching: false,
+    addressSubscribeCache: {},
+    addressTransactionCache: {},
+    addressUtxoCache: {},
+    rawUtxoCache: {},
+    processorUtxoCache: {},
+    updateTransactionCache: {}
+  }
+
+  const clearTaskCache = () => {
+    taskCache.addressWatching = false
+    taskCache.blockWatching = false
+    taskCache.addressSubscribeCache = {}
+    taskCache.addressTransactionCache = {}
+    taskCache.addressUtxoCache = {}
+    taskCache.rawUtxoCache = {}
+    taskCache.processorUtxoCache = {}
+    taskCache.updateTransactionCache = {}
+  }
+
+  /**
+   * There are two processes that need to complete per address:
+   * 1. Address transaction processing
+   * 2. Address UTXO processing
+   **/
+  const processesPerAddress = 2
+  let processedCount = 0
+  let processedPercent = 0
+  const updateProgressRatio = async () => {
+    // We expect the total number of progress updates to equal the number of
+    // addresses multiplied the number of processes per address.
+    const expectedProccessCount =
+      Object.keys(taskCache.addressSubscribeCache).length * processesPerAddress
+
+    // Increment the processed count
+    processedCount = processedCount + 1
+
+    // If we have no addresses, we should not have not yet began processing.
+    if (expectedProccessCount === 0) throw new Error('No addresses to process')
+
+    const percent = processedCount / expectedProccessCount
+    if (percent - processedPercent > _constants.CACHE_THROTTLE || percent === 1) {
+      log(
+        `processed changed, percent: ${percent}, processedCount: ${processedCount}, totalCount: ${expectedProccessCount}`
+      )
+      processedPercent = percent
+      emitter.emit(_makeEngineEmitter.EngineEvent.ADDRESSES_CHECKED, percent)
+    }
+  }
+
+  const lock = new (0, _awaitlock2.default)()
+
+  const serverStates = _makeServerStates.makeServerStates.call(void 0, {
+    engineEmitter: emitter,
+    log,
+    pluginInfo,
+    pluginState,
+    walletInfo
+  })
+  const commonArgs = {
+    pluginInfo,
+    walletInfo,
+    walletTools,
+    processor,
+    emitter,
+    taskCache,
+    updateProgressRatio,
+    io: config.io,
+    log,
+    serverStates,
+    walletFormats,
+    lock
+  }
+  const { currencyInfo } = pluginInfo
+  const { requiredConfirmations = 1 } = currencyInfo
+
+  const pickNextTaskCB = async (
+    uri
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  ) => {
+    return await exports.pickNextTask.call(void 0, { ...commonArgs, uri })
+  }
+
+  serverStates.setPickNextTaskCB(pickNextTaskCB)
+
+  let running = false
+  const run = async () => {
+    if (running) return
+    running = true
+
+    await initializeAddressSubscriptions()
+    await setLookAhead(commonArgs)
+  }
+
+  emitter.on(
+    _makeEngineEmitter.EngineEvent.BLOCK_HEIGHT_CHANGED,
+    async (_uri, walletBlockHeight) => {
+      // Check if unconfirmed transaction have been confirmed.
+      const unconfirmedTxs = await processor.fetchTransactions({
+        blockHeight: 0
+      })
+      for (const tx of unconfirmedTxs) {
+        if (tx == null) continue
+        taskCache.updateTransactionCache[tx.txid] = { processing: false }
+      }
+
+      // Trigger transactions change event to update its confirmation status
+      const pendingTxs = await processor.fetchTransactions({
+        blockHeight: walletBlockHeight - requiredConfirmations,
+        blockHeightMax: walletBlockHeight
+      })
+      for (const tx of pendingTxs) {
+        if (tx == null) continue
+        await exports.transactionChanged.call(void 0, {
+          emitter,
+          processor,
+          pluginInfo,
+          tx,
+          walletBlockHeight,
+          walletTools
+        })
+      }
+    }
+  )
+
+  emitter.on(
+    _makeEngineEmitter.EngineEvent.NEW_ADDRESS_TRANSACTION,
+    async (_uri, response) => {
+      const state = taskCache.addressSubscribeCache[response.address]
+      if (state != null) {
+        const { path } = state
+        taskCache.addressUtxoCache[response.address] = {
+          processing: false,
+          path
+        }
+        addToAddressTransactionCache(
+          commonArgs,
+          response.address,
+          path.format,
+          path.changeIndex,
+          0,
+          taskCache.addressTransactionCache
+        ).catch(() => {
+          throw new Error('failed to add to transaction cache')
+        })
+        setLookAhead(commonArgs).catch(e => {
+          log(e)
+        })
+      }
+    }
+  )
+
+  // Initialize the addressSubscribeCache with the existing addresses already
+  // processed by the processor. This happens only once before any call to
+  // setLookAhead.
+  const initializeAddressSubscriptions = async () => {
+    for (const format of walletFormats) {
+      const branches = _utils3.getFormatSupportedBranches.call(void 0, format)
+      for (const branch of branches) {
+        const addressesToSubscribe = new Set()
+        const branchAddressCount = processor.numAddressesByFormatPath({
+          format,
+          changeIndex: branch
+        })
+        // If the processor has not processed any addresses then the loop
+        // condition will only iterate once when branchAddressCount is 0 for the
+        // first address in the derivation path.
+        for (
+          let addressIndex = 0;
+          addressIndex < branchAddressCount;
+          addressIndex++
+        ) {
+          const processorAddress = await processor.fetchAddress({
+            format,
+            changeIndex: branch,
+            addressIndex
+          })
+          if (processorAddress == null) {
+            throw new Error(
+              'Missing processor during address subscription initialization'
+            )
+          }
+          const { address } = walletTools.scriptPubkeyToAddress({
+            scriptPubkey: processorAddress.scriptPubkey,
+            format
+          })
+          addressesToSubscribe.add(address)
+        }
+        addToAddressSubscribeCache(commonArgs.taskCache, addressesToSubscribe, {
+          format,
+          changeIndex: branch
+        })
+      }
+    }
+  }
+
+  return {
+    processedPercent,
+    async start() {
+      processedCount = 0
+      processedPercent = 0
+
+      await run()
+      serverStates.refillServers()
+    },
+
+    async stop() {
+      serverStates.stop()
+      clearTaskCache()
+      running = false
+    },
+
+    async getFreshAddress(branch = 0) {
+      const walletPurpose = _utils3.currencyFormatToPurposeType.call(void 0, 
+        walletInfo.keys.primaryFormat
+      )
+      if (walletPurpose === _keymanager.BIP43PurposeTypeEnum.Segwit) {
+        const { address: publicAddress } = await internalGetFreshAddress({
+          ...commonArgs,
+          format: _utils3.getCurrencyFormatFromPurposeType.call(void 0, 
+            _keymanager.BIP43PurposeTypeEnum.WrappedSegwit
+          ),
+          changeIndex: branch
+        })
+
+        const { address: segwitAddress } = await internalGetFreshAddress({
+          ...commonArgs,
+          format: _utils3.getCurrencyFormatFromPurposeType.call(void 0, _keymanager.BIP43PurposeTypeEnum.Segwit),
+          changeIndex: branch
+        })
+
+        return {
+          publicAddress,
+          segwitAddress
+        }
+      } else {
+        // Airbitz wallets only use branch 0
+        if (walletPurpose === _keymanager.BIP43PurposeTypeEnum.Airbitz) {
+          branch = 0
+        }
+
+        const {
+          address: publicAddress,
+          legacyAddress
+        } = await internalGetFreshAddress({
+          ...commonArgs,
+          format: _utils3.getCurrencyFormatFromPurposeType.call(void 0, walletPurpose),
+          changeIndex: branch
+        })
+
+        return {
+          publicAddress,
+          legacyAddress:
+            legacyAddress !== publicAddress ? legacyAddress : undefined
+        }
+      }
+    },
+
+    async deriveScriptAddress(script) {
+      const { address } = await internalDeriveScriptAddress({
+        walletTools: commonArgs.walletTools,
+        engineInfo: commonArgs.pluginInfo.engineInfo,
+        processor: commonArgs.processor,
+        taskCache: commonArgs.taskCache,
+        format: walletInfo.keys.primaryFormat,
+        script
+      })
+      return {
+        publicAddress: address
+      }
+    },
+
+    async addGapLimitAddresses(addresses) {
+      const promises = addresses.map(async address => {
+        const scriptPubkey = walletTools.addressToScriptPubkey(address)
+        await processor.saveAddress(
+          _types5.makeIAddress.call(void 0, {
+            scriptPubkey,
+            used: true
+          })
+        )
+      })
+      await Promise.all(promises)
+      await run()
+    },
+
+    async broadcastTx(transaction) {
+      const txId = await serverStates.broadcastTx(transaction)
+
+      /*
+      After successfully broadcasting the tx, we must mark each UTXO, used as
+      an input for the tx, as spent so we don't try to reuse the UTXO.
+      */
+      const txInputs = _nullishCoalesce(_optionalChain([transaction, 'access', _ => _.otherParams, 'optionalAccess', _2 => _2.psbt, 'access', _3 => _3.inputs]), () => ( []))
+      for (const input of txInputs) {
+        const scriptPubkey = Buffer.from(input.scriptPubkey).toString('hex')
+        const [utxo] = await processor.fetchUtxos({
+          scriptPubkey
+        })
+        if (utxo != null) {
+          utxo.spent = true
+          await processor.saveUtxo(utxo)
+        }
+      }
+
+      return txId
+    },
+    refillServers() {
+      serverStates.refillServers()
+    },
+    getServerList() {
+      return serverStates.getServerList()
+    },
+    setServerList(serverList) {
+      serverStates.setServerList(serverList)
+    },
+
+    async loadWifs(wifs) {
+      for (const wif of wifs) {
+        for (const format of walletFormats) {
+          const changePath = {
+            format,
+            changeIndex: 0
+          }
+          const path = {
+            ...changePath,
+            addressIndex: 0
+          }
+          const {
+            scriptPubkey,
+            redeemScript
+          } = walletTools.getScriptPubkeyFromWif(wif, format)
+          const { address } = walletTools.scriptPubkeyToAddress({
+            scriptPubkey,
+            format: path.format
+          })
+
+          // Make a new IAddress and save it
+          await processor.saveAddress(
+            _types5.makeIAddress.call(void 0, { scriptPubkey, redeemScript, path })
+          )
+
+          taskCache.addressSubscribeCache[address] = {
+            path: changePath,
+            processing: false
+          }
+        }
+      }
+      taskCache.addressWatching = false
+    }
+  }
+} exports.makeUtxoEngineState = makeUtxoEngineState;
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+const setLookAhead = async (common) => {
+  const {
+    pluginInfo: { engineInfo },
+    lock,
+    processor,
+    walletFormats,
+    walletTools
+  } = common
+
+  // Wait for the lock to be released before continuing invocation.
+  // This is to ensure that setLockAhead is not called while the lock is held.
+  await lock.acquireAsync()
+
+  try {
+    for (const format of walletFormats) {
+      const branches = _utils3.getFormatSupportedBranches.call(void 0, format)
+      for (const branch of branches) {
+        await deriveKeys({ format, changeIndex: branch })
+      }
+    }
+  } finally {
+    lock.release()
+  }
+
+  async function deriveKeys(changePath) {
+    const addressesToSubscribe = new Set()
+    const totalAddressCount = processor.numAddressesByFormatPath(changePath)
+    let lastUsedIndex = await processor.lastUsedIndexByFormatPath(changePath)
+
+    // Loop until the total address count equals the lookahead count
+    let lookAheadIndex = lastUsedIndex + engineInfo.gapLimit
+    let nextAddressIndex = totalAddressCount
+    while (nextAddressIndex <= lookAheadIndex) {
+      const path = {
+        ...changePath,
+        addressIndex: nextAddressIndex
+      }
+
+      const { scriptPubkey, redeemScript } = walletTools.getScriptPubkey(path)
+      const { address } = walletTools.scriptPubkeyToAddress({
+        scriptPubkey,
+        format: path.format
+      })
+
+      // Make a new IAddress and save it
+      await processor.saveAddress(
+        _types5.makeIAddress.call(void 0, { scriptPubkey, redeemScript, path })
+      )
+
+      // Add the displayAddress to the set of addresses to subscribe to after loop
+      addressesToSubscribe.add(address)
+
+      // Update the state for the loop
+      lastUsedIndex = await processor.lastUsedIndexByFormatPath(changePath)
+      lookAheadIndex = lastUsedIndex + engineInfo.gapLimit
+      nextAddressIndex = processor.numAddressesByFormatPath(changePath)
+    }
+
+    // Add all the addresses to the subscribe cache for registering subscriptions later
+    addToAddressSubscribeCache(
+      common.taskCache,
+      addressesToSubscribe,
+      changePath
+    )
+  }
+}
+
+const addToAddressSubscribeCache = (
+  taskCache,
+  addresses,
+  path
+) => {
+  addresses.forEach(address => {
+    taskCache.addressSubscribeCache[address] = {
+      path,
+      processing: false
+    }
+    taskCache.addressWatching = false
+  })
+}
+
+const addToAddressTransactionCache = async (
+  args,
+  address,
+  format,
+  branch,
+  blockHeight,
+  addressTransactionCache
+) => {
+  const { walletTools, processor } = args
+  // Fetch the blockHeight for the address from the database
+  const scriptPubkey = walletTools.addressToScriptPubkey(address)
+
+  if (blockHeight === 0) {
+    const { lastQueriedBlockHeight = 0 } =
+      await _asyncNullishCoalesce((await processor.fetchAddress(scriptPubkey)), async () => ( {}))
+    blockHeight = lastQueriedBlockHeight
+  }
+
+  addressTransactionCache[address] = {
+    processing: false,
+    path: {
+      format,
+      changeIndex: branch
+    },
+    page: 1, // Page starts on 1
+    blockHeight
+  }
+}
+
+
+
+
+
+
+
+
+
+
+ const transactionChanged = async (
+  args
+) => {
+  const {
+    emitter,
+    walletBlockHeight,
+    walletTools,
+    processor,
+    pluginInfo,
+    tx
+  } = args
+  const { currencyInfo } = pluginInfo
+
+  emitter.emit(_makeEngineEmitter.EngineEvent.TRANSACTIONS_CHANGED, [
+    await _ProcessorTransaction.toEdgeTransaction.call(void 0, {
+      tx,
+      currencyCode: currencyInfo.currencyCode,
+      pluginInfo,
+      processor,
+      walletBlockHeight: walletBlockHeight,
+      walletTools
+    })
+  ])
+}; exports.transactionChanged = transactionChanged
+
+
+
+
+
+ const pickNextTask = async (
+  args
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+) => {
+  const { taskCache, uri, serverStates } = args
+
+  const {
+    addressSubscribeCache,
+    addressUtxoCache,
+    rawUtxoCache,
+    processorUtxoCache,
+    addressTransactionCache,
+    updateTransactionCache
+  } = taskCache
+
+  const serverState = serverStates.getServerState(uri)
+  if (serverState == null) return
+
+  // subscribe all servers to new blocks
+  if (serverState.blockSubscriptionStatus === 'unsubscribed') {
+    serverStates.watchBlocks(uri)
+    return true
+  }
+
+  // Loop processed utxos, these are just database ops, triggers setLookAhead
+  if (Object.keys(processorUtxoCache).length > 0) {
+    for (const [scriptPubkey, state] of Object.entries(processorUtxoCache)) {
+      // Only process when all utxos for a specific address have been gathered
+      if (!state.processing && state.full) {
+        state.processing = true
+        await processProcessorUtxos({
+          ...args,
+          scriptPubkey,
+          utxos: state.utxos,
+          path: state.path
+        })
+        _utils.removeItem.call(void 0, processorUtxoCache, scriptPubkey)
+        return true
+      }
+    }
+  }
+
+  const blockHeight = serverStates.getBlockHeight(uri)
+
+  // Loop unparsed utxos, some require a network call to get the full tx data
+  for (const [utxoString, state] of Object.entries(rawUtxoCache)) {
+    const utxo = JSON.parse(utxoString)
+    if (utxo == null) continue
+    if (!state.processing) {
+      // check if we need to fetch additional network content for legacy purpose type
+      const purposeType = _utils3.currencyFormatToPurposeType.call(void 0, state.path.format)
+      if (
+        purposeType === _keymanager.BIP43PurposeTypeEnum.Airbitz ||
+        purposeType === _keymanager.BIP43PurposeTypeEnum.Legacy
+      ) {
+        // if we do need to make a network call, check with the serverState
+        if (!serverStates.serverCanGetTx(uri, utxo.txid)) return
+      }
+      state.processing = true
+      _utils.removeItem.call(void 0, rawUtxoCache, utxoString)
+      const wsTask = await processRawUtxo({
+        ...args,
+        ...state,
+        utxo,
+        id: `${utxo.txid}_${utxo.vout}`
+      })
+      return _nullishCoalesce(wsTask, () => ( true))
+    }
+  }
+
+  // Loop to process addresses to utxos
+  for (const [address, state] of Object.entries(addressUtxoCache)) {
+    // Check if we need to fetch address UTXOs
+    if (!state.processing && serverStates.serverCanGetAddress(uri, address)) {
+      state.processing = true
+
+      _utils.removeItem.call(void 0, addressUtxoCache, address)
+
+      // Fetch and process address UTXOs
+      const wsTask = await processAddressUtxos({
+        ...args,
+        ...state,
+        address
+      })
+      wsTask.deferred.promise
+        .then(() => {
+          serverState.addresses.add(address)
+        })
+        .catch(e => {
+          throw e
+        })
+      return wsTask
+    }
+  }
+
+  // Check if there are any addresses pending to be subscribed
+  if (
+    Object.keys(addressSubscribeCache).length > 0 &&
+    !taskCache.addressWatching
+  ) {
+    // Loop each address that needs to be subscribed
+    for (const [address, state] of Object.entries(addressSubscribeCache)) {
+      // Add address in the cache to the set of addresses to watch
+      const { path, processing: subscribed } = state
+      // only process newly watched addresses
+      if (subscribed) continue
+      if (path != null) {
+        // Add the newly watched addresses to the UTXO cache
+        addressUtxoCache[address] = {
+          processing: false,
+          path
+        }
+        await addToAddressTransactionCache(
+          args,
+          address,
+          path.format,
+          path.changeIndex,
+          blockHeight,
+          addressTransactionCache
+        )
+      }
+      state.processing = true
+    }
+
+    taskCache.addressWatching = true
+
+    const queryTime = Date.now()
+    const deferred = new _Deferred2.default()
+    deferred.promise
+      .then(() => {
+        serverStates.serverScoreUp(uri, Date.now() - queryTime)
+      })
+      .catch(() => {
+        taskCache.addressWatching = false
+      })
+    deferred.promise.catch(() => {
+      taskCache.addressWatching = false
+    })
+    serverStates.watchAddresses(
+      uri,
+      Array.from(Object.keys(addressSubscribeCache)),
+      deferred
+    )
+    return true
+  }
+
+  // filled when transactions potentially changed (e.g. through new block notification)
+  if (Object.keys(updateTransactionCache).length > 0) {
+    let hasProcessedAtLeastOnce = false
+    for (const [txId, state] of Object.entries(updateTransactionCache)) {
+      if (!state.processing && serverStates.serverCanGetTx(uri, txId)) {
+        hasProcessedAtLeastOnce = true
+        state.processing = true
+        _utils.removeItem.call(void 0, updateTransactionCache, txId)
+        const updateTransactionTask = updateTransactions({
+          ...args,
+          walletBlockHeight: blockHeight,
+          txId
+        })
+        // once resolved, add the txid to the server cache
+        updateTransactionTask.deferred.promise
+          .then(() => {
+            serverState.txids.add(txId)
+          })
+          .catch(e => {
+            throw e
+          })
+        return updateTransactionTask
+      }
+    }
+    // This condition prevents infinite loops
+    if (hasProcessedAtLeastOnce) return true
+  }
+
+  // loop to get and process transaction history of single addresses, triggers setLookAhead
+  for (const [address, state] of Object.entries(addressTransactionCache)) {
+    if (!state.processing && serverStates.serverCanGetAddress(uri, address)) {
+      state.processing = true
+
+      _utils.removeItem.call(void 0, addressTransactionCache, address)
+
+      // Fetch and process address UTXOs
+      const wsTask = await processAddressTransactions({
+        ...args,
+        ...state,
+        address
+      })
+      wsTask.deferred.promise
+        .then(() => {
+          serverState.addresses.add(address)
+        })
+        .catch(e => {
+          throw e
+        })
+      return wsTask
+    }
+  }
+}; exports.pickNextTask = pickNextTask
+
+
+
+
+
+
+const updateTransactions = (
+  args
+) => {
+  const {
+    walletBlockHeight,
+    emitter,
+    walletTools,
+    txId,
+    pluginInfo,
+    processor,
+    taskCache
+  } = args
+  const deferred = new _Deferred2.default()
+  deferred.promise
+    .then(async (txResponse) => {
+      // check if raw tx is still not confirmed, if so, don't change anything
+      if (txResponse.blockHeight < 1) return
+      // Create new tx from raw tx
+      const tx = processTransactionResponse({
+        ...args,
+        txResponse
+      })
+      // Remove any existing input utxos from the processor
+      for (const input of tx.inputs) {
+        await processor.removeUtxos([`${input.txId}_${input.outputIndex}`])
+      }
+      // Update output utxos's blockHeight any existing input utxos from the processor
+      const utxoIds = tx.outputs.map(output => `${tx.txid}_${output.n}`)
+      const utxos = await processor.fetchUtxos({
+        utxoIds
+      })
+      for (const utxo of utxos) {
+        if (utxo == null) continue
+        utxo.blockHeight = tx.blockHeight
+        await processor.saveUtxo(utxo)
+      }
+      // Process and save new tx
+      const processedTx = await processor.saveTransaction({
+        tx
+      })
+      await exports.transactionChanged.call(void 0, {
+        emitter,
+        processor,
+        pluginInfo,
+        tx: processedTx,
+        walletBlockHeight,
+        walletTools
+      })
+    })
+    .catch(() => {
+      taskCache.updateTransactionCache[txId] = { processing: false }
+    })
+  return {
+    ..._BlockBookAPI.transactionMessage.call(void 0, txId),
+    deferred
+  }
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+const internalDeriveScriptAddress = async ({
+  walletTools,
+  engineInfo,
+  processor,
+  format,
+  taskCache,
+  script
+}) => {
+  if (engineInfo.scriptTemplates == null) {
+    throw new Error(
+      `cannot derive script address ${script} without defined script template`
+    )
+  }
+
+  const scriptTemplate = engineInfo.scriptTemplates[script]
+
+  const path = {
+    format,
+    changeIndex: _keymanager.derivationLevelScriptHash.call(void 0, scriptTemplate),
+    addressIndex: 0
+  }
+
+  // save the address to the processor and add it to the cache
+  const { address, scriptPubkey, redeemScript } = walletTools.getScriptAddress({
+    path,
+    scriptTemplate
+  })
+  await processor.saveAddress(
+    _types5.makeIAddress.call(void 0, { scriptPubkey, redeemScript, path })
+  )
+  const addresses = new Set()
+  addresses.add(address)
+  addToAddressSubscribeCache(taskCache, addresses, {
+    format: path.format,
+    changeIndex: path.changeIndex
+  })
+  return { address, scriptPubkey, redeemScript }
+}
+
+
+
+
+
+
+
+
+const internalGetFreshAddress = async (
+  args
+) => {
+  const { format, changeIndex: branch, walletTools, processor } = args
+
+  const numAddresses = processor.numAddressesByFormatPath({
+    format,
+    changeIndex: branch
+  })
+
+  const path = {
+    format,
+    changeIndex: branch,
+    // while syncing, we may hit negative numbers when only subtracting. Use the address at /0 in that case.
+    addressIndex: Math.max(
+      numAddresses - args.pluginInfo.engineInfo.gapLimit,
+      0
+    )
+  }
+  const { scriptPubkey } =
+    await _asyncNullishCoalesce((await processor.fetchAddress(path)), async () => (
+    (await walletTools.getScriptPubkey(path))))
+  if (scriptPubkey == null) {
+    throw new Error('Unknown address path')
+  }
+  return walletTools.scriptPubkeyToAddress({
+    scriptPubkey,
+    format
+  })
+}
+
+
+
+
+
+
+
+
+
+
+const processAddressTransactions = async (
+  args
+) => {
+  const {
+    address,
+    page = 1,
+    blockHeight: walletBlockHeight,
+    emitter,
+    pluginInfo,
+    processor,
+    walletTools,
+    path,
+    taskCache,
+    serverStates,
+    uri
+  } = args
+  const {
+    engineInfo: { asBlockbookAddress }
+  } = pluginInfo
+  const addressTransactionCache = taskCache.addressTransactionCache
+
+  const scriptPubkey = walletTools.addressToScriptPubkey(address)
+  const addressData = await processor.fetchAddress(scriptPubkey)
+  if (addressData == null) {
+    throw new Error(`could not find address with script pubkey ${scriptPubkey}`)
+  }
+
+  const queryTime = Date.now()
+  const deferred = new _Deferred2.default()
+  deferred.promise
+    .then(async (value) => {
+      serverStates.serverScoreUp(uri, Date.now() - queryTime)
+      const { transactions = [], txs, unconfirmedTxs, totalPages } = value
+
+      // If address is used and previously not marked as used, mark as used.
+      const used = txs > 0 || unconfirmedTxs > 0
+      if (!addressData.used && used && page === 1) {
+        addressData.used = true
+      }
+
+      // Process and save the address's transactions
+      for (const txResponse of transactions) {
+        const tx = processTransactionResponse({
+          ...args,
+          txResponse
+        })
+        const processedTx = await processor.saveTransaction({
+          tx,
+          scriptPubkeys: [scriptPubkey]
+        })
+        await exports.transactionChanged.call(void 0, {
+          emitter,
+          processor,
+          pluginInfo,
+          tx: processedTx,
+          walletBlockHeight,
+          walletTools
+        })
+      }
+
+      // Halt on finishing the processing of address transaction until
+      // we have progressed through all of the blockbook pages
+      if (page < totalPages) {
+        // Add the address back to the cache, incrementing the page
+        addressTransactionCache[address] = {
+          path,
+          processing: false,
+          blockHeight: walletBlockHeight,
+          page: page + 1
+        }
+        return
+      }
+
+      // Update the lastQueriedBlockHeight for the address
+      addressData.lastQueriedBlockHeight = walletBlockHeight
+
+      // Save/update the fully-processed address
+      await processor.saveAddress(addressData)
+
+      // Update the progress now that the transactions for an address have processed
+      await args.updateProgressRatio()
+
+      // Call setLookAhead to update the lookahead
+      await setLookAhead(args)
+    })
+    .catch(err => {
+      // Log the error for debugging purposes without crashing the engine
+      // This will cause frozen wallet syncs
+      console.error(err.toString())
+      console.log(err.stack)
+    })
+
+  return {
+    ..._BlockBookAPI.addressMessage.call(void 0, address, asBlockbookAddress, {
+      details: 'txs',
+      from: addressData.lastQueriedBlockHeight,
+      pageSize: _constants.BLOCKBOOK_TXS_PER_PAGE,
+      page
+    }),
+    deferred
+  }
+}
+
+
+
+
+
+const processTransactionResponse = (
+  args
+) => {
+  const { txResponse, pluginInfo } = args
+  const { coinInfo } = pluginInfo
+
+  return {
+    txid: txResponse.txid,
+    hex: txResponse.hex,
+    // Blockbook can return a blockHeight of -1 when the tx is pending in the mempool
+    blockHeight: txResponse.blockHeight > 0 ? txResponse.blockHeight : 0,
+    date: txResponse.blockTime,
+    fees: txResponse.fees,
+    inputs: txResponse.vin.map(input => ({
+      txId: input.txid,
+      outputIndex: input.vout, // case for tx `fefac8c22ba1178df5d7c90b78cc1c203d1a9f5f5506f7b8f6f469fa821c2674` no `vout` for input
+      n: input.n,
+      scriptPubkey: _utils3.validScriptPubkeyFromAddress.call(void 0, {
+        address: input.addresses[0],
+        coin: coinInfo.name
+      }),
+      amount: input.value
+    })),
+    outputs: txResponse.vout.map(output => ({
+      n: output.n,
+      scriptPubkey:
+        _nullishCoalesce(output.hex, () => (
+        _utils3.validScriptPubkeyFromAddress.call(void 0, {
+          address: output.addresses[0],
+          coin: coinInfo.name
+        }))),
+      amount: output.value
+    })),
+    ourIns: [],
+    ourOuts: [],
+    ourAmount: '0'
+  }
+}
+
+
+
+
+
+
+
+
+const processAddressUtxos = async (
+  args
+) => {
+  const {
+    address,
+    walletTools,
+    processor,
+    taskCache,
+    path,
+    pluginInfo,
+    serverStates,
+    uri
+  } = args
+  const {
+    engineInfo: { asBlockbookAddress }
+  } = pluginInfo
+  const { addressUtxoCache, rawUtxoCache, processorUtxoCache } = taskCache
+  const queryTime = Date.now()
+  const deferred = new _Deferred2.default()
+  deferred.promise
+    .then(async (utxos) => {
+      serverStates.serverScoreUp(uri, Date.now() - queryTime)
+      const scriptPubkey = walletTools.addressToScriptPubkey(address)
+      const addressData = await processor.fetchAddress(scriptPubkey)
+      if (addressData == null || addressData.path == null) {
+        return
+      }
+
+      if (utxos.length === 0) {
+        addToProcessorUtxoCache(processorUtxoCache, path, scriptPubkey, 0)
+        return
+      }
+
+      for (const utxo of utxos) {
+        rawUtxoCache[JSON.stringify(utxo)] = {
+          processing: false,
+          requiredCount: utxos.length,
+          path,
+          // TypeScript yells otherwise
+          address: { ...addressData, path: addressData.path }
+        }
+      }
+    })
+    .catch(() => {
+      args.processing = false
+      addressUtxoCache[address] = {
+        processing: args.processing,
+        path
+      }
+    })
+
+  return {
+    ..._BlockBookAPI.addressUtxosMessage.call(void 0, address, asBlockbookAddress),
+    deferred
+  }
+}
+
+
+
+
+
+
+
+const processProcessorUtxos = async (
+  args
+) => {
+  const {
+    utxos,
+    processor,
+    scriptPubkey,
+    log,
+    emitter,
+    pluginInfo: { currencyInfo }
+  } = args
+
+  const currentUtxos = await processor.fetchUtxos({ scriptPubkey })
+  const currentUtxoIds = []
+  let oldBalance = '0'
+  for (const utxo of currentUtxos) {
+    if (utxo == null)
+      throw new Error(
+        'Unexpected undefined utxo when processing unspent transactions'
+      )
+    oldBalance = _biggystring.add.call(void 0, utxo.value, oldBalance)
+    currentUtxoIds.push(utxo.id)
+  }
+  await processor.removeUtxos(currentUtxoIds)
+
+  let newBalance = '0'
+  for (const utxo of Array.from(utxos)) {
+    newBalance = _biggystring.add.call(void 0, utxo.value, newBalance)
+    await processor.saveUtxo(utxo)
+  }
+
+  const diff = _biggystring.sub.call(void 0, newBalance, oldBalance)
+  if (diff !== '0') {
+    log('balance changed:', { scriptPubkey, diff })
+    emitter.emit(
+      _makeEngineEmitter.EngineEvent.ADDRESS_BALANCE_CHANGED,
+      currencyInfo.currencyCode,
+      diff
+    )
+
+    // Update balances for address that have this scriptPubkey
+    const address = await processor.fetchAddress(scriptPubkey)
+
+    if (address == null) {
+      throw new Error('address not found when processing UTXO transactions')
+    }
+
+    await processor.saveAddress({
+      ...address,
+      balance: newBalance,
+      used: true
+    })
+  }
+
+  // Update the progress now that the UTXOs for an address have been processed
+  await args.updateProgressRatio()
+
+  await setLookAhead(args).catch(err => {
+    log.error(err)
+    throw err
+  })
+}
+
+
+
+
+
+
+
+
+
+
+const processRawUtxo = async (
+  args
+) => {
+  const {
+    utxo,
+    id,
+    address,
+    processor,
+    path,
+    taskCache,
+    requiredCount,
+    serverStates,
+    uri,
+    log
+  } = args
+  const { rawUtxoCache, processorUtxoCache } = taskCache
+  let scriptType
+  let script
+  let redeemScript
+
+  // Function to call once we are finished
+  const done = () =>
+    addToProcessorUtxoCache(
+      processorUtxoCache,
+      path,
+      address.scriptPubkey,
+      requiredCount,
+      {
+        id,
+        txid: utxo.txid,
+        vout: utxo.vout,
+        value: utxo.value,
+        scriptPubkey: address.scriptPubkey,
+        script,
+        redeemScript,
+        scriptType,
+        blockHeight: _nullishCoalesce(utxo.height, () => ( -1)),
+        spent: false
+      }
+    )
+
+  switch (_utils3.currencyFormatToPurposeType.call(void 0, path.format)) {
+    case _keymanager.BIP43PurposeTypeEnum.Airbitz:
+    case _keymanager.BIP43PurposeTypeEnum.Legacy:
+      scriptType = _keymanager.ScriptTypeEnum.p2pkh
+      if (address.redeemScript != null) {
+        scriptType = _keymanager.ScriptTypeEnum.p2sh
+        redeemScript = address.redeemScript
+      }
+
+      // Legacy UTXOs need the previous transaction hex as the script
+      // If we do not currently have it, add it to the queue to fetch it
+      {
+        const [tx] = await processor.fetchTransactions({ txId: utxo.txid })
+        if (tx == null) {
+          const queryTime = Date.now()
+          const deferred = new _Deferred2.default()
+          deferred.promise
+            .then((txResponse) => {
+              serverStates.serverScoreUp(uri, Date.now() - queryTime)
+              const processedTx = processTransactionResponse({
+                ...args,
+                txResponse
+              })
+              script = processedTx.hex
+              // Only after we have successfully fetched the tx, set our script and call done
+              done()
+            })
+            .catch(e => {
+              // If something went wrong, add the UTXO back to the queue
+              log('error in processed utxos cache, re-adding utxo to cache:', e)
+              rawUtxoCache[JSON.stringify(utxo)] = {
+                processing: false,
+                path,
+                address,
+                requiredCount
+              }
+            })
+          return {
+            ..._BlockBookAPI.transactionMessage.call(void 0, utxo.txid),
+            deferred
+          }
+        } else {
+          script = tx.hex
+        }
+      }
+
+      break
+    case _keymanager.BIP43PurposeTypeEnum.WrappedSegwit:
+      scriptType = _keymanager.ScriptTypeEnum.p2wpkhp2sh
+      script = address.scriptPubkey
+      if (address.redeemScript == null) {
+        throw new Error(
+          'Address redeem script not defined, but required for p2sh wrapped segwit utxo processing'
+        )
+      }
+      redeemScript = address.redeemScript
+
+      break
+    case _keymanager.BIP43PurposeTypeEnum.Segwit:
+      scriptType = _keymanager.ScriptTypeEnum.p2wpkh
+      script = address.scriptPubkey
+
+      break
+  }
+
+  // Since we have everything, call done
+  done()
+}
+
+const addToProcessorUtxoCache = (
+  processorUtxosCache,
+  path,
+  scriptPubkey,
+  requiredCount,
+  utxo
+) => {
+  const processorUtxos = _nullishCoalesce(processorUtxosCache[scriptPubkey], () => ( {
+    utxos: new Set(),
+    processing: false,
+    path,
+    full: false
+  }))
+  if (utxo != null) processorUtxos.utxos.add(utxo)
+  processorUtxosCache[scriptPubkey] = processorUtxos
+  processorUtxos.full = processorUtxos.utxos.size >= requiredCount
+}