@bopen-io/wallet-toolbox 1.7.18

package/src/monitor/tasks/TaskCheckForProofs.ts

@@ -0,0 +1,243 @@
+import { GetMerklePathResult } from '../../sdk'
+import { EntityProvenTx, EntityProvenTxReq } from '../../storage/schema/entities'
+import { TableProvenTxReq } from '../../storage/schema/tables'
+import { doubleSha256BE } from '../../utility/utilityHelpers'
+import { asString } from '../../utility/utilityHelpers.noBuffer'
+import { Monitor } from '../Monitor'
+import { WalletMonitorTask } from './WalletMonitorTask'
+
+/**
+ * `TaskCheckForProofs` is a WalletMonitor task that retreives merkle proofs for
+ * transactions.
+ *
+ * It is normally triggered by the Chaintracks new block header event.
+ *
+ * When a new block is found, cwi-external-services are used to obtain proofs for
+ * any transactions that are currently in the 'unmined' or 'unknown' state.
+ *
+ * If a proof is obtained and validated, a new ProvenTx record is created and
+ * the original ProvenTxReq status is advanced to 'notifying'.
+ */
+export class TaskCheckForProofs extends WalletMonitorTask {
+  static taskName = 'CheckForProofs'
+
+  /**
+   * An external service such as the chaintracks new block header
+   * listener can set this true to cause
+   */
+  static checkNow = false
+
+  constructor(
+    monitor: Monitor,
+    public triggerMsecs = 0
+  ) {
+    super(monitor, TaskCheckForProofs.taskName)
+  }
+
+  /**
+   * Normally triggered by checkNow getting set by new block header found event from chaintracks
+   */
+  trigger(nowMsecsSinceEpoch: number): { run: boolean } {
+    return {
+      run: TaskCheckForProofs.checkNow
+      // Check only when checkNow flag is set.
+      // || (this.triggerMsecs > 0 && nowMsecsSinceEpoch - this.lastRunMsecsSinceEpoch > this.triggerMsecs)
+    }
+  }
+
+  async runTask(): Promise<string> {
+    let log = ''
+    const countsAsAttempt = TaskCheckForProofs.checkNow
+    TaskCheckForProofs.checkNow = false
+
+    const maxAcceptableHeight = this.monitor.lastNewHeader?.height
+    if (maxAcceptableHeight === undefined) {
+      return log
+    }
+
+    const limit = 100
+    let offset = 0
+    for (;;) {
+      const reqs = await this.storage.findProvenTxReqs({
+        partial: {},
+        status: ['callback', 'unmined', 'sending', 'unknown', 'unconfirmed'],
+        paged: { limit, offset }
+      })
+      if (reqs.length === 0) break
+      log += `${reqs.length} reqs with status 'callback', 'unmined', 'sending', 'unknown', or 'unconfirmed'\n`
+      const r = await getProofs(this, reqs, 2, countsAsAttempt, false, maxAcceptableHeight)
+      log += `${r.log}\n`
+      //console.log(log);
+      if (reqs.length < limit) break
+      offset += limit
+    }
+    return log
+  }
+}
+
+/**
+ * Process an array of table.ProvenTxReq (typically with status 'unmined' or 'unknown')
+ *
+ * If req is invalid, set status 'invalid'
+ *
+ * Verify the requests are valid, lookup proofs or updated transaction status using the array of getProofServices,
+ *
+ * When proofs are found, create new ProvenTxApi records and transition the requests' status to 'unconfirmed' or 'notifying',
+ * depending on chaintracks succeeding on proof verification.
+ *
+ * Increments attempts if proofs where requested.
+ *
+ * @param reqs
+ * @returns reqs partitioned by status
+ */
+export async function getProofs(
+  task: WalletMonitorTask,
+  reqs: TableProvenTxReq[],
+  indent = 0,
+  countsAsAttempt = false,
+  ignoreStatus = false,
+  maxAcceptableHeight: number
+): Promise<{
+  proven: TableProvenTxReq[]
+  invalid: TableProvenTxReq[]
+  log: string
+}> {
+  const proven: TableProvenTxReq[] = []
+  const invalid: TableProvenTxReq[] = []
+
+  let log = ''
+  for (const reqApi of reqs) {
+    log += ' '.repeat(indent)
+    log += `reqId ${reqApi.provenTxReqId} txid ${reqApi.txid}: `
+
+    if (
+      !ignoreStatus &&
+      reqApi.status !== 'callback' &&
+      reqApi.status !== 'unmined' &&
+      reqApi.status !== 'unknown' &&
+      reqApi.status !== 'unconfirmed' &&
+      reqApi.status !== 'nosend' &&
+      reqApi.status !== 'sending'
+    ) {
+      log += `status of '${reqApi.status}' is not ready to be proven.\n`
+      continue
+    }
+
+    const req = new EntityProvenTxReq(reqApi)
+
+    if (Number.isInteger(req.provenTxId)) {
+      log += `Already linked to provenTxId ${req.provenTxId}.\n`
+      req.notified = false
+      req.status = 'completed'
+      await req.updateStorageDynamicProperties(task.storage)
+      proven.push(reqApi)
+      continue
+    }
+
+    log += '\n'
+
+    let reqIsValid = false
+    if (req.rawTx) {
+      const txid = asString(doubleSha256BE(req.rawTx))
+      if (txid === req.txid) reqIsValid = true
+    }
+
+    if (!reqIsValid) {
+      log += ` rawTx doesn't hash to txid. status => invalid.\n`
+      req.notified = false
+      req.status = 'invalid'
+      await req.updateStorageDynamicProperties(task.storage)
+      invalid.push(reqApi)
+      continue
+    }
+
+    const limit =
+      task.monitor.chain === 'main'
+        ? task.monitor.options.unprovenAttemptsLimitMain
+        : task.monitor.options.unprovenAttemptsLimitTest
+    if (!ignoreStatus && req.attempts > limit) {
+      log += ` too many failed attempts ${req.attempts}\n`
+      req.notified = false
+      req.status = 'invalid'
+      await req.updateStorageDynamicProperties(task.storage)
+      invalid.push(reqApi)
+      continue
+    }
+
+    const since = new Date()
+
+    let r: GetMerklePathResult
+    let ptx: EntityProvenTx | undefined
+
+    // External services will try multiple providers until one returns a proof,
+    // or they all fail.
+    // There may also be an array of proofs to consider when a transaction
+    // is recently mined and appears in orphan blocks in addition to active chain blocks.
+    // Since orphan blocks can end up on chain again, multiple proofs has value.
+    //
+    // On failure, there may be a mapi response, or an error.
+    //
+    // The proofs returned are considered sequentially, validating and chaintracks confirming.
+    //
+    // If a good proof is found, proceed to using it.
+    //
+    // When all received proofs fail, force a bump to the next service provider and try
+    // one more time.
+    //
+    r = await task.monitor.services.getMerklePath(req.txid)
+    if (r.header && r.header.height > maxAcceptableHeight) {
+      // Ignore proofs from bleeding edge of new blocks as these are the most often re-orged.
+      log += ` ignoring possible proof from very new block at height ${r.header.height} ${r.header.hash}\n`
+      continue
+    }
+    ptx = await EntityProvenTx.fromReq(req, r, countsAsAttempt && req.status !== 'nosend')
+
+    if (ptx) {
+      // We have a merklePath proof for the request (and a block header)
+      await req.updateStorageDynamicProperties(task.storage)
+      await req.refreshFromStorage(task.storage)
+      const { provenTxReqId, status, txid, attempts, history } = req.toApi()
+      const { index, height, blockHash, merklePath, merkleRoot } = ptx.toApi()
+      const r = await task.storage.runAsStorageProvider(async sp => {
+        return await sp.updateProvenTxReqWithNewProvenTx({
+          provenTxReqId,
+          status,
+          txid,
+          attempts,
+          history,
+          index,
+          height,
+          blockHash,
+          merklePath,
+          merkleRoot
+        })
+      })
+      req.status = r.status
+      req.apiHistory = r.history
+      req.provenTxId = r.provenTxId
+      req.notified = true
+
+      task.monitor.callOnProvenTransaction({
+        txid,
+        txIndex: index,
+        blockHeight: height,
+        blockHash,
+        merklePath,
+        merkleRoot
+      })
+    } else {
+      if (countsAsAttempt && req.status !== 'nosend') {
+        req.attempts++
+      }
+    }
+    await req.updateStorageDynamicProperties(task.storage)
+    await req.refreshFromStorage(task.storage)
+
+    log += req.historyPretty(since, indent + 2) + '\n'
+
+    if (req.status === 'completed') proven.push(req.api)
+    if (req.status === 'invalid') invalid.push(req.api)
+  }
+
+  return { proven, invalid, log }
+}

package/src/monitor/tasks/TaskCheckNoSends.ts

@@ -0,0 +1,73 @@
+import { Monitor } from '../Monitor'
+import { getProofs } from './TaskCheckForProofs'
+import { WalletMonitorTask } from './WalletMonitorTask'
+
+/**
+ * `TaskCheckNoSends` is a WalletMonitor task that retreives merkle proofs for
+ * 'nosend' transactions that MAY have been shared externally.
+ *
+ * Unlike intentionally processed transactions, 'nosend' transactions are fully valid
+ * transactions which have not been processed by the wallet.
+ *
+ * By default, this task runs once a day to check if any 'nosend' transaction has
+ * managed to get mined by some external process.
+ *
+ * If a proof is obtained and validated, a new ProvenTx record is created and
+ * the original ProvenTxReq status is advanced to 'notifying'.
+ */
+export class TaskCheckNoSends extends WalletMonitorTask {
+  static taskName = 'CheckNoSends'
+
+  /**
+   * An external service such as the chaintracks new block header
+   * listener can set this true to cause
+   */
+  static checkNow = false
+
+  constructor(
+    monitor: Monitor,
+    public triggerMsecs = Monitor.oneDay * 1
+  ) {
+    super(monitor, TaskCheckNoSends.taskName)
+  }
+
+  /**
+   * Normally triggered by checkNow getting set by new block header found event from chaintracks
+   */
+  trigger(nowMsecsSinceEpoch: number): { run: boolean } {
+    return {
+      run:
+        TaskCheckNoSends.checkNow ||
+        (this.triggerMsecs > 0 && nowMsecsSinceEpoch - this.lastRunMsecsSinceEpoch > this.triggerMsecs)
+    }
+  }
+
+  async runTask(): Promise<string> {
+    let log = ''
+    const countsAsAttempt = TaskCheckNoSends.checkNow
+    TaskCheckNoSends.checkNow = false
+
+    const maxAcceptableHeight = this.monitor.lastNewHeader?.height
+    if (maxAcceptableHeight === undefined) {
+      return log
+    }
+
+    const limit = 100
+    let offset = 0
+    for (;;) {
+      const reqs = await this.storage.findProvenTxReqs({
+        partial: {},
+        status: ['nosend'],
+        paged: { limit, offset }
+      })
+      if (reqs.length === 0) break
+      log += `${reqs.length} reqs with status 'nosend'\n`
+      const r = await getProofs(this, reqs, 2, countsAsAttempt, false, maxAcceptableHeight)
+      log += `${r.log}\n`
+      //console.log(log);
+      if (reqs.length < limit) break
+      offset += limit
+    }
+    return log
+  }
+}

package/src/monitor/tasks/TaskClock.ts

@@ -0,0 +1,33 @@
+import { Monitor } from '../Monitor'
+import { WalletMonitorTask } from './WalletMonitorTask'
+
+export class TaskClock extends WalletMonitorTask {
+  static taskName = 'Clock'
+  nextMinute: number
+
+  constructor(
+    monitor: Monitor,
+    public triggerMsecs = 1 * Monitor.oneSecond
+  ) {
+    super(monitor, TaskClock.taskName)
+    this.nextMinute = this.getNextMinute()
+  }
+
+  trigger(nowMsecsSinceEpoch: number): { run: boolean } {
+    const s = this.storage
+
+    const run = Date.now() > this.nextMinute
+
+    return { run }
+  }
+
+  async runTask(): Promise<string> {
+    const log = `${new Date(this.nextMinute).toISOString()}`
+    this.nextMinute = this.getNextMinute()
+    return log
+  }
+
+  getNextMinute(): number {
+    return Math.ceil(Date.now() / Monitor.oneMinute) * Monitor.oneMinute
+  }
+}

package/src/monitor/tasks/TaskFailAbandoned.ts

@@ -0,0 +1,54 @@
+import { Monitor } from '../Monitor'
+import { WalletMonitorTask } from './WalletMonitorTask'
+
+/**
+ * Handles transactions which do not have terminal status and have not been
+ * updated for an extended time period.
+ *
+ * Calls `updateTransactionStatus` to set `status` to `failed`.
+ * This returns inputs to spendable status and verifies that any
+ * outputs are not spendable.
+ */
+
+export class TaskFailAbandoned extends WalletMonitorTask {
+  static taskName = 'FailAbandoned'
+
+  constructor(
+    monitor: Monitor,
+    public triggerMsecs = 1000 * 60 * 5
+  ) {
+    super(monitor, TaskFailAbandoned.taskName)
+  }
+
+  trigger(nowMsecsSinceEpoch: number): { run: boolean } {
+    return {
+      run: nowMsecsSinceEpoch > this.lastRunMsecsSinceEpoch + this.triggerMsecs
+    }
+  }
+
+  async runTask(): Promise<string> {
+    let log = ''
+    const limit = 100
+    let offset = 0
+    for (;;) {
+      const now = new Date()
+      const abandoned = new Date(now.getTime() - this.monitor.options.abandonedMsecs)
+      const done = await this.storage.runAsStorageProvider(async sp => {
+        const txsAll = await sp.findTransactions({
+          partial: {},
+          status: ['unprocessed', 'unsigned'],
+          paged: { limit, offset }
+        })
+        const txs = txsAll.filter(t => t.updated_at && t.updated_at < abandoned)
+        for (const tx of txs) {
+          await sp.updateTransactionStatus('failed', tx.transactionId)
+          log += `updated tx ${tx.transactionId} status to 'failed'\n`
+        }
+        return txs.length < limit
+      })
+      if (done) break
+      offset += limit
+    }
+    return log
+  }
+}

package/src/monitor/tasks/TaskMonitorCallHistory.ts

@@ -0,0 +1,26 @@
+import { TableMonitorEvent } from '../../storage/index.client'
+import { Monitor } from '../Monitor'
+import { WalletMonitorTask } from './WalletMonitorTask'
+
+export class TaskMonitorCallHistory extends WalletMonitorTask {
+  static taskName = 'MonitorCallHistory'
+
+  constructor(
+    monitor: Monitor,
+    public triggerMsecs = Monitor.oneMinute * 12
+  ) {
+    super(monitor, TaskMonitorCallHistory.taskName)
+  }
+
+  trigger(nowMsecsSinceEpoch: number): { run: boolean } {
+    return {
+      run: nowMsecsSinceEpoch > this.lastRunMsecsSinceEpoch + this.triggerMsecs
+    }
+  }
+
+  async runTask(): Promise<string> {
+    const r = await this.monitor.services.getServicesCallHistory(true)
+    const log = JSON.stringify(r)
+    return log
+  }
+}

package/src/monitor/tasks/TaskNewHeader.ts

@@ -0,0 +1,93 @@
+import { BlockHeader } from '../../sdk/WalletServices.interfaces'
+import { Monitor } from '../Monitor'
+import { WalletMonitorTask } from './WalletMonitorTask'
+
+/**
+ * This task polls for new block headers performing two essential functions:
+ * 1. The arrival of a new block is the right time to check for proofs for recently broadcast transactions.
+ * 2. The height of the block is used to limit which proofs are accepted with the aim of avoiding re-orged proofs.
+ *
+ * The most common new block orphan is one which is almost immediately orphaned.
+ * Waiting a minute before pursuing proof requests avoids almost all the re-org work that could be done.
+ * Thus this task queues new headers for one cycle.
+ * If a new header arrives during that cycle, it replaces the queued header and delays again.
+ * Only when there is an elapsed cycle without a new header does proof solicitation get triggered,
+ * with that header height as the limit for which proofs are accepted.
+ */
+export class TaskNewHeader extends WalletMonitorTask {
+  static taskName = 'NewHeader'
+  /**
+   * This is always the most recent chain tip header returned from the chaintracker.
+   */
+  header?: BlockHeader
+  /**
+   * Tracks the value of `header` except that it is set to undefined
+   * when a cycle without a new header occurs and `processNewBlockHeader` is called.
+   */
+  queuedHeader?: BlockHeader
+  queuedHeaderWhen?: Date
+
+  constructor(
+    monitor: Monitor,
+    public triggerMsecs = 1 * Monitor.oneMinute
+  ) {
+    super(monitor, TaskNewHeader.taskName)
+  }
+
+  async getHeader(): Promise<BlockHeader> {
+    return await this.monitor.chaintracks.findChainTipHeader()
+  }
+
+  /**
+   * TODO: This is a temporary incomplete solution for which a full chaintracker
+   * with new header and reorg event notification is required.
+   *
+   * New header events drive retrieving merklePaths for newly mined transactions.
+   * This implementation performs this function.
+   *
+   * Reorg events are needed to know when previously retrieved mekrlePaths need to be
+   * updated in the proven_txs table (and ideally notifications delivered to users).
+   * Note that in-general, a reorg only shifts where in the block a transaction is mined,
+   * and sometimes which block. In the case of coinbase transactions, a transaction may
+   * also fail after a reorg.
+   */
+  override async asyncSetup(): Promise<void> {}
+
+  trigger(nowMsecsSinceEpoch: number): { run: boolean } {
+    const run = true
+    return { run }
+  }
+
+  async runTask(): Promise<string> {
+    let log = ''
+    const oldHeader = this.header
+    this.header = await this.getHeader()
+    let isNew = true
+    if (!oldHeader) {
+      log = `first header: ${this.header.height} ${this.header.hash}`
+    } else if (oldHeader.height > this.header.height) {
+      log = `old header: ${this.header.height} vs ${oldHeader.height}`
+      this.header = oldHeader // Revert to old header with the higher height
+      isNew = false
+    } else if (oldHeader.height < this.header.height) {
+      const skip = this.header.height - oldHeader.height - 1
+      const skipped = skip > 0 ? ` SKIPPED ${skip}` : ''
+      log = `new header: ${this.header.height} ${this.header.hash}${skipped}`
+    } else if (oldHeader.height === this.header.height && oldHeader.hash != this.header.hash) {
+      log = `reorg header: ${this.header.height} ${this.header.hash}`
+    } else {
+      isNew = false
+    }
+    if (isNew) {
+      this.queuedHeader = this.header
+      this.queuedHeaderWhen = new Date()
+    } else if (this.queuedHeader) {
+      // Only process new block header if it has remained the chain tip for a full cycle
+      const delay = (new Date().getTime() - this.queuedHeaderWhen!.getTime()) / 1000 // seconds
+      log = `process header: ${this.header.height} ${this.header.hash} delayed ${delay.toFixed(1)} secs`
+      this.monitor.processNewBlockHeader(this.queuedHeader)
+      this.queuedHeader = undefined
+    }
+    return log
+  }
+}

package/src/monitor/tasks/TaskPurge.ts

@@ -0,0 +1,68 @@
+import { PurgeParams } from '../../sdk/WalletStorage.interfaces'
+import { Monitor } from '../Monitor'
+import { WalletMonitorTask } from './WalletMonitorTask'
+
+/**
+ * The database stores a variety of data that may be considered transient.
+ *
+ * At one extreme, the data that must be preserved:
+ *   - unspent outputs (UTXOs)
+ *   - in-use metadata (labels, baskets, tags...)
+ *
+ * At the other extreme, everything can be preserved to fully log all transaction creation and processing actions.
+ *
+ * The following purge actions are available to support sustained operation:
+ *   - Failed transactions, delete all associated data including:
+ *       + Delete tag and label mapping records
+ *       + Delete output records
+ *       + Delete transaction records
+ *       + Delete mapi_responses records
+ *       + Delete proven_tx_reqs records
+ *       + Delete commissions records
+ *       + Update output records marked spentBy failed transactions
+ *   - Completed transactions, delete transient data including:
+ *       + transactions table set truncatedExternalInputs = null
+ *       + transactions table set beef = null
+ *       + transactions table set rawTx = null
+ *       + Delete mapi_responses records
+ *       + proven_tx_reqs table delete records
+ */
+export interface TaskPurgeParams extends PurgeParams {}
+
+export class TaskPurge extends WalletMonitorTask {
+  static taskName = 'Purge'
+
+  /**
+   * Set to true to trigger running this task
+   */
+  static checkNow = false
+
+  constructor(
+    monitor: Monitor,
+    public params: TaskPurgeParams,
+    public triggerMsecs = 0
+  ) {
+    super(monitor, TaskPurge.taskName)
+  }
+
+  trigger(nowMsecsSinceEpoch: number): { run: boolean } {
+    return {
+      run:
+        TaskPurge.checkNow ||
+        (this.triggerMsecs > 0 && nowMsecsSinceEpoch - this.lastRunMsecsSinceEpoch > this.triggerMsecs)
+    }
+  }
+
+  async runTask(): Promise<string> {
+    let log = ''
+    TaskPurge.checkNow = false
+
+    const r = await this.storage.runAsStorageProvider(async sp => {
+      return await sp.purgeData(this.params)
+    })
+
+    if (r.count > 0) log = `${r.count} records updated or deleted.\n${r.log}`
+
+    return log
+  }
+}

package/src/monitor/tasks/TaskReorg.ts

@@ -0,0 +1,89 @@
+import { TableProvenTx } from '../../index.client'
+import { BlockHeader } from '../../services/chaintracker/chaintracks/Api/BlockHeaderApi'
+import { DeactivedHeader, Monitor } from '../Monitor'
+import { WalletMonitorTask } from './WalletMonitorTask'
+
+/**
+ * Check the `monitor.deactivatedHeaders` for any headers that have been deactivated.
+ *
+ * When headers are found, review matching ProvenTx records and update proof data as appropriate.
+ *
+ * New deactivated headers are pushed onto the `deactivatedHeaders` array.
+ * They must be shifted out as they are processed.
+ *
+ * The current implementation ages deactivation notifications by 10 minutes with each retry.
+ * If a successful proof update confirms original proof data after 3 retries, the original is retained.
+ *
+ * In normal operation there should rarely be any work for this task to perform.
+ * The most common result is that there are no matching proven_txs records because
+ * generating new proven_txs records intentionally lags new block generation to
+ * minimize this disruption.
+ *
+ * It is very disruptive to update a proven_txs record because:
+ * - Sync'ed storage is impacted.
+ * - Generated beefs are impacted.
+ * - Updated proof data may be unavailable at the time a reorg is first reported.
+ *
+ * Proper reorg handling also requires repairing invalid beefs for new transactions when
+ * createAction fails to verify a generated beef against the chaintracker.
+ */
+export class TaskReorg extends WalletMonitorTask {
+  static taskName = 'Reorg'
+
+  process: DeactivedHeader[] = []
+
+  constructor(
+    monitor: Monitor,
+    public agedMsecs = Monitor.oneMinute * 10,
+    public maxRetries = 3
+  ) {
+    super(monitor, TaskReorg.taskName)
+  }
+
+  /**
+   * Shift aged deactivated headers onto `process` array.
+   * @param nowMsecsSinceEpoch current time in milliseconds since epoch.
+   * @returns `run` true iff there are aged deactivated headers to process.
+   */
+  trigger(nowMsecsSinceEpoch: number): { run: boolean } {
+    const cutoff = nowMsecsSinceEpoch - this.agedMsecs
+    const q = this.monitor.deactivatedHeaders
+    while (q.length > 0 && cutoff > q[0].whenMsecs) {
+      // Prepare to process deactivated headers that have aged sufficiently (agedMsecs)
+      const header = q.shift()!
+      this.process.push(header)
+    }
+    return {
+      run: this.process.length > 0
+    }
+  }
+
+  async runTask(): Promise<string> {
+    let log = ''
+
+    for (;;) {
+      // Loop over deactivated headers to process
+      const header = this.process.shift()
+      if (!header) break
+
+      const r = await this.storage.reproveHeader(header.header.hash)
+
+      log += r.log
+
+      if (r.unavailable.length > 0 || r.unchanged.length > 0) {
+        if (header.tries + 1 >= this.maxRetries) {
+          log += `      maximum retries ${this.maxRetries} exceeded\n`
+        } else {
+          log += `    retrying...\n`
+          this.monitor.deactivatedHeaders.push({
+            header: header.header,
+            whenMsecs: Date.now(),
+            tries: header.tries + 1
+          })
+        }
+      }
+    }
+
+    return log
+  }
+}