@bsv/wallet-toolbox 1.1.0

package/src/monitor/tasks/TaskCheckForProofs.ts

@@ -0,0 +1,235 @@
+import {
+  asString,
+  doubleSha256BE,
+  entity,
+  sdk,
+  table
+} from '../../index.client'
+import { Monitor } from '../Monitor'
+import { WalletMonitorTask } from './WalletMonitorTask'
+
+/**
+ * `TaskCheckForProofs` is a WalletMonitor task that retreives merkle proofs for
+ * transactions.
+ *
+ * It is normally triggered by the Chaintracks new block header event.
+ *
+ * When a new block is found, cwi-external-services are used to obtain proofs for
+ * any transactions that are currently in the 'unmined' or 'unknown' state.
+ *
+ * If a proof is obtained and validated, a new ProvenTx record is created and
+ * the original ProvenTxReq status is advanced to 'notifying'.
+ */
+
+export class TaskCheckForProofs extends WalletMonitorTask {
+  static taskName = 'CheckForProofs'
+
+  /**
+   * An external service such as the chaintracks new block header
+   * listener can set this true to cause
+   */
+  static checkNow = false
+
+  constructor(
+    monitor: Monitor,
+    public triggerMsecs = 0
+  ) {
+    super(monitor, TaskCheckForProofs.taskName)
+  }
+
+  /**
+   * Normally triggered by checkNow getting set by new block header found event from chaintracks
+   */
+  trigger(nowMsecsSinceEpoch: number): { run: boolean } {
+    return {
+      run:
+        TaskCheckForProofs.checkNow ||
+        (this.triggerMsecs > 0 &&
+          nowMsecsSinceEpoch - this.lastRunMsecsSinceEpoch > this.triggerMsecs)
+    }
+  }
+
+  async runTask(): Promise<string> {
+    let log = ''
+    const countsAsAttempt = TaskCheckForProofs.checkNow
+    TaskCheckForProofs.checkNow = false
+
+    const limit = 100
+    let offset = 0
+    for (;;) {
+      const reqs = await this.storage.findProvenTxReqs({
+        partial: {},
+        status: [
+          'callback',
+          'unmined',
+          'nosend',
+          'sending',
+          'unknown',
+          'unconfirmed'
+        ],
+        paged: { limit, offset }
+      })
+      if (reqs.length === 0) break
+      log += `${reqs.length} reqs with status 'callback', 'unmined', 'nosend', 'sending', 'unknown', or 'unconfirmed'\n`
+      const r = await this.getProofs(reqs, 2, countsAsAttempt)
+      log += `${r.log}\n`
+      //console.log(log);
+      if (reqs.length < limit) break
+      offset += limit
+    }
+    return log
+  }
+
+  /**
+   * Process an array of table.ProvenTxReq (typically with status 'unmined' or 'unknown')
+   *
+   * If req is invalid, set status 'invalid'
+   *
+   * Verify the requests are valid, lookup proofs or updated transaction status using the array of getProofServices,
+   *
+   * When proofs are found, create new ProvenTxApi records and transition the requests' status to 'unconfirmed' or 'notifying',
+   * depending on chaintracks succeeding on proof verification.
+   *
+   * Increments attempts if proofs where requested.
+   *
+   * @param reqs
+   * @returns reqs partitioned by status
+   */
+  async getProofs(
+    reqs: table.ProvenTxReq[],
+    indent = 0,
+    countsAsAttempt = false,
+    ignoreStatus = false
+  ): Promise<{
+    proven: table.ProvenTxReq[]
+    invalid: table.ProvenTxReq[]
+    log: string
+  }> {
+    const proven: table.ProvenTxReq[] = []
+    const invalid: table.ProvenTxReq[] = []
+
+    let log = ''
+    for (const reqApi of reqs) {
+      log += ' '.repeat(indent)
+      log += `reqId ${reqApi.provenTxReqId} txid ${reqApi.txid}: `
+
+      if (
+        !ignoreStatus &&
+        reqApi.status !== 'callback' &&
+        reqApi.status !== 'unmined' &&
+        reqApi.status !== 'unknown' &&
+        reqApi.status !== 'unconfirmed' &&
+        reqApi.status !== 'nosend' &&
+        reqApi.status !== 'sending'
+      ) {
+        log += `status of '${reqApi.status}' is not ready to be proven.\n`
+        continue
+      }
+
+      const req = new entity.ProvenTxReq(reqApi)
+
+      if (Number.isInteger(req.provenTxId)) {
+        log += `Already linked to provenTxId ${req.provenTxId}.\n`
+        req.notified = false
+        req.status = 'completed'
+        await req.updateStorageDynamicProperties(this.storage)
+        proven.push(reqApi)
+        continue
+      }
+
+      log += '\n'
+
+      let reqIsValid = false
+      if (req.rawTx) {
+        const txid = asString(doubleSha256BE(req.rawTx))
+        if (txid === req.txid) reqIsValid = true
+      }
+
+      if (!reqIsValid) {
+        log += ` rawTx doesn't hash to txid. status => invalid.\n`
+        req.notified = false
+        req.status = 'invalid'
+        await req.updateStorageDynamicProperties(this.storage)
+        invalid.push(reqApi)
+        continue
+      }
+
+      const limit =
+        this.monitor.chain === 'main'
+          ? this.monitor.options.unprovenAttemptsLimitMain
+          : this.monitor.options.unprovenAttemptsLimitTest
+      if (!ignoreStatus && req.attempts > limit) {
+        log += ` too many failed attempts ${req.attempts}\n`
+        req.notified = false
+        req.status = 'invalid'
+        await req.updateStorageDynamicProperties(this.storage)
+        invalid.push(reqApi)
+        continue
+      }
+
+      const since = new Date()
+
+      let r: sdk.GetMerklePathResult
+      let ptx: entity.ProvenTx | undefined
+
+      // External services will try multiple providers until one returns a proof,
+      // or they all fail.
+      // There may also be an array of proofs to consider when a transaction
+      // is recently mined and appears in orphan blocks in addition to active chain blocks.
+      // Since orphan blocks can end up on chain again, multiple proofs has value.
+      //
+      // On failure, there may be a mapi response, or an error.
+      //
+      // The proofs returned are considered sequentially, validating and chaintracks confirming.
+      //
+      // If a good proof is found, proceed to using it.
+      //
+      // When all received proofs fail, force a bump to the next service provider and try
+      // one more time.
+      //
+      r = await this.monitor.services.getMerklePath(req.txid)
+      ptx = await entity.ProvenTx.fromReq(
+        req,
+        r,
+        countsAsAttempt && req.status !== 'nosend'
+      )
+
+      if (ptx) {
+        // We have a merklePath proof for the request (and a block header)
+        const { provenTxReqId, status, txid, attempts, history } = req.toApi()
+        const { index, height, blockHash, merklePath, merkleRoot } = ptx.toApi()
+        const r = await this.storage.runAsStorageProvider(async sp => {
+          return await sp.updateProvenTxReqWithNewProvenTx({
+            provenTxReqId,
+            status,
+            txid,
+            attempts,
+            history,
+            index,
+            height,
+            blockHash,
+            merklePath,
+            merkleRoot
+          })
+        })
+        req.status = r.status
+        req.apiHistory = r.history
+        req.provenTxId = r.provenTxId
+        req.notified = true
+      } else {
+        if (countsAsAttempt && req.status !== 'nosend') {
+          req.attempts++
+        }
+        await req.updateStorageDynamicProperties(this.storage)
+        await req.refreshFromStorage(this.storage)
+      }
+
+      log += req.historyPretty(since, indent + 2) + '\n'
+
+      if (req.status === 'completed') proven.push(req.api)
+      if (req.status === 'invalid') invalid.push(req.api)
+    }
+
+    return { proven, invalid, log }
+  }
+}

package/src/monitor/tasks/TaskClock.ts

@@ -0,0 +1,35 @@
+import { Monitor } from '../Monitor'
+import { WalletMonitorTask } from './WalletMonitorTask'
+
+export class TaskClock extends WalletMonitorTask {
+  static taskName = 'Clock'
+  nextMinute: number
+
+  constructor(
+    monitor: Monitor,
+    public triggerMsecs = 1 * monitor.oneSecond
+  ) {
+    super(monitor, TaskClock.taskName)
+    this.nextMinute = this.getNextMinute()
+  }
+
+  trigger(nowMsecsSinceEpoch: number): { run: boolean } {
+    const s = this.storage
+
+    const run = Date.now() > this.nextMinute
+
+    return { run }
+  }
+
+  async runTask(): Promise<string> {
+    const log = `${new Date(this.nextMinute).toISOString()}`
+    this.nextMinute = this.getNextMinute()
+    return log
+  }
+
+  getNextMinute(): number {
+    return (
+      Math.ceil(Date.now() / this.monitor.oneMinute) * this.monitor.oneMinute
+    )
+  }
+}

package/src/monitor/tasks/TaskFailAbandoned.ts

@@ -0,0 +1,56 @@
+import { Monitor } from '../Monitor'
+import { WalletMonitorTask } from './WalletMonitorTask'
+
+/**
+ * Handles transactions which do not have terminal status and have not been
+ * updated for an extended time period.
+ *
+ * Calls `updateTransactionStatus` to set `status` to `failed`.
+ * This returns inputs to spendable status and verifies that any
+ * outputs are not spendable.
+ */
+
+export class TaskFailAbandoned extends WalletMonitorTask {
+  static taskName = 'FailAbandoned'
+
+  constructor(
+    monitor: Monitor,
+    public triggerMsecs = 1000 * 60 * 5
+  ) {
+    super(monitor, TaskFailAbandoned.taskName)
+  }
+
+  trigger(nowMsecsSinceEpoch: number): { run: boolean } {
+    return {
+      run: nowMsecsSinceEpoch > this.lastRunMsecsSinceEpoch + this.triggerMsecs
+    }
+  }
+
+  async runTask(): Promise<string> {
+    let log = ''
+    const limit = 100
+    let offset = 0
+    for (;;) {
+      const now = new Date()
+      const abandoned = new Date(
+        now.getTime() - this.monitor.options.abandonedMsecs
+      )
+      const done = await this.storage.runAsStorageProvider(async sp => {
+        const txsAll = await sp.findTransactions({
+          partial: {},
+          status: ['unprocessed', 'unsigned'],
+          paged: { limit, offset }
+        })
+        const txs = txsAll.filter(t => t.updated_at && t.updated_at < abandoned)
+        for (const tx of txs) {
+          await sp.updateTransactionStatus('failed', tx.transactionId)
+          log += `updated tx ${tx.transactionId} status to 'failed'\n`
+        }
+        return txs.length < limit
+      })
+      if (done) break
+      offset += limit
+    }
+    return log
+  }
+}

package/src/monitor/tasks/TaskNewHeader.ts

@@ -0,0 +1,48 @@
+import { sdk } from '../../index.client'
+import { Monitor } from '../Monitor'
+import { TaskCheckForProofs } from './TaskCheckForProofs'
+import { WalletMonitorTask } from './WalletMonitorTask'
+
+export class TaskNewHeader extends WalletMonitorTask {
+  static taskName = 'NewHeader'
+  header?: sdk.BlockHeader
+
+  constructor(
+    monitor: Monitor,
+    public triggerMsecs = 1 * monitor.oneMinute
+  ) {
+    super(monitor, TaskNewHeader.taskName)
+  }
+
+  async getHeader(): Promise<sdk.BlockHeader> {
+    return await this.monitor.chaintracks.findChainTipHeader()
+  }
+
+  trigger(nowMsecsSinceEpoch: number): { run: boolean } {
+    const run = true
+    return { run }
+  }
+
+  async runTask(): Promise<string> {
+    let log = ''
+    const oldHeader = this.header
+    this.header = await this.getHeader()
+    let isNew = true
+    if (!oldHeader) {
+      log = `first header: ${this.header.height} ${this.header.hash}`
+    } else if (oldHeader.height < this.header.height) {
+      const skip = this.header.height - oldHeader.height - 1
+      const skipped = skip > 0 ? ` SKIPPED ${skip}` : ''
+      log = `new header: ${this.header.height} ${this.header.hash}${skipped}`
+    } else if (
+      oldHeader.height === this.header.height &&
+      oldHeader.hash != this.header.hash
+    ) {
+      log = `reorg header: ${this.header.height} ${this.header.hash}`
+    } else {
+      isNew = false
+    }
+    if (isNew) TaskCheckForProofs.checkNow = true
+    return log
+  }
+}

package/src/monitor/tasks/TaskPurge.ts

@@ -0,0 +1,69 @@
+import { sdk } from '../../index.client'
+import { Monitor } from '../Monitor'
+import { WalletMonitorTask } from './WalletMonitorTask'
+
+/**
+ * The database stores a variety of data that may be considered transient.
+ *
+ * At one extreme, the data that must be preserved:
+ *   - unspent outputs (UTXOs)
+ *   - in-use metadata (labels, baskets, tags...)
+ *
+ * At the other extreme, everything can be preserved to fully log all transaction creation and processing actions.
+ *
+ * The following purge actions are available to support sustained operation:
+ *   - Failed transactions, delete all associated data including:
+ *       + Delete tag and label mapping records
+ *       + Delete output records
+ *       + Delete transaction records
+ *       + Delete mapi_responses records
+ *       + Delete proven_tx_reqs records
+ *       + Delete commissions records
+ *       + Update output records marked spentBy failed transactions
+ *   - Completed transactions, delete transient data including:
+ *       + transactions table set truncatedExternalInputs = null
+ *       + transactions table set beef = null
+ *       + transactions table set rawTx = null
+ *       + Delete mapi_responses records
+ *       + proven_tx_reqs table delete records
+ */
+export interface TaskPurgeParams extends sdk.PurgeParams {}
+
+export class TaskPurge extends WalletMonitorTask {
+  static taskName = 'Purge'
+
+  /**
+   * Set to true to trigger running this task
+   */
+  static checkNow = false
+
+  constructor(
+    monitor: Monitor,
+    public params: TaskPurgeParams,
+    public triggerMsecs = 0
+  ) {
+    super(monitor, TaskPurge.taskName)
+  }
+
+  trigger(nowMsecsSinceEpoch: number): { run: boolean } {
+    return {
+      run:
+        TaskPurge.checkNow ||
+        (this.triggerMsecs > 0 &&
+          nowMsecsSinceEpoch - this.lastRunMsecsSinceEpoch > this.triggerMsecs)
+    }
+  }
+
+  async runTask(): Promise<string> {
+    let log = ''
+    TaskPurge.checkNow = false
+
+    const r = await this.storage.runAsStorageProvider(async sp => {
+      return await sp.purgeData(this.params)
+    })
+
+    if (r.count > 0) log = `${r.count} records updated or deleted.\n${r.log}`
+
+    return log
+  }
+}

package/src/monitor/tasks/TaskReviewStatus.ts

@@ -0,0 +1,51 @@
+import { sdk } from '../../index.client'
+import { Monitor } from '../Monitor'
+import { WalletMonitorTask } from './WalletMonitorTask'
+
+/**
+ * Notify Transaction records of changes in ProvenTxReq records they may have missed.
+ *
+ * The `notified` property flags reqs that do not need to be checked.
+ *
+ * Looks for aged Transactions with provenTxId with status != 'completed', sets status to 'completed'.
+ *
+ * Looks for reqs with 'invalid' status that
+ */
+export class TaskReviewStatus extends WalletMonitorTask {
+  static taskName = 'ReviewStatus'
+
+  /**
+   * Set to true to trigger running this task
+   */
+  static checkNow = false
+
+  constructor(
+    monitor: Monitor,
+    public triggerMsecs = 1000 * 60 * 15,
+    public agedMsecs = 1000 * 60 * 5
+  ) {
+    super(monitor, TaskReviewStatus.taskName)
+  }
+
+  trigger(nowMsecsSinceEpoch: number): { run: boolean } {
+    return {
+      run:
+        this.triggerMsecs > 0 &&
+        nowMsecsSinceEpoch - this.lastRunMsecsSinceEpoch > this.triggerMsecs
+    }
+  }
+
+  async runTask(): Promise<string> {
+    let log = ''
+
+    const agedLimit = new Date(Date.now() - this.agedMsecs)
+    const r = await this.storage.runAsStorageProvider(async sp => {
+      const r = await sp.reviewStatus({ agedLimit })
+      return r
+    })
+
+    if (r.log.length > 0) log += `${r.log}`
+
+    return log
+  }
+}

package/src/monitor/tasks/TaskSendWaiting.ts

@@ -0,0 +1,107 @@
+import { entity, table } from '../../storage/index.client'
+import { verifyTruthy } from '../../utility/index.client'
+import { Monitor } from '../Monitor'
+import { WalletMonitorTask } from './WalletMonitorTask'
+import { attemptToPostReqsToNetwork } from '../../storage/methods/attemptToPostReqsToNetwork'
+
+export class TaskSendWaiting extends WalletMonitorTask {
+  static taskName = 'SendWaiting'
+
+  constructor(
+    monitor: Monitor,
+    public triggerMsecs = 1000 * 60 * 5,
+    public agedMsecs = 0
+  ) {
+    super(monitor, TaskSendWaiting.taskName)
+  }
+
+  trigger(nowMsecsSinceEpoch: number): { run: boolean } {
+    return {
+      run: nowMsecsSinceEpoch > this.lastRunMsecsSinceEpoch + this.triggerMsecs
+    }
+  }
+
+  async runTask(): Promise<string> {
+    let log = ''
+    const limit = 100
+    let offset = 0
+    const agedLimit = new Date(Date.now() - this.agedMsecs)
+    for (;;) {
+      const reqs = await this.storage.findProvenTxReqs({
+        partial: {},
+        status: ['unsent'],
+        paged: { limit, offset }
+      })
+      if (reqs.length === 0) break
+      log += `${reqs.length} reqs with status 'unsent'\n`
+      const agedReqs = reqs.filter(
+        req => verifyTruthy(req.updated_at) < agedLimit
+      )
+      log += `  Of those reqs, ${agedReqs.length} where last updated before ${agedLimit.toISOString()}.\n`
+      log += await this.processUnsent(agedReqs, 2)
+      if (reqs.length < limit) break
+      offset += limit
+    }
+    return log
+  }
+
+  /**
+   * Process an array of 'unsent' status table.ProvenTxReq
+   *
+   * Send rawTx to transaction processor(s), requesting proof callbacks when possible.
+   *
+   * Set status 'invalid' if req is invalid.
+   *
+   * Set status to 'callback' on successful network submission with callback service.
+   *
+   * Set status to 'unmined' on successful network submission without callback service.
+   *
+   * Add mapi responses to database table if received.
+   *
+   * Increments attempts if sending was attempted.
+   *
+   * @param reqApis
+   */
+  async processUnsent(
+    reqApis: table.ProvenTxReq[],
+    indent = 0
+  ): Promise<string> {
+    let log = ''
+    for (let i = 0; i < reqApis.length; i++) {
+      const reqApi = reqApis[i]
+      log += ' '.repeat(indent)
+      log += `${i} reqId=${reqApi.provenTxReqId} attempts=${reqApi.attempts} txid=${reqApi.txid}: \n`
+      if (reqApi.status !== 'unsent') {
+        log += `  status now ${reqApi.status}\n`
+        continue
+      }
+      const req = new entity.ProvenTxReq(reqApi)
+      const reqs: entity.ProvenTxReq[] = []
+      if (req.batch) {
+        // Make sure wew process entire batch together for efficient beef generation
+        const batchReqApis = await this.storage.findProvenTxReqs({
+          partial: { batch: req.batch, status: 'unsent' }
+        })
+        for (const bra of batchReqApis) {
+          // Remove any matching batchReqApis from reqApis
+          const index = reqApis.findIndex(
+            ra => ra.provenTxReqId === bra.provenTxReqId
+          )
+          if (index > -1) reqApis.slice(index, index + 1)
+          // And add to reqs being processed now:
+          reqs.push(new entity.ProvenTxReq(bra))
+        }
+      } else {
+        // Just a single non-batched req...
+        reqs.push(req)
+      }
+
+      const r = await this.storage.runAsStorageProvider(async sp => {
+        return attemptToPostReqsToNetwork(sp, reqs)
+      })
+
+      log += r.log
+    }
+    return log
+  }
+}

package/src/monitor/tasks/TaskSyncWhenIdle.ts

@@ -0,0 +1,26 @@
+import { Monitor } from '../Monitor'
+import { WalletMonitorTask } from './WalletMonitorTask'
+
+export class TaskSyncWhenIdle extends WalletMonitorTask {
+  static taskName = 'SyncWhenIdle'
+
+  constructor(
+    monitor: Monitor,
+    public triggerMsecs = 1000 * 60 * 1
+  ) {
+    super(monitor, TaskSyncWhenIdle.taskName)
+  }
+
+  trigger(nowMsecsSinceEpoch: number): { run: boolean } {
+    const s = this.storage
+
+    const run = false
+
+    return { run }
+  }
+
+  async runTask(): Promise<string> {
+    // TODO...
+    return ''
+  }
+}

package/src/monitor/tasks/WalletMonitorTask.ts

@@ -0,0 +1,47 @@
+import { MonitorStorage, Monitor } from '../Monitor'
+
+/**
+ * A monitor task performs some periodic or state triggered maintenance function
+ * on the data managed by a wallet (Bitcoin UTXO manager, aka wallet)
+ *
+ * The monitor maintains a collection of tasks.
+ *
+ * It runs each task's non-asynchronous trigger to determine if the runTask method needs to run.
+ *
+ * Tasks that need to be run are run consecutively by awaiting their async runTask override method.
+ *
+ * The monitor then waits a fixed interval before repeating...
+ *
+ * Tasks may use the monitor_events table to persist their execution history.
+ * This is done by accessing the wathman.storage object.
+ */
+
+export abstract class WalletMonitorTask {
+  /**
+   * Set by monitor each time runTask completes
+   */
+  lastRunMsecsSinceEpoch = 0
+
+  storage: MonitorStorage
+
+  constructor(
+    public monitor: Monitor,
+    public name: string
+  ) {
+    this.storage = monitor.storage
+  }
+
+  /**
+   * Override to handle async task setup configuration.
+   *
+   * Called before first call to `trigger`
+   */
+  async asyncSetup(): Promise<void> {}
+
+  /**
+   * Return true if `runTask` needs to be called now.
+   */
+  abstract trigger(nowMsecsSinceEpoch: number): { run: boolean }
+
+  abstract runTask(): Promise<string>
+}