@bsv/wallet-toolbox 1.1.0

package/src/storage/methods/purgeData.ts

@@ -0,0 +1,276 @@
+import { Beef } from '@bsv/sdk'
+import { Knex } from 'knex'
+import { table } from '../index.client'
+import { sdk } from '../../index.client'
+import { StorageKnex } from '../StorageKnex'
+
+export async function purgeData(
+  storage: StorageKnex,
+  params: sdk.PurgeParams,
+  trx?: sdk.TrxToken
+): Promise<sdk.PurgeResults> {
+  const r: sdk.PurgeResults = { count: 0, log: '' }
+  const defaultAge = 1000 * 60 * 60 * 24 * 14
+
+  const runPurgeQuery = async <T extends object>(
+    pq: PurgeQuery
+  ): Promise<void> => {
+    try {
+      pq.sql = pq.q.toString()
+      const count = await pq.q
+      if (count > 0) {
+        r.count += count
+        r.log += `${count} ${pq.log}\n`
+      }
+    } catch (eu: unknown) {
+      // eslint-disable-next-line @typescript-eslint/no-unused-vars
+      const e = sdk.WalletError.fromUnknown(eu)
+      throw eu
+    }
+  }
+
+  if (params.purgeCompleted) {
+    const age = params.purgeCompletedAge || defaultAge
+    const before = toSqlWhereDate(new Date(Date.now() - age))
+
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const qs: PurgeQuery[] = []
+
+    // select * from transactions where updated_at < '2024-08-20' and status = 'completed' and not provenTxId is null and (not truncatedExternalInputs is null or not beef is null or not rawTx is null)
+    qs.push({
+      log: 'conpleted transactions purged of transient data',
+      q: storage
+        .toDb(trx)('transactions')
+        .update({
+          inputBEEF: null,
+          rawTx: null
+        })
+        .where('updated_at', '<', before)
+        .where('status', 'completed')
+        .whereNotNull('provenTxId')
+        .where(function () {
+          this.orWhereNotNull('inputBEEF')
+          this.orWhereNotNull('rawTx')
+        })
+    })
+
+    const completedReqs = await storage
+      .toDb(trx)<{ provenTxReqId: number }>('proven_tx_reqs')
+      .select('provenTxReqId')
+      .where('updated_at', '<', before)
+      .where('status', 'completed')
+      .whereNotNull('provenTxId')
+      .where('notified', 1)
+    const completedReqIds = completedReqs.map(o => o.provenTxReqId)
+
+    if (completedReqIds.length > 0) {
+      qs.push({
+        log: 'completed proven_tx_reqs deleted',
+        q: storage
+          .toDb(trx)('proven_tx_reqs')
+          .whereIn('provenTxReqId', completedReqIds)
+          .delete()
+      })
+    }
+
+    for (const q of qs) await runPurgeQuery(q)
+  }
+
+  if (params.purgeFailed) {
+    const age = params.purgeFailedAge || defaultAge
+    const before = toSqlWhereDate(new Date(Date.now() - age))
+
+    const qs: PurgeQuery[] = []
+
+    const failedTxsQ = storage
+      .toDb(trx)<{ transactionId: number }>('transactions')
+      .select('transactionId')
+      .where('updated_at', '<', before)
+      .where('status', 'failed')
+    const txs = await failedTxsQ
+    const failedTxIds = txs.map(tx => tx.transactionId)
+
+    await deleteTransactions(failedTxIds, qs, 'failed', true)
+
+    const invalidReqs = await storage
+      .toDb(trx)<{ provenTxReqId: number }>('proven_tx_reqs')
+      .select('provenTxReqId')
+      .where('updated_at', '<', before)
+      .where('status', 'invalid')
+    const invalidReqIds = invalidReqs.map(o => o.provenTxReqId)
+    if (invalidReqIds.length > 0)
+      qs.push({
+        log: 'invalid proven_tx_reqs deleted',
+        q: storage
+          .toDb(trx)('proven_tx_reqs')
+          .whereIn('provenTxReqId', invalidReqIds)
+          .delete()
+      })
+
+    const doubleSpendReqs = await storage
+      .toDb(trx)<{ provenTxReqId: number }>('proven_tx_reqs')
+      .select('provenTxReqId')
+      .where('updated_at', '<', before)
+      .where('status', 'doubleSpend')
+    const doubleSpendReqIds = doubleSpendReqs.map(o => o.provenTxReqId)
+    if (doubleSpendReqIds.length > 0)
+      qs.push({
+        log: 'doubleSpend proven_tx_reqs deleted',
+        q: storage
+          .toDb(trx)('proven_tx_reqs')
+          .whereIn('provenTxReqId', doubleSpendReqIds)
+          .delete()
+      })
+
+    for (const q of qs) await runPurgeQuery(q)
+  }
+
+  if (params.purgeSpent) {
+    const age = params.purgeSpentAge || defaultAge
+    const before = toSqlWhereDate(new Date(Date.now() - age))
+
+    const beef = new Beef()
+    const utxos = await storage.findOutputs({
+      partial: { spendable: true },
+      txStatus: ['sending', 'unproven', 'completed', 'nosend']
+    })
+    for (const utxo of utxos) {
+      // Figure out all the txids required to prove the validity of this utxo and merge proofs into beef.
+      const options: sdk.StorageGetBeefOptions = {
+        mergeToBeef: beef,
+        ignoreServices: true
+      }
+      if (utxo.txid) await storage.getBeefForTransaction(utxo.txid, options)
+    }
+    const proofTxids: Record<string, boolean> = {}
+    for (const btx of beef.txs) proofTxids[btx.txid] = true
+
+    let qs: PurgeQuery[] = []
+
+    const spentTxsQ = storage
+      .toDb(trx)<table.Transaction>('transactions')
+      .where('updated_at', '<', before)
+      .where('status', 'completed')
+      .whereRaw(
+        `not exists(select outputId from outputs as o where o.transactionId = transactions.transactionId and o.spendable = 1)`
+      )
+    const txs: table.Transaction[] = await spentTxsQ
+    // Save any spent txid still needed to prove a utxo:
+    const nptxs = txs.filter(t => !proofTxids[t.txid || ''])
+    let spentTxIds = nptxs.map(tx => tx.transactionId)
+
+    if (spentTxIds.length > 0) {
+      const update: Partial<table.Output> = { spentBy: undefined }
+      qs.push({
+        log: 'spent outputs no longer tracked by spentBy',
+        q: storage
+          .toDb(trx)<table.Output>('outputs')
+          .update(
+            storage.validatePartialForUpdate(update, undefined, ['spendable'])
+          )
+          .where('spendable', false)
+          .whereIn('spentBy', spentTxIds)
+      })
+
+      await deleteTransactions(spentTxIds, qs, 'spent', false)
+
+      for (const q of qs) await runPurgeQuery(q)
+    }
+  }
+
+  // Delete proven_txs no longer referenced by remaining transactions.
+  const qs: PurgeQuery[] = []
+  qs.push({
+    log: 'orphan proven_txs deleted',
+    q: storage
+      .toDb(trx)('proven_txs')
+      .whereRaw(
+        `not exists(select * from transactions as t where t.txid = proven_txs.txid or t.provenTxId = proven_txs.provenTxId)`
+      )
+      .whereRaw(
+        `not exists(select * from proven_tx_reqs as r where r.txid = proven_txs.txid or r.provenTxId = proven_txs.provenTxId)`
+      )
+      .delete()
+  })
+  for (const q of qs) await runPurgeQuery(q)
+
+  return r
+
+  async function deleteTransactions(
+    transactionIds: number[],
+    qs: PurgeQuery[],
+    reason: string,
+    markNotSpentBy: boolean
+  ) {
+    if (transactionIds.length > 0) {
+      const outputs = await storage
+        .toDb(trx)<{ outputId: number }>('outputs')
+        .select('outputId')
+        .whereIn('transactionId', transactionIds)
+      const outputIds = outputs.map(o => o.outputId)
+      if (outputIds.length > 0) {
+        qs.push({
+          log: `${reason} output_tags_map deleted`,
+          q: storage
+            .toDb(trx)<table.OutputTagMap>('output_tags_map')
+            .whereIn('outputId', outputIds)
+            .delete()
+        })
+        qs.push({
+          log: `${reason} outputs deleted`,
+          q: storage
+            .toDb(trx)<table.Output>('outputs')
+            .whereIn('outputId', outputIds)
+            .delete()
+        })
+      }
+
+      qs.push({
+        log: `${reason} tx_labels_map deleted`,
+        q: storage
+          .toDb(trx)<table.TxLabelMap>('tx_labels_map')
+          .whereIn('transactionId', transactionIds)
+          .delete()
+      })
+
+      qs.push({
+        log: `${reason} commissions deleted`,
+        q: storage
+          .toDb(trx)<table.Commission>('commissions')
+          .whereIn('transactionId', transactionIds)
+          .delete()
+      })
+
+      if (markNotSpentBy) {
+        qs.push({
+          log: 'unspent outputs updated to spendable',
+          q: storage
+            .toDb(trx)<table.Output>('outputs')
+            .update({ spendable: true, spentBy: undefined })
+            .whereIn('spentBy', transactionIds)
+        })
+      }
+
+      qs.push({
+        log: `${reason} transactions deleted`,
+        q: storage
+          .toDb(trx)<table.Transaction>('transactions')
+          .whereIn('transactionId', transactionIds)
+          .delete()
+      })
+    }
+  }
+}
+
+interface PurgeQuery {
+  q: Knex.QueryBuilder<any, number>
+  sql?: string
+  log: string
+}
+
+function toSqlWhereDate(d: Date): string {
+  let s = d.toISOString()
+  s = s.replace('T', ' ')
+  s = s.replace('Z', '')
+  return s
+}

package/src/storage/methods/reviewStatus.ts

@@ -0,0 +1,95 @@
+import { Beef } from '@bsv/sdk'
+import { Knex } from 'knex'
+import { table } from '../index.client'
+import { sdk } from '../../index.client'
+import { StorageKnex } from '../StorageKnex'
+
+export async function reviewStatus(
+  storage: StorageKnex,
+  args: { agedLimit: Date; trx?: sdk.TrxToken }
+): Promise<{ log: string }> {
+  const r: { log: string } = { log: '' }
+
+  const runReviewStatusQuery = async <T extends object>(
+    pq: ReviewStatusQuery
+  ): Promise<void> => {
+    try {
+      pq.sql = pq.q.toString()
+      const count = await pq.q
+      if (count > 0) {
+        r.log += `${count} ${pq.log}\n`
+      }
+    } catch (eu: unknown) {
+      // eslint-disable-next-line @typescript-eslint/no-unused-vars
+      const e = sdk.WalletError.fromUnknown(eu)
+      throw eu
+    }
+  }
+
+  const k = storage.toDb(args.trx)
+
+  const qs: ReviewStatusQuery[] = []
+
+  qs.push({
+    log: `transactions updated to status of 'failed' where provenTxReq with matching txid is 'invalid'`,
+    /*
+        UPDATE transactions SET status = 'failed'
+        WHERE exists(select 1 from proven_tx_reqs as r where transactions.txid = r.txid and r.status = 'invalid')
+        */
+    q: k<table.Transaction>('transactions')
+      .update({ status: 'failed' })
+      .whereNot({ status: 'failed' })
+      .whereExists(function () {
+        this.select(k.raw(1))
+          .from('proven_tx_reqs as r')
+          .whereRaw(`transactions.txid = r.txid and r.status = 'invalid'`)
+      })
+  })
+
+  qs.push({
+    log: `outputs updated to spendable where spentBy is a transaction with status 'failed'`,
+    /*
+        UPDATE outputs SET spentBy = null, spendable = 1
+        where exists(select 1 from transactions as t where outputs.spentBy = t.transactionId and t.status = 'failed')
+        */
+    q: k<table.Output>('outputs')
+      .update({ spentBy: undefined, spendable: true })
+      .whereExists(function () {
+        this.select(k.raw(1))
+          .from('transactions as t')
+          .whereRaw(`outputs.spentBy = t.transactionId and t.status = 'failed'`)
+      })
+  })
+
+  qs.push({
+    log: `transactions updated with provenTxId and status of 'completed' where provenTx with matching txid exists`,
+    /*
+        UPDATE transactions SET status = 'completed', provenTxId = p.provenTxId
+        FROM proven_txs p
+        WHERE transactions.txid = p.txid AND transactions.provenTxId IS NULL
+        */
+    q: k<table.Transaction>('transactions')
+      .update({
+        status: 'completed',
+        provenTxId: k.raw(
+          '(SELECT provenTxId FROM proven_txs AS p WHERE transactions.txid = p.txid)'
+        )
+      })
+      .whereNull('provenTxId')
+      .whereExists(function () {
+        this.select(k.raw(1))
+          .from('proven_txs as p')
+          .whereRaw('transactions.txid = p.txid')
+      })
+  })
+
+  for (const q of qs) await runReviewStatusQuery(q)
+
+  return r
+}
+
+interface ReviewStatusQuery {
+  q: Knex.QueryBuilder<any, number>
+  sql?: string
+  log: string
+}

package/src/storage/remoting/StorageClient.ts

@@ -0,0 +1,306 @@
+/**
+ * StorageClient.ts
+ *
+ * A client-side "remoted" WalletStorage that fulfills the WalletStorage interface
+ * by sending JSON-RPC calls to a configured remote WalletStorageServer.
+ */
+
+import {
+  AbortActionArgs,
+  AbortActionResult,
+  InternalizeActionArgs,
+  InternalizeActionResult,
+  ListActionsResult,
+  ListCertificatesResult,
+  ListOutputsResult,
+  RelinquishCertificateArgs,
+  RelinquishOutputArgs,
+  WalletInterface
+} from '@bsv/sdk'
+import { sdk, table } from '../../index.client'
+import { AuthFetch } from '@bsv/sdk'
+
+export class StorageClient implements sdk.WalletStorageProvider {
+  private endpointUrl: string
+  private nextId = 1
+  private authClient: AuthFetch
+
+  // Track ephemeral (in-memory) "settings" if you wish to align with isAvailable() checks
+  public settings?: table.Settings
+
+  constructor(wallet: WalletInterface, endpointUrl: string) {
+    this.authClient = new AuthFetch(wallet)
+    this.endpointUrl = endpointUrl
+  }
+
+  isStorageProvider(): boolean {
+    return false
+  }
+
+  //////////////////////////////////////////////////////////////////////////////
+  // JSON-RPC helper
+  //////////////////////////////////////////////////////////////////////////////
+
+  /**
+   * Make a JSON-RPC call to the remote server.
+   * @param method The WalletStorage method name to call.
+   * @param params The array of parameters to pass to the method in order.
+   */
+  private async rpcCall<T>(method: string, params: unknown[]): Promise<T> {
+    const id = this.nextId++
+    const body = {
+      jsonrpc: '2.0',
+      method,
+      params,
+      id
+    }
+
+    const response = await this.authClient.fetch(this.endpointUrl, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify(body)
+    })
+
+    if (!response.ok) {
+      throw new Error(
+        `WalletStorageClient rpcCall: network error ${response.status} ${response.statusText}`
+      )
+    }
+
+    const json = await response.json()
+    if (json.error) {
+      const { code, message, data } = json.error
+      const err = new Error(`RPC Error: ${message}`)
+      // You could attach more info here if you like:
+      ;(err as any).code = code
+      ;(err as any).data = data
+      throw err
+    }
+
+    return json.result
+  }
+
+  //////////////////////////////////////////////////////////////////////////////
+  // In a real environment, you might do lazy or real "makeAvailable" logic
+  // For demonstration, we assume that the remote store might return its "settings"
+  // and we store them locally in `this.settings`.
+  //////////////////////////////////////////////////////////////////////////////
+
+  isAvailable(): boolean {
+    // We'll just say "yes" if we have settings
+    return !!this.settings
+  }
+
+  getSettings(): table.Settings {
+    if (!this.settings) {
+      throw new sdk.WERR_INVALID_OPERATION(
+        'call makeAvailable at least once before getSettings'
+      )
+    }
+    return this.settings
+  }
+
+  async makeAvailable(): Promise<table.Settings> {
+    if (!this.settings) {
+      this.settings = await this.rpcCall<table.Settings>('makeAvailable', [])
+    }
+    return this.settings
+  }
+
+  //////////////////////////////////////////////////////////////////////////////
+  //
+  // Implementation of all WalletStorage interface methods
+  // They are simple pass-thrus to rpcCall
+  //
+  // IMPORTANT: The parameter ordering must match exactly as in your interface.
+  //////////////////////////////////////////////////////////////////////////////
+
+  async destroy(): Promise<void> {
+    return this.rpcCall<void>('destroy', [])
+  }
+
+  async migrate(
+    storageName: string,
+    storageIdentityKey: string
+  ): Promise<string> {
+    return this.rpcCall<string>('migrate', [storageName])
+  }
+
+  getServices(): sdk.WalletServices {
+    // Typically, the client would not store or retrieve "Services" from a remote server.
+    // The "services" in local in-memory usage is a no-op or your own approach:
+    throw new Error(
+      'getServices() not implemented in remote client. This method typically is not used remotely.'
+    )
+  }
+
+  setServices(v: sdk.WalletServices): void {
+    // Typically no-op for remote client
+    // Because "services" are usually local definitions to the Storage.
+  }
+
+  async internalizeAction(
+    auth: sdk.AuthId,
+    args: InternalizeActionArgs
+  ): Promise<InternalizeActionResult> {
+    return this.rpcCall<InternalizeActionResult>('internalizeAction', [
+      auth,
+      args
+    ])
+  }
+
+  async createAction(
+    auth: sdk.AuthId,
+    args: sdk.ValidCreateActionArgs
+  ): Promise<sdk.StorageCreateActionResult> {
+    return this.rpcCall<sdk.StorageCreateActionResult>('createAction', [
+      auth,
+      args
+    ])
+  }
+
+  async processAction(
+    auth: sdk.AuthId,
+    args: sdk.StorageProcessActionArgs
+  ): Promise<sdk.StorageProcessActionResults> {
+    return this.rpcCall<sdk.StorageProcessActionResults>('processAction', [
+      auth,
+      args
+    ])
+  }
+
+  async abortAction(
+    auth: sdk.AuthId,
+    args: AbortActionArgs
+  ): Promise<AbortActionResult> {
+    return this.rpcCall<AbortActionResult>('abortAction', [auth, args])
+  }
+
+  async findOrInsertUser(
+    identityKey
+  ): Promise<{ user: table.User; isNew: boolean }> {
+    return this.rpcCall<{ user: table.User; isNew: boolean }>(
+      'findOrInsertUser',
+      [identityKey]
+    )
+  }
+
+  async findOrInsertSyncStateAuth(
+    auth: sdk.AuthId,
+    storageIdentityKey: string,
+    storageName: string
+  ): Promise<{ syncState: table.SyncState; isNew: boolean }> {
+    return this.rpcCall<{ syncState: table.SyncState; isNew: boolean }>(
+      'findOrInsertSyncStateAuth',
+      [auth, storageIdentityKey, storageName]
+    )
+  }
+
+  async insertCertificateAuth(
+    auth: sdk.AuthId,
+    certificate: table.CertificateX
+  ): Promise<number> {
+    return this.rpcCall<number>('insertCertificateAuth', [auth, certificate])
+  }
+
+  async listActions(
+    auth: sdk.AuthId,
+    vargs: sdk.ValidListActionsArgs
+  ): Promise<ListActionsResult> {
+    return this.rpcCall<ListActionsResult>('listActions', [auth, vargs])
+  }
+
+  async listOutputs(
+    auth: sdk.AuthId,
+    vargs: sdk.ValidListOutputsArgs
+  ): Promise<ListOutputsResult> {
+    return this.rpcCall<ListOutputsResult>('listOutputs', [auth, vargs])
+  }
+
+  async listCertificates(
+    auth: sdk.AuthId,
+    vargs: sdk.ValidListCertificatesArgs
+  ): Promise<ListCertificatesResult> {
+    return this.rpcCall<ListCertificatesResult>('listCertificates', [
+      auth,
+      vargs
+    ])
+  }
+
+  async findCertificatesAuth(
+    auth: sdk.AuthId,
+    args: sdk.FindCertificatesArgs
+  ): Promise<table.Certificate[]> {
+    return this.rpcCall<table.Certificate[]>('findCertificatesAuth', [
+      auth,
+      args
+    ])
+  }
+
+  async findOutputBasketsAuth(
+    auth: sdk.AuthId,
+    args: sdk.FindOutputBasketsArgs
+  ): Promise<table.OutputBasket[]> {
+    return this.rpcCall<table.OutputBasket[]>('findOutputBaskets', [auth, args])
+  }
+
+  async findOutputsAuth(
+    auth: sdk.AuthId,
+    args: sdk.FindOutputsArgs
+  ): Promise<table.Output[]> {
+    return this.rpcCall<table.Output[]>('findOutputsAuth', [auth, args])
+  }
+
+  findProvenTxReqs(
+    args: sdk.FindProvenTxReqsArgs
+  ): Promise<table.ProvenTxReq[]> {
+    return this.rpcCall<table.ProvenTxReq[]>('findProvenTxReqs', [args])
+  }
+
+  async relinquishCertificate(
+    auth: sdk.AuthId,
+    args: RelinquishCertificateArgs
+  ): Promise<number> {
+    return this.rpcCall<number>('relinquishCertificate', [auth, args])
+  }
+
+  async relinquishOutput(
+    auth: sdk.AuthId,
+    args: RelinquishOutputArgs
+  ): Promise<number> {
+    return this.rpcCall<number>('relinquishOutput', [auth, args])
+  }
+
+  async processSyncChunk(
+    args: sdk.RequestSyncChunkArgs,
+    chunk: sdk.SyncChunk
+  ): Promise<sdk.ProcessSyncChunkResult> {
+    return this.rpcCall<sdk.ProcessSyncChunkResult>('processSyncChunk', [
+      args,
+      chunk
+    ])
+  }
+
+  async getSyncChunk(args: sdk.RequestSyncChunkArgs): Promise<sdk.SyncChunk> {
+    return this.rpcCall<sdk.SyncChunk>('getSyncChunk', [args])
+  }
+
+  async updateProvenTxReqWithNewProvenTx(
+    args: sdk.UpdateProvenTxReqWithNewProvenTxArgs
+  ): Promise<sdk.UpdateProvenTxReqWithNewProvenTxResult> {
+    return this.rpcCall<sdk.UpdateProvenTxReqWithNewProvenTxResult>(
+      'updateProvenTxReqWithNewProvenTx',
+      [args]
+    )
+  }
+
+  async setActive(
+    auth: sdk.AuthId,
+    newActiveStorageIdentityKey: string
+  ): Promise<number> {
+    return this.rpcCall<number>('setActive', [
+      auth,
+      newActiveStorageIdentityKey
+    ])
+  }
+}