balanceofsatoshis 11.5.1 → 11.7.0

package/CHANGELOG.md

@@ -1,5 +1,13 @@
 # Versions
 
+## Version 11.7.0
+
+- `fund`: Add `MAX` variable to allow spending down specified UTXOs
+
+## Version 11.6.2
+
+- `clean-failed-payments`: Add method to clean out failed past payments
+
 ## Version 11.5.1
 
 - `--avoid`: Correct naming`OUT_FEE_RATE`/`IN_FEE_RATE` to `OPPOSITE_FEE_RATE`

package/bos

@@ -395,6 +395,26 @@ prog
     });
   })
 
+  // Clean out failed payments
+  .command('clean-failed-payments', 'Clean out past failed payment data')
+  .help('Remove old failed payment data for probes and other failed payments')
+  .option('--dryrun', 'Avoid actually deleting the failed payment record')
+  .option('--node <node_name>', 'Clean failed payments on a saved node')
+  .action((args, options, logger) => {
+    return new Promise(async (resolve, reject) => {
+      try {
+        return wallets.cleanFailedPayments({
+          logger,
+          is_dry_run: !!options.dryrun,
+          lnd: (await lnd.authenticatedLnd({logger, node: options.node})).lnd,
+        },
+        responses.returnObject({logger, reject, resolve}));
+      } catch (err) {
+        return reject(err);
+      }
+    });
+  })
+
   // Determine the outcomes of channel closings
   .command('closed', 'Get the status of a channel closings')
   .help('Channel closes with chain-transaction derived resolution details')
@@ -616,6 +636,7 @@ prog
   .command('fund', 'Make a signed transaction spending on-chain funds')
   .help('Use LND UTXOs to craft a signed raw transaction sending to addresses')
   .help('Specify <address> <amount> <address> <amount> for addresses, amounts')
+  .help('Amounts support formulas, use MAX to reference selected utxos total')
   .argument('<address_amount...>', 'Address and amount to send')
   .option('--dryrun', 'Avoid locking up UTXOs')
   .option('--fee-rate <fee>', 'Per vbyte fee rate for on-chain tx fee', INT)

package/chain/fund_transaction.js

@@ -3,9 +3,11 @@ const asyncEach = require('async/each');
 const {formatTokens} = require('ln-sync');
 const {fundPsbt} = require('ln-service');
 const {getChainFeeRate} = require('ln-service');
+const {getMaxFundAmount} = require('ln-sync');
 const {getUtxos} = require('ln-service');
 const {returnResult} = require('asyncjs-util');
 const {signPsbt} = require('ln-service');
+const {Transaction} = require('bitcoinjs-lib');
 const {unlockUtxo} = require('ln-service');
 
 const {parseAmount} = require('./../display');
@@ -15,11 +17,15 @@ const asOutpoint = utxo => `${utxo.transaction_id}:${utxo.transaction_vout}`;
 const asInput = n => ({transaction_id: n.id, transaction_vout: n.vout});
 const asUtxo = n => ({id: n.slice(0, 64), vout: Number(n.slice(65))});
 const dustValue = 293;
+const formattedFeeRate = n => n.toFixed(2);
+const {fromHex} = Transaction;
+const hasMaxAmount = amounts => !!amounts.find(n => !!n && !!/max/gim.test(n));
 const {isArray} = Array;
 const isOutpoint = n => !!n && /^[0-9A-F]{64}:[0-9]{1,6}$/i.test(n);
 const isPublicKey = n => !!n && /^0[2-3][0-9A-F]{64}$/i.test(n);
 const minConfs = 1;
 const sumOf = arr => arr.reduce((sum, n) => sum + n, Number());
+const txHashAsTxId = hash => hash.reverse().toString('hex');
 
 /** Fund and sign a transaction
 
@@ -92,8 +98,13 @@ module.exports = (args, cbk) => {
       // Get the current fee rate
       getFee: ['validate', ({}, cbk) => getChainFeeRate({lnd: args.lnd}, cbk)],
 
-      // Derive exact outputs
+      // Derive a list of outputs to guide input selection
       outputs: ['validate', ({}, cbk) => {
+        // Exit early when the amount is open ended and thus depends on inputs
+        if (hasMaxAmount(args.amounts)) {
+          return cbk();
+        }
+
         try {
           const outputs = args.addresses.map((address, i) => {
             const {tokens} = parseAmount({amount: args.amounts[i]});
@@ -107,19 +118,12 @@ module.exports = (args, cbk) => {
         }
       }],
 
-      // Get UTXOs
-      getUtxos: ['validate', ({}, cbk) => {
-        // Exit early when not selecting UTXOs
-        if (!args.is_selecting_utxos) {
-          return cbk();
-        }
-
-        return getUtxos({lnd: args.lnd, min_confirmations: minConfs}, cbk);
-      }],
+      // Get UTXOs to use for input selection and final fee rate calculation
+      getUtxos: ['validate', ({}, cbk) => getUtxos({lnd: args.lnd}, cbk)],
 
       // Select inputs to spend
       utxos: ['getUtxos', 'outputs', ({getUtxos, outputs}, cbk) => {
-        // Exit early when UTXOs are specified already
+        // Exit early when UTXOs are all specified already
         if (!!args.utxos.length) {
           return cbk(null, args.utxos);
         }
@@ -129,15 +133,16 @@ module.exports = (args, cbk) => {
           return cbk(null, []);
         }
 
+        // Only selecting confirmed utxos is supported
+        const utxos = getUtxos.utxos.filter(n => !!n.confirmation_count);
+
         // Make sure there are some UTXOs to select
-        if (!getUtxos.utxos.length) {
+        if (!utxos.length) {
           return cbk([400, 'WalletHasZeroConfirmedUtxos']);
         }
 
-        const amounts = outputs.map(n => n.tokens);
-
         return args.ask({
-          choices: getUtxos.utxos.map(utxo => ({
+          choices: utxos.map(utxo => ({
             name: `${asBigUnit(utxo.tokens)} ${asOutpoint(utxo)}`,
             value: asOutpoint(utxo),
           })),
@@ -145,14 +150,22 @@ module.exports = (args, cbk) => {
           name: 'inputs',
           type: 'checkbox',
           validate: input => {
+            // A selection is required
             if (!input.length) {
               return false;
             }
 
             const tokens = sumOf(input.map(utxo => {
-              return getUtxos.utxos.find(n => asOutpoint(n) === utxo).tokens;
+              return utxos.find(n => asOutpoint(n) === utxo).tokens;
             }));
 
+            // Exit early when the amount is open ended
+            if (hasMaxAmount(args.amounts)) {
+              return true;
+            }
+
+            const amounts = outputs.map(n => n.tokens);
+
             const missingTok = asBigUnit(sumOf(amounts) - tokens);
 
             if (tokens < sumOf(amounts)) {
@@ -165,33 +178,113 @@ module.exports = (args, cbk) => {
         ({inputs}) => cbk(null, inputs));
       }],
 
+      // Calculate the maximum possible amount to fund for selected inputs
+      getMax: [
+        'getFee',
+        'getUtxos',
+        'outputs',
+        'utxos',
+        ({getFee, getUtxos, outputs, utxos}, cbk) =>
+      {
+        // Exit early when the amount is not open ended
+        if (!hasMaxAmount(args.amounts)) {
+          return cbk(null, {});
+        }
+
+        // Because of anchor channel requirements, don't allow open ended max
+        if (!utxos.length) {
+          return cbk([400, 'MaxAmountOnlySupportedWhenUtxosSpecified']);
+        }
+
+        const feeRate = args.fee_tokens_per_vbyte || getFee.tokens_per_vbyte;
+
+        // Find the local UTXOs in order to get the input values
+        const spend = utxos.map(outpoint => {
+          return getUtxos.utxos.find(n => asOutpoint(n) === outpoint);
+        });
+
+        // Make sure that all inputs are known
+        if (spend.filter(n => !n).length) {
+          return cbk([400, 'UnknownInputSelected', {known_utxos: spend}]);
+        }
+
+        return getMaxFundAmount({
+          addresses: args.addresses,
+          fee_tokens_per_vbyte: feeRate,
+          inputs: spend.map(utxo => ({
+            tokens: utxo.tokens,
+            transaction_id: utxo.transaction_id,
+            transaction_vout: utxo.transaction_vout,
+          })),
+          lnd: args.lnd,
+        },
+        cbk);
+      }],
+
+      // Parse amounts and put together the final set of outputs
+      finalOutputs: ['getMax', ({getMax}, cbk) => {
+        try {
+          const outputs = args.addresses.map((address, i) => {
+            const amount = args.amounts[i];
+            const variables = {max: getMax.max_tokens};
+
+            return {address, tokens: parseAmount({amount, variables}).tokens};
+          });
+
+          return cbk(null, outputs);
+        } catch (err) {
+          return cbk([400, err.message]);
+        }
+      }],
+
       // Create a funded PSBT
-      fund: ['getFee', 'outputs', 'utxos', ({getFee, outputs, utxos}, cbk) => {
+      fund: [
+        'finalOutputs',
+        'getFee',
+        'utxos',
+        ({finalOutputs, getFee, utxos}, cbk) =>
+      {
         const inputs = utxos.map(asUtxo).map(asInput);
-        const fee = args.fee_tokens_per_vbyte || getFee.tokens_per_vbyte;
+        const feeRate = args.fee_tokens_per_vbyte || getFee.tokens_per_vbyte;
 
-        if (!!outputs.filter(n => n.tokens < dustValue).length) {
+        if (!!finalOutputs.filter(n => n.tokens < dustValue).length) {
           return cbk([400, 'ExpectedNonDustAmountValueForFundingAmount']);
         }
 
         args.logger.info({
-          fee_rate: fee,
-          send_to: outputs.map(output => ({
-            [output.address]: formatTokens({tokens: output.tokens}).display,
+          send_to: finalOutputs.map(({address, tokens}) => ({
+            [address]: formatTokens({tokens}).display,
           })),
+          requested_fee_rate: feeRate,
         });
 
         return fundPsbt({
-          outputs,
+          fee_tokens_per_vbyte: feeRate,
           inputs: !!inputs.length ? inputs : undefined,
           lnd: args.lnd,
-          fee_tokens_per_vbyte: fee,
+          outputs: finalOutputs,
         },
         cbk);
       }],
 
-      // Unlock the locked UTXO in a dry run scenario
-      unlock: ['fund', ({fund}, cbk) => {
+      // Sign the funded PSBT
+      sign: ['fund', ({fund}, cbk) => {
+        const [change] = fund.outputs.filter(n => !!n.is_change);
+        const total = sumOf(fund.outputs.map(n => n.tokens));
+
+        const tokens = !!change ? change.tokens : undefined;
+
+        args.logger.info({
+          change: !!tokens ? formatTokens({tokens}).display : undefined,
+          sum_of_outputs: formatTokens({tokens: total}).display,
+          spending_utxos: fund.inputs.map(asOutpoint),
+        });
+
+        return signPsbt({lnd: args.lnd, psbt: fund.psbt}, cbk);
+      }],
+
+      // Unlock the locked UTXOs in a dry run scenario
+      unlock: ['fund', 'sign', ({fund}, cbk) => {
         // Exit early and keep UTXOs locked when not a dry run
         if (!args.is_dry_run) {
           return cbk();
@@ -209,25 +302,35 @@ module.exports = (args, cbk) => {
         cbk);
       }],
 
-      // Sign the funded PSBT
-      sign: ['fund', ({fund}, cbk) => {
-        const [change] = fund.outputs.filter(n => !!n.is_change);
-        const total = sumOf(fund.outputs.map(n => n.tokens));
-
-        const tokens = !!change ? change.tokens : undefined;
+      // Final funded transaction
+      funded: ['getUtxos', 'sign', ({getUtxos, sign}, cbk) => {
+        // Match the inputs of the tx up to the wallet outputs
+        const tx = fromHex(sign.transaction);
+
+        // Find the UTXOs that are being spent in the final transaction
+        const spending = tx.ins.map(input => {
+          const outpoint = asOutpoint({
+            transaction_id: txHashAsTxId(input.hash),
+            transaction_vout: input.index,
+          });
 
-        args.logger.info({
-          change: !!tokens ? formatTokens({tokens}).display : undefined,
-          sum_of_outputs: formatTokens({tokens: total}).display,
-          spending_utxos: fund.inputs.map(asOutpoint),
+          return getUtxos.utxos.find(n => asOutpoint(n) === outpoint);
         });
 
-        return signPsbt({lnd: args.lnd, psbt: fund.psbt}, cbk);
-      }],
+        // Make sure the spending UTXOs are known
+        if (spending.filter(n => !n).length) {
+          return cbk([503, 'ExpectedSpendingKnownUtxosForFundedTx']);
+        }
 
-      // Final funded transaction
-      funded: ['sign', ({sign}, cbk) => {
-        return cbk(null, {signed_transaction: sign.transaction});
+        const inputsValue = sumOf(spending.map(n => n.tokens));
+        const outputsValue = sumOf(tx.outs.map(n => n.value));
+
+        const feeTotal = inputsValue - outputsValue;
+
+        return cbk(null, {
+          fee_tokens_per_vbyte: formattedFeeRate(feeTotal / tx.virtualSize()),
+          signed_transaction: sign.transaction,
+        });
       }],
     },
     returnResult({reject, resolve, of: 'funded'}, cbk));

package/commands/api.json

@@ -287,6 +287,22 @@
     {
       "method": "getConnectedWatchtowers"
     },
+    {
+      "arguments": [
+        {
+          "description": "Results limit",
+          "named": "limit",
+          "optional": true,
+          "type": "number"
+        },
+        {
+          "description": "Paging token",
+          "named": "token",
+          "optional": true
+        }
+      ],
+      "method": "getFailedPayments"
+    },
     {
       "method": "getFeeRates"
     },

package/package.json

@@ -35,8 +35,8 @@
     "inquirer": "8.2.0",
     "invoices": "2.0.0",
     "ln-accounting": "5.0.3",
-    "ln-service": "52.11.1",
-    "ln-sync": "2.0.2",
+    "ln-service": "52.12.1",
+    "ln-sync": "2.1.0",
     "ln-telegram": "3.3.1",
     "moment": "2.29.1",
     "paid-services": "3.0.0",
@@ -80,5 +80,5 @@
     "postpublish": "docker buildx build --platform linux/amd64,linux/arm64,linux/arm/v7 -t alexbosworth/balanceofsatoshis --push .",
     "test": "tap --branches=1 --functions=1 --lines=1 --statements=1 -t 60 test/arrays/*.js test/balances/*.js test/chain/*.js test/display/*.js test/encryption/*.js test/fiat/*.js test/lnd/*.js test/network/*.js test/nodes/*.js test/peers/*.js test/responses/*.js test/routing/*.js test/services/*.js test/swaps/*.js test/tags/*.js test/wallets/*.js"
   },
-  "version": "11.5.1"
+  "version": "11.7.0"
 }

package/wallets/clean_failed_payments.js

@@ -0,0 +1,153 @@
+const asyncAuto = require('async/auto');
+const asyncEachSeries = require('async/eachSeries');
+const asyncWhilst = require('async/whilst');
+const {deletePayment} = require('ln-service');
+const {getFailedPayments} = require('ln-service');
+const {getWalletVersion} = require('ln-service');
+const {returnResult} = require('asyncjs-util');
+
+const defaultPagingLimit = 1000;
+const {isArray} = Array;
+const unsupported = 501;
+
+/** Clean out failed payments from the wallet
+
+  {
+    is_dry_run: <Avoid Actually Deleting Payments>
+    lnd: <Authenticated LND API Object>
+    logger: <Winston Logger Object>
+  }
+
+  @returns via cbk or Promise
+*/
+module.exports = (args, cbk) => {
+  return new Promise((resolve, reject) => {
+    return asyncAuto({
+      // Check arguments
+      validate: cbk => {
+        if (!args.lnd) {
+          return cbk([400, 'ExpectedAuthenticatedLndApiToCleanPayments']);
+        }
+
+        if (!args.logger) {
+          return cbk([400, 'ExpectedWinstonLoggerToCleanFailedPayments']);
+        }
+
+        return cbk();
+      },
+
+      // Try and determine what wallet version this is
+      getWalletVersion: ['validate', ({}, cbk) => {
+        return getWalletVersion({lnd: args.lnd}, cbk);
+      }],
+
+      // Deny known unsupported versions
+      checkWalletVersion: ['getWalletVersion', ({getWalletVersion}, cbk) => {
+        // Exit early when not actually deleting
+        if (!!args.is_dry_run) {
+          return cbk();
+        }
+
+        switch (getWalletVersion.version) {
+        case '0.11.0-beta':
+        case '0.11.1-beta':
+        case '0.12.0-beta':
+        case '0.12.1-beta':
+        case '0.13.0-beta':
+        case '0.13.1-beta':
+        case '0.13.2-beta':
+        case '0.13.3-beta':
+          return cbk([501, 'CleanFailedPaymentsUnsupportedOnThisLndVersion']);
+
+        default:
+          return cbk();
+        }
+      }],
+
+      // Delete failed payments
+      fetchAndDelete: ['checkWalletVersion', ({}, cbk) => {
+        const ids = [];
+        let token;
+
+        if (!!args.is_dry_run) {
+          args.logger.info({finding_failed_payments: true});
+        } else {
+          args.logger.info({deleting_failed_payments: true});
+        }
+
+        return asyncWhilst(
+          cbk => cbk(null, token !== false),
+          cbk => {
+            return getFailedPayments({
+              token,
+              limit: !token ? defaultPagingLimit : undefined,
+              lnd: args.lnd,
+            },
+            (err, res) => {
+              if (!!err) {
+                return cbk([503, 'UnexpectedErrorGettingPayFailures', {err}]);
+              }
+
+              // Setting token to false will signal the end of paging
+              token = res.next || false;
+
+              // Exit early when there
+              if (!res.payments.length) {
+                const date = new Date().toISOString();
+
+                args.logger.info({searching_for_failed_payments: date});
+
+                return cbk();
+              }
+
+              res.payments.forEach(({id}) => ids.push(id));
+
+              // Delete each payment
+              return asyncEachSeries(res.payments, ({id}, cbk) => {
+                // Exit early when doing a dry run
+                if (!!args.is_dry_run) {
+                  return cbk();
+                }
+
+                return deletePayment({id, lnd: args.lnd}, err => {
+                  // LND 0.13.3 and below do not support deleting payments
+                  if (!!isArray(err) && err.slice().shift() === unsupported) {
+                    return cbk([501, 'LndVersionDoesNotSupportFailDeletion']);
+                  }
+
+                  if (!!err) {
+                    return cbk([503, 'UnexpectedErrorDeletingPayment', {err}]);
+                  }
+
+                  return cbk();
+                });
+              },
+              err => {
+                if (!!err) {
+                  return cbk(err);
+                }
+
+                args.logger.info({failed_payments: ids.length});
+
+                return cbk();
+              });
+            });
+          },
+          err => {
+            if (!!err) {
+              return cbk(err);
+            }
+
+            // Exit early when this is a test
+            if (!!args.is_dry_run) {
+              return cbk(null, {total_failed_payments_found: ids.length});
+            }
+
+            return cbk(null, {total_failed_payments_deleted: ids.length});
+          }
+        );
+      }],
+    },
+    returnResult({reject, resolve, of: 'fetchAndDelete'}, cbk));
+  });
+};

package/wallets/index.js

@@ -1,5 +1,11 @@
+const cleanFailedPayments = require('./clean_failed_payments');
 const getReceivedChart = require('./get_received_chart');
 const getReport = require('./get_report');
 const unlockWallet = require('./unlock_wallet');
 
-module.exports = {getReceivedChart, getReport, unlockWallet};
+module.exports = {
+  cleanFailedPayments,
+  getReceivedChart,
+  getReport,
+  unlockWallet,
+};