balanceofsatoshis 11.22.2 → 11.24.0

package/CHANGELOG.md

@@ -1,6 +1,14 @@
 # Versions
 
-## 11.22.2
+## 11.24.0
+
+- `fees`: Add `--set-cltv-delta` to control the forwarding CLTV delta with peer
+
+## 11.23.0
+
+- `balance`:  `--detailed`: Support unconfirmed/invalid/conflicting balances
+
+## 11.22.4
 
 - `telegram`: Fix issue when moving a created invoice to a saved node
 

package/balances/conflicting_balances.js

@@ -0,0 +1,93 @@
+const {Transaction} = require('bitcoinjs-lib');
+
+const inputAsOutpoint = (txId, outputIndex) => `${txId}:${outputIndex}`;
+const {fromHex} = Transaction;
+const sumOf = arr => arr.reduce((sum, n) => sum + n, Number());
+const txIdFromHash = hash => hash.slice().reverse().toString('hex');
+const uniq = arr => Array.from(new Set(arr));
+
+/** Derive conflicted on-chain pending balances where funds are double spent or
+  multiple versions of the spend exist
+
+  {
+    transactions: [{
+      is_confirmed: <Transaction is Confirmed Bool>
+      transaction: <Raw Transaction Hex String>
+    }]
+    utxos: [{
+      confirmation_count: <UTXO Confirmation Count Number>
+      tokens: <UTXO Tokens Number>
+      transaction_id: <Outpoint Transaction Id Hex String>
+    }]
+  }
+
+  @returns
+  {
+    conflicting_pending_balance: <Conflicting Pending Balance Tokens Number>
+    invalid_pending_balance: <Invalid Pending Balance Tokens Number>
+  }
+*/
+module.exports = ({transactions, utxos}) => {
+  const conflictingUtxos = [];
+  const invalidUtxos = [];
+  const spends = {}
+
+  // Look at unconfirmed UTXOs and collect spends of outpoints
+  utxos.filter(n => !n.confirmation_count).forEach((utxo, i) => {
+    const tx = transactions.find(n => n.id === utxo.transaction_id);
+
+    // Exit early when the raw transaction is not known
+    if (!tx || !tx.transaction) {
+      return;
+    }
+
+    // Register all the inputs into the spends map
+    return fromHex(tx.transaction).ins.forEach(input => {
+      const outpoint = inputAsOutpoint(txIdFromHash(input.hash), input.index);
+
+      const existing = spends[outpoint];
+
+      // When existing UTXO spends the same input this is a conflict
+      if (!!existing) {
+        conflictingUtxos.push(i);
+      }
+
+      // Collect spends of the outpoint
+      const spending = !existing ? [i] : [].concat(existing).concat(i);
+
+      return spends[outpoint] = spending;
+    });
+  });
+
+  // Look at confirmed txs and see if any unspents spend a confirmed outpoint
+  transactions.forEach(tx => {
+    // Exit early when there is no confirmed tx
+    if (!tx.transaction || !tx.is_confirmed) {
+      return;
+    }
+
+    // Look for pending inputs that are conflicted with a confirmed tx
+    return fromHex(tx.transaction).ins.forEach(input => {
+      const outpoint = inputAsOutpoint(txIdFromHash(input.hash), input.index);
+
+      // Exit early when nothing pending spends this outpoint
+      if (!spends[outpoint]) {
+        return;
+      }
+
+      // Pending things that spend a confirmed input are invalid
+      return spends[outpoint].forEach(n => invalidUtxos.push(n));
+    });
+  });
+
+  const conflictingTokens = uniq(conflictingUtxos)
+    .filter(utxoIndex => !invalidUtxos.includes(utxoIndex))
+    .map(n => utxos[n].tokens);
+
+  const invalidTokens = uniq(invalidUtxos).map(n => utxos[n].tokens);
+
+  return {
+    conflicting_pending_balance: sumOf(conflictingTokens),
+    invalid_pending_balance: sumOf(invalidTokens),
+  };
+};

package/balances/detailed_balances.js

@@ -1,3 +1,5 @@
+const conflictingBalances = require('./conflicting_balances');
+
 const {ceil} = Math;
 const flatten = arr => [].concat(...arr);
 const inputsCounterVBytesLength = 3;
@@ -61,6 +63,8 @@ const witnessSizeCounterVByteLength = 0.25;
   @returns
   {
     closing_balance: <Balance of Tokens Moving Out Of Channels Tokens Number>
+    conflicted_pending: <Conflicting Pending Balance Tokens Number>
+    invalid_pending: <Invalid Pending Balance Tokens Number>
     offchain_balance: <Balance of Owned Tokens In Channels Tokens Number>
     offchain_pending: <Total Pending Local Balance Tokens Number>
     onchain_balance: <Balance of Transaction Outputs Number>
@@ -170,8 +174,12 @@ module.exports = ({channels, locked, pending, transactions, utxos}) => {
     .concat(transactionLockTimeVBytesLength)
     .concat(inputElements));
 
+  const conflicts = conflictingBalances({transactions, utxos});
+
   return {
     closing_balance: sumOf(closing),
+    conflicted_pending: conflicts.conflicting_pending_balance,
+    invalid_pending: conflicts.invalid_pending_balance,
     offchain_balance: channelBalance,
     offchain_pending: pendingBalance,
     onchain_balance: chainBalance,

package/balances/get_detailed_balance.js

@@ -15,6 +15,7 @@ const {fromHex} = Transaction;
 /** Get a detailed balance that categorizes balance of tokens on the node
 
   {
+    [is_confirmed]: <Only Consider Confirmed Transactions Bool>
     lnd: <Authenticated LND API Object>
   }
 
@@ -113,12 +114,26 @@ module.exports = (args, cbk) => {
           locked,
           channels: getChannels.channels,
           pending: getPending.pending_channels,
-          transactions: getTx.transactions,
-          utxos: getUtxos.utxos,
+          transactions: getTx.transactions.filter(tx => {
+            if (!!args.is_confirmed && !tx.is_confirmed) {
+              return false;
+            }
+
+            return true;
+          }),
+          utxos: getUtxos.utxos.filter(utxo => utxo => {
+            if (!!args.is_confirmed && !utxo.confirmation_count) {
+              return false;
+            }
+
+            return true;
+          }),
         });
 
         return cbk(null, {
           closing_balance: format(balances.closing_balance) || undefined,
+          conflicted_pending: format(balances.conflicted_pending) || undefined,
+          invalid_pending: format(balances.invalid_pending) || undefined,
           offchain_balance: format(balances.offchain_balance) || undefined,
           offchain_pending: format(balances.offchain_pending) || undefined,
           onchain_balance: format(balances.onchain_balance) || undefined,

package/bos

@@ -17,7 +17,7 @@ const inquirer = importLazy('inquirer');
 const lnService = importLazy('ln-service');
 const lnSync = importLazy('ln-sync');
 const paidServices = importLazy('paid-services');
-const prog = require('caporal');
+const prog = require('@alexbosworth/caporal');
 
 const commandConstants = require('./commands/constants');
 
@@ -167,11 +167,7 @@ prog
         if (!!options.detailed) {
           return balances.getDetailedBalance({
             lnd,
-            above: options.above,
-            below: options.below,
             is_confirmed: options.confirmed,
-            is_offchain_only: options.offchain,
-            is_onchain_only: options.onchain,
           },
           responses.returnObject({logger, reject, resolve}));
         }
@@ -587,6 +583,7 @@ prog
   .help('You can use INBOUND_FEE_RATE to mirror an inbound fee')
   .help('You can use FEE_RATE_OF_<PUBKEY> to reference other node rates')
   .option('--node <node_name>', 'Saved node (not peer to set fees on)')
+  .option('--set-cltv-delta <count>', 'Set the number of blocks for CLTV', INT)
   .option('--set-fee-rate <rate>', 'Fee in parts per million or use a formula')
   .option('--to <peer>', 'Peer public key or alias to set fees', REPEATABLE)
   .action((args, options, logger) => {
@@ -594,6 +591,7 @@ prog
       try {
         return routing.adjustFees({
           logger,
+          cltv_delta: options.setCltvDelta,
           fee_rate: options.setFeeRate,
           fs: {getFile: readFile},
           lnd: (await lnd.authenticatedLnd({logger, node: options.node})).lnd,
@@ -1627,6 +1625,7 @@ prog
           logger,
           ask: (n, cbk) => inquirer.prompt([n]).then(res => cbk(null, res)),
           lnd: (await lndForNode(logger, options.node)).lnd,
+          separator: () => new inquirer.Separator(),
         },
         responses.returnObject({exit, logger, reject, resolve}));
       } catch (err) {

package/package.json

@@ -10,19 +10,19 @@
     "url": "https://github.com/alexbosworth/balanceofsatoshis/issues"
   },
   "dependencies": {
-    "@alexbosworth/fiat": "1.0.0",
+    "@alexbosworth/caporal": "1.4.0",
+    "@alexbosworth/fiat": "1.0.1",
     "@alexbosworth/html2unicode": "1.1.5",
     "@alexbosworth/node-fetch": "2.6.2",
     "abort-controller": "3.0.0",
     "asciichart": "1.5.25",
-    "async": "3.2.2",
-    "asyncjs-util": "1.2.7",
+    "async": "3.2.3",
+    "asyncjs-util": "1.2.8",
     "bip66": "1.1.5",
     "bitcoinjs-lib": "6.0.1",
     "bolt01": "1.2.3",
-    "bolt03": "1.2.12",
+    "bolt03": "1.2.13",
     "bolt07": "1.8.0",
-    "caporal": "1.4.0",
     "cbor": "8.1.0",
     "colorette": "2.0.16",
     "crypto-js": "4.1.1",
@@ -33,14 +33,14 @@
     "import-lazy": "4.0.0",
     "ini": "2.0.0",
     "inquirer": "8.2.0",
-    "invoices": "2.0.2",
+    "invoices": "2.0.3",
     "ln-accounting": "5.0.5",
-    "ln-service": "53.2.0",
+    "ln-service": "53.4.0",
     "ln-sync": "3.6.1",
     "ln-telegram": "3.5.1",
     "moment": "2.29.1",
     "paid-services": "3.5.1",
-    "probing": "2.0.1",
+    "probing": "2.0.2",
     "psbt": "1.1.10",
     "qrcode-terminal": "0.12.0",
     "sanitize-filename": "1.6.3",
@@ -51,7 +51,7 @@
   "description": "Lightning balance CLI",
   "devDependencies": {
     "@alexbosworth/tap": "15.0.10",
-    "ln-docker-daemons": "2.1.0",
+    "ln-docker-daemons": "2.2.0",
     "mock-lnd": "1.4.1",
     "secp256k1": "4.0.3"
   },
@@ -69,8 +69,15 @@
   "license": "MIT",
   "main": "index.js",
   "name": "balanceofsatoshis",
-  "resolutions": {
-    "colors": "1.1.2"
+  "overrides": {
+    "caporal@1.4.0": {
+      "cli-table3@0.5.1": {
+        "colors@1.4.2": "1.4.0"
+      },
+      "prettyjson@1.2.1": {
+        "colors@1.4.2": "1.4.0"
+      }
+    }
   },
   "repository": {
     "type": "git",
@@ -81,8 +88,7 @@
     "integration-tests": "tap --branches=1 --functions=1 --lines=1 --statements=1 -t 120 test/integration/*.js",
     "postpack": "PACKAGE_VERSION=$(cat package.json | grep \\\"version\\\" | head -1 | awk -F: '{ print $2 }' | sed 's/[\",]//g' | tr -d '[[:space:]]') && git tag -s v$PACKAGE_VERSION -m v$PACKAGE_VERSION && git push github --tags",
     "postpublish": "docker buildx build --platform linux/amd64,linux/arm64,linux/arm/v7 -t alexbosworth/balanceofsatoshis --push .",
-    "preinstall": "npx npm-force-resolutions",
     "test": "tap --branches=1 --functions=1 --lines=1 --statements=1 -t 60 test/arrays/*.js test/balances/*.js test/chain/*.js test/display/*.js test/encryption/*.js test/lnd/*.js test/network/*.js test/nodes/*.js test/peers/*.js test/responses/*.js test/routing/*.js test/services/*.js test/swaps/*.js test/tags/*.js test/wallets/*.js"
   },
-  "version": "11.22.2"
+  "version": "11.24.0"
 }

package/routing/adjust_fees.js

@@ -13,13 +13,13 @@ const {getPendingChannels} = require('ln-service');
 const {gray} = require('colorette');
 const {green} = require('colorette');
 const moment = require('moment');
-const {Parser} = require('hot-formula-parser');
 const {returnResult} = require('asyncjs-util');
 const {updateChannelFee} = require('ln-sync');
 
 const {chartAliasForPeer} = require('./../display');
 const {formatFeeRate} = require('./../display');
 const {getIcons} = require('./../display');
+const parseFeeRateFormula = require('./parse_fee_rate_formula');
 
 const asTxOut = n => `${n.transaction_id}:${n.transaction_vout}`;
 const {ceil} = Math;
@@ -27,16 +27,19 @@ const flatten = arr => [].concat(...arr);
 const interval = 1000 * 60 * 2;
 const {isArray} = Array;
 const {max} = Math;
+const minCltvDelta = 18;
 const nodeMatch = /\bFEE_RATE_OF_[0-9A-F]{66}\b/gim;
 const noFee = gray('Unknown Rate');
 const pubKeyForNodeMatch = n => n.substring(12).toLowerCase();
 const shortKey = key => key.substring(0, 20);
+const sumOf = arr => arr.reduce((sum, n) => sum + n, 0);
 const times = 360;
 const uniq = arr => Array.from(new Set(arr));
 
 /** View and adjust routing fees
 
   {
+    [cltv_delta]: <Set CLTV Delta Number>
     [fee_rate]: <Fee Rate String>
     fs: {
       getFile: <Read File Contents Function> (path, cbk) => {}
@@ -72,6 +75,14 @@ module.exports = (args, cbk) => {
           return cbk([400, 'ExpectedArrayOfPeersToAdjustFeesTowards']);
         }
 
+        if (args.cltv_delta !== undefined && !args.to.length) {
+          return cbk([400, 'SettingGlobalCltvDeltaNotSupported']);
+        }
+
+        if (args.cltv_delta !== undefined && args.cltv_delta < minCltvDelta) {
+          return cbk([400, 'SettingLowCltvDeltaIsNotSupported']);
+        }
+
         if (args.fee_rate !== undefined && !args.to.length) {
           return cbk([400, 'SettingGlobalFeeRateNotSupported']);
         }
@@ -227,7 +238,8 @@ module.exports = (args, cbk) => {
         },
         cbk) =>
       {
-        if (args.fee_rate === undefined) {
+        // Exit early when not updating policy
+        if (args.cltv_delta === undefined && args.fee_rate === undefined) {
           return cbk();
         }
 
@@ -240,16 +252,6 @@ module.exports = (args, cbk) => {
             .concat(getPending.pending_channels.filter(n => !!n.is_opening))
             .filter(channel => channel.partner_public_key === key);
 
-          const inboundLiquidity = channels.reduce((sum, n) => {
-            return sum + n.remote_balance;
-          },
-          Number());
-
-          const outboundLiquidity = channels.reduce((sum, n) => {
-            return sum + n.local_balance;
-          },
-          Number());
-
           const peerPolicies = getPolicies
             .filter(n => !!n)
             .filter(n => channels.find(chan => asTxOut(chan) === asTxOut(n)))
@@ -258,51 +260,6 @@ module.exports = (args, cbk) => {
 
           const inboundFeeRate = max(...peerPolicies.map(n => n.fee_rate));
 
-          const parser = new Parser();
-
-          parser.setVariable('INBOUND', inboundLiquidity);
-          parser.setVariable('INBOUND_FEE_RATE', inboundFeeRate);
-          parser.setVariable('OUTBOUND', outboundLiquidity);
-
-          getNodeRates.forEach(({key, rate}) => parser.setVariable(key, rate));
-
-          parser.setFunction('BIPS', params => {
-            const [param] = params;
-
-            return params * 1e2;
-          });
-
-          parser.setFunction('PERCENT', params => {
-            const [param] = params;
-
-            return params * 1e4;
-          });
-
-          const parsedRate = parser.parse(args.fee_rate.toUpperCase());
-
-          switch (parsedRate.error) {
-          case '#DIV/0!':
-            return cbk([503, 'FeeRateCalculationCannotDivideByZeroFormula']);
-
-          case '#ERROR!':
-            return cbk([503, 'FailedToParseFeeRateFormula']);
-
-          case '#N/A':
-          case '#NAME?':
-            return cbk([503, 'UnrecognizedVariableOrFunctionInFeeRateFormula']);
-
-          case '#NUM':
-            return cbk([503, 'InvalidNumberFoundInFeeRateFormula']);
-
-          case '#VALUE!':
-            return cbk([503, 'UnexpectedValueTypeInFeeRateFormula']);
-
-          default:
-            break;
-          }
-
-          const feeRate = ceil(parsedRate.result);
-
           const feeRates = getFeeRates.channels.filter(rate => {
             return channels.find(n => asTxOut(n) === asTxOut(rate));
           });
@@ -317,20 +274,39 @@ module.exports = (args, cbk) => {
             .map(n => BigInt(n.base_fee_mtokens))
             .reduce((sum, fee) => fee > sum ? fee : sum, BigInt(Number()));
 
+          const {failure, rate} = parseFeeRateFormula({
+            fee_rate: args.fee_rate,
+            inbound_fee_rate: inboundFeeRate,
+            inbound_liquidity: sumOf(channels.map(n => n.remote_balance)),
+            outbound_liquidity: sumOf(channels.map(n => n.local_balance)),
+            node_rates: getNodeRates,
+          });
+
+          if (!!failure) {
+            return cbk([400, failure]);
+          }
+
           return cbk(null, channels.map(channel => {
             // Exit early when there is no known policy
             if (!currentPolicies.length) {
               return {
-                fee_rate: feeRate,
+                cltv_delta: cltvDelta,
+                fee_rate: rate,
                 transaction_id: channel.transaction_id,
                 transaction_vout: channel.transaction_vout,
               };
             }
 
+            // Only the highest CLTV delta across all peer channels applies
+            const cltvDelta = max(...currentPolicies.map(n => n.cltv_delta));
+
+            // Only the highest fee rate across all peer channels applies
+            const maxFeeRate = max(...currentPolicies.map(n => n.fee_rate));
+
             return {
               base_fee_mtokens: baseFeeMillitokens.toString(),
-              cltv_delta: max(...currentPolicies.map(n => n.cltv_delta)),
-              fee_rate: feeRate,
+              cltv_delta: args.cltv_delta || cltvDelta,
+              fee_rate: rate !== undefined ? rate : maxFeeRate,
               transaction_id: channel.transaction_id,
               transaction_vout: channel.transaction_vout,
             };

package/routing/parse_fee_rate_formula.js

@@ -0,0 +1,58 @@
+const {Parser} = require('hot-formula-parser');
+
+const bipsAsPpm = bips => bips * 1e2;
+const {ceil} = Math;
+const percentAsPpm = percent => percent * 1e4;
+
+/** Parse a fee rate formula
+
+  {
+    [fee_rate]: <PPM Fee Rate String>
+    inbound_fee_rate: <Inbound PPM Fee Rate Number>
+    inbound_liquidity: <Inbound Tokens Number>
+    outbound_liquidity: <Outbound Tokens Number>
+    node_rates: [{
+      key: <Node Key String>
+      rate: <Node PPM Rate Number>
+    }]
+  }
+
+  @returns
+  {
+    [failure]: <Failure to Parse String>
+    [rate]: <PPM Fee Rate Number>
+  }
+*/
+module.exports = args => {
+  if (args.fee_rate === undefined) {
+    return {};
+  }
+
+  const parser = new Parser();
+
+  parser.setFunction('BIPS', params => bipsAsPpm(params.slice().pop()));
+  parser.setFunction('PERCENT', params => percentAsPpm(params.slice().pop()));
+  parser.setVariable('INBOUND', args.inbound_liquidity);
+  parser.setVariable('INBOUND_FEE_RATE', args.inbound_fee_rate);
+  parser.setVariable('OUTBOUND', args.outbound_liquidity);
+
+  args.node_rates.forEach(({key, rate}) => parser.setVariable(key, rate));
+
+  const parsedRate = parser.parse(args.fee_rate.toUpperCase());
+
+  switch (parsedRate.error) {
+  case null:
+    break;
+
+  case '#DIV/0!':
+    return {failure: 'FeeRateCalculationCannotDivideByZeroFormula'};
+
+  case '#ERROR!':
+    return {failure: 'FailedToParseFeeRateFormula'};
+
+  default:
+    return {failure: 'UnrecognizedVariableOrFunctionInFeeRateFormula'};
+  }
+
+  return {rate: ceil(parsedRate.result)};
+};

package/test/balances/test_detailed_balance.js

@@ -93,6 +93,8 @@ const tests = [
     description: 'Balance totals are calculated',
     expected: {
       closing_balance: 1,
+      conflicted_pending: 0,
+      invalid_pending: 0,
       offchain_balance: 14,
       offchain_pending: 35,
       onchain_balance: 4,
@@ -104,6 +106,8 @@ const tests = [
     description: 'Balance totals are calculated when there are no funds',
     expected: {
       closing_balance: 0,
+      conflicted_pending: 0,
+      invalid_pending: 0,
       offchain_balance: 0,
       offchain_pending: 0,
       onchain_balance: 0,

package/test/balances/test_get_accounting_report.js

@@ -98,6 +98,10 @@ const tests = [
           'income',
         ],
       ],
+      rows_summary: [
+        ['Total', 'Asset', 'Report Date', 'Total Fiat'],
+        [1, 'BTC', '', '0.00'],
+      ],
     },
   },
   {
@@ -114,6 +118,10 @@ tests.forEach(({args, description, error, expected}) => {
     } else {
       const res = await getAccountingReport(args);
 
+      if (!args.is_csv) {
+        res.rows_summary[1][2] = '';
+      }
+
       strictSame(res, expected, 'Got expected response');
     }
 

package/test/routing/test_parse_fee_rate_formula.js

@@ -0,0 +1,67 @@
+const {test} = require('@alexbosworth/tap');
+
+const method = require('./../../routing/parse_fee_rate_formula');
+
+const makeArgs = overrides => {
+  const args = {
+    fee_rate: '1',
+    inbound_fee_rate: 1,
+    inbound_liquidity: 1,
+    outbound_liquidity: 1,
+    node_rates: [{key: 'key', rate: 1}],
+  };
+
+  Object.keys(overrides).forEach(k => args[k] = overrides[k]);
+
+  return args;
+};
+
+const tests = [
+  {
+    args: makeArgs({fee_rate: undefined}),
+    description: 'Fee rate is optional',
+    expected: {},
+  },
+  {
+    args: makeArgs({}),
+    description: 'Fee rate formula is parsed',
+    expected: {rate: 1},
+  },
+  {
+    args: makeArgs({fee_rate: 'BIPS(25)'}),
+    description: 'BIPs function is parsed',
+    expected: {rate: 2500},
+  },
+  {
+    args: makeArgs({fee_rate: 'PERCENT(0.25)'}),
+    description: 'PERCENT function is parsed',
+    expected: {rate: 2500},
+  },
+  {
+    args: makeArgs({fee_rate: '1/0'}),
+    description: 'Cannot divide by zero',
+    expected: {failure: 'FeeRateCalculationCannotDivideByZeroFormula'},
+  },
+  {
+    args: makeArgs({fee_rate: '/'}),
+    description: 'Formula must be valid',
+    expected: {failure: 'FailedToParseFeeRateFormula'},
+  },
+  {
+    args: makeArgs({fee_rate: 'fee_rate'}),
+    description: 'Formula must be valid',
+    expected: {failure: 'UnrecognizedVariableOrFunctionInFeeRateFormula'},
+  },
+];
+
+tests.forEach(({args, description, error, expected}) => {
+  return test(description, async ({end, strictSame, throws}) => {
+    if (!!error) {
+      throws(() => method(args), new Error(error), 'Got expected error');
+    } else {
+      strictSame(method(args), expected, 'Got expected result');
+    }
+
+    return end();
+  });
+});