balanceofsatoshis 25.6.2 → 26.0.0

package/CHANGELOG.md

@@ -1,5 +1,43 @@
 # Versions
 
+## 26.0.0
+
+- `swap-api-key`: Add `--max-fee` to limit what is paid for an API key
+- `swap-in`: Use the shared address swap in protocol of the swap service,
+    swapping in a deposit of the exact amount once it has 3 confirmations
+- `swap-in`: Add `--max-fee-rate` to limit the swap fee in parts per million
+- `swap-in`, `swap-out`: Add `--max-api-key-fee` to limit what is paid for a
+    swap service API key
+- `swap-out`: Use the Taproot MuSig2 swap out protocol of the swap service,
+    like `swap` does
+- `swap-out`: Add `--out` to select the peer or tag to increase inbound
+    liquidity from, with `--out-filter` formulas like `rebalance`
+
+### Breaking Changes
+
+- `increase-inbound-liquidity`: Rename command to `swap-out`
+- `swap-in`: `--recovery` takes a shared address secret, not old recovery codes
+- `swap-in`: `--refund-address` with `--recovery` refunds expired deposits
+    and unpaid swaps that timed out
+- `swap-in`: Remove `--test-refund`
+- `swap-in`: Use 25.7.0 to refund swaps in started with earlier versions
+- `swap-out`: `--recovery` takes the restart code of the new swap protocol,
+    which `swap` also accepts, not the previous recovery code
+- `swap-out`: Remove `--max-hours`, `--show-raw-recovery`, `--spend-address`
+    and `--spend-amount`
+- `swap-out`: Rename `--with` to `--out`
+
+## 25.7.0
+
+- `accounting`: Add `lightning-fees` category to report the routing fees paid
+    on payments
+- `accounting`: Keep the payments of recent blocks in `chain_blocks.json`, so
+    that later runs only fetch the blocks that are new
+- `accounting`: Pass `--node` more than once to get one report of several
+    nodes, with their records together in order of date
+- `fund`: Add `--fee-rate-precise` to set a fee rate per 1000 vbytes
+- `fund`: `MAX` amounts send as much as possible without overpaying the fee
+
 ## 25.6.2
 
 - `telegram`: Fix invoices settled before the bot started being reported as new

package/README.md

@@ -123,9 +123,6 @@ bos inbound-liquidity
 # Enforce rules on inbound channels
 bos inbound-channel-rules
 
-# Increase inbound liquidity to the node
-bos increase-inbound-liquidity
-
 # Increase the outbound liquidity of the node
 bos increase-outbound-liquidity
 
@@ -171,6 +168,12 @@ bos server
 # Stream a payment to a lnurl or lightning address in a series of payments
 bos stream "amount" --lnurl "lnurl/lightning_address"
 
+# Trade on-chain funds for off-chain funds to increase outbound liquidity
+bos swap-in
+
+# Trade off-chain funds for on-chain funds to increase inbound liquidity
+bos swap-out
+
 # Connect up to a Telegram bot
 bos telegram
 
@@ -305,6 +308,9 @@ You can also add a JSON file to your .bos directory: config.json, add
 
 `bos` keeps its files in `~/.bos`, or in `BOS_DATA_PATH` when that is set:
 
+- `chain_blocks.json`: the payments in the last few days of blocks that
+    `bos accounting` derives prices from, kept so that later runs only fetch
+    the blocks that are new. The file can be deleted
 - `chain_prices.json`: USD prices of BTC that `bos accounting` derived from
     chain data, kept so that a day is only derived once. Prices are derived
     this way when no `--rate-provider` is given, and the prices of the days
@@ -614,6 +620,7 @@ Formula amounts are supported in the following commands:
 - `probe`
 - `rebalance`
 - `send`
+- `swap-out`
 
 When passing an amount you can pass a formula expression, and the following variables are
 defined:
@@ -673,6 +680,25 @@ Example:
 bos send <key> --amount "1*usd"
 ```
 
+#### `swap-out`
+
+Swap out defines variables for `--out-filter`:
+
+- `capacity`: The total capacity with the peer
+- `heights`: The set of heights of the channels with the peer
+- `inbound_base_fee`: The base fee the peer is charging
+- `inbound_fee_rate`: The fee rate the peer is charging
+- `inbound_liquidity`: The inbound liquidity with the peer
+- `outbound_liquidity`: The outbound liquidity with the peer
+- `pending_payments`: The number of pending payments
+
+Example:
+
+```shell
+// Swap out 1,000,000 with a peer of the tag that has under 2,000,000 inbound
+bos swap-out --amount "1*m" --out "tag" --out-filter "inbound_liquidity < 2*m"
+```
+
 #### `transfer`
 
 Transfer variables:

package/balances/accounting_categories.json

@@ -4,5 +4,6 @@
   "chain-sends": "chain_sends",
   "forwards": "forwards",
   "invoices": "invoices",
+  "lightning-fees": "lightning_fees",
   "payments": "payments"
 }

package/balances/chain_price_range.js

@@ -0,0 +1,30 @@
+const {isFinite} = Number;
+const {round} = Math;
+const {sqrt} = Math;
+
+/** Get the range of prices that the round dollar payments of a day point to
+
+  {
+    [dispersion]: <Peak Width Beyond Smoothing As Price Fraction Number>
+    price: <Price String>
+  }
+
+  @returns
+  {
+    [range]: <Low Price to High Price String>
+  }
+*/
+module.exports = ({dispersion, price}) => {
+  // Exit early when there is no width to describe
+  if (!isFinite(dispersion)) {
+    return {};
+  }
+
+  // The peak reaches the square root of the dispersion factor either side
+  const reach = sqrt(1 + dispersion);
+
+  const high = round(Number(price) * reach);
+  const low = round(Number(price) / reach);
+
+  return {range: `${low} to ${high}`};
+};

package/balances/chain_prices_logger.js

@@ -0,0 +1,78 @@
+const chainPriceRange = require('./chain_price_range');
+const {chainPriceFlagNotes} = require('./constants');
+
+const {floor} = Math;
+const {keys} = Object;
+const msPerDay = 1000 * 60 * 60 * 24;
+
+/** Make a logger that describes deriving prices from chain data
+
+  {
+    [logger]: <Winston Logger Object>
+  }
+
+  @returns
+  {
+    [logger]: {
+      error: <Error Function>
+      info: <Info Function>
+    }
+  }
+*/
+module.exports = ({logger}) => {
+  // Exit early when there is no logger to describe progress with
+  if (!logger) {
+    return {};
+  }
+
+  return {
+    logger: {
+      error: err => logger.error(err),
+      info: line => {
+        // Exit early describing fetching blocks as calculating prices
+        if (!!line && !!line.fetching_blocks) {
+          const {blocks, date} = line.fetching_blocks;
+
+          const days = floor((Date.now() - Date.parse(date)) / msPerDay);
+
+          const isToday = days === 0;
+
+          return logger.info({
+            calculating_prices_from_blocks: {
+              blocks,
+              day: date,
+              days_out_of_date: !isNaN(days) && !isToday ? days : undefined,
+            },
+          });
+        }
+
+        // Exit early when the line is not a price derived from chain data
+        if (!line || !line.chain_price || !line.chain_price.price) {
+          return logger.info(line);
+        }
+
+        const price = line.chain_price;
+
+        const flags = keys(chainPriceFlagNotes).filter(flag => !!price[flag]);
+
+        const note = flags.map(flag => chainPriceFlagNotes[flag]).join(', ');
+
+        // Only a price from a full day of blocks has the measures behind it
+        const isFullDay = price.outputs !== undefined;
+
+        // Keys are in reading order, the price first and then its evidence
+        return logger.info({
+          chain_price: {
+            date: price.date,
+            price: price.price,
+            price_range: isFullDay ? chainPriceRange(price).range : undefined,
+            blocks: isFullDay ? price.blocks : undefined,
+            payment_outputs: isFullDay ? price.outputs : undefined,
+            round_dollar_payments: isFullDay ? price.payments : undefined,
+            note: note || undefined,
+          },
+        });
+      },
+    },
+  };
+};

package/balances/constants.json

@@ -1,6 +1,14 @@
 {
+  "chainBlocksFile": "chain_blocks.json",
+  "chainPriceFlagNotes": {
+    "is_provisional": "provisional, the day is not over yet",
+    "is_sample": "rough estimate from a sample of the blocks",
+    "is_unsettled": "the estimate was still shifting when refining stopped",
+    "is_widened": "found outside the expected price range"
+  },
   "chainPricesFile": "chain_prices.json",
   "coinbaseProvider": "coinbase",
+  "dayFormat": "YYYY-MM-DD",
   "defaultCurrency": "BTC",
   "defaultFiat": "USD",
   "failedRequestErrors": ["UnexpectedErrorGettingCoinbaseCandles"],
@@ -38,5 +46,14 @@
     "FailedToReplacePricesFile",
     "FailedToWritePricesFile",
     "UnsupportedPricesFileVersion"
+  ],
+  "summaryHeadings": [
+    "Total",
+    "Asset",
+    "Report Date",
+    "Total Fiat",
+    "Category",
+    "Start Date",
+    "End Date"
   ]
 }

package/balances/get_accounting_report.js

@@ -1,12 +1,20 @@
 const asyncAuto = require('async/auto');
+const asyncify = require('async/asyncify');
+const asyncMap = require('async/map');
+const asyncMapSeries = require('async/mapSeries');
 const {chainRates} = require('@alexbosworth/fiat');
 const {getAccountingReport} = require('ln-accounting');
 const {getNetwork} = require('ln-sync');
+const {harmonize} = require('ln-accounting');
+const moment = require('moment');
 const {returnResult} = require('asyncjs-util');
 
 const categories = require('./accounting_categories');
+const {chainBlocksFile} = require('./constants');
 const {chainPricesFile} = require('./constants');
+const chainPricesLogger = require('./chain_prices_logger');
 const {coinbaseProvider} = require('./constants');
+const {dayFormat} = require('./constants');
 const {defaultCurrency} = require('./constants');
 const {defaultFiat} = require('./constants');
 const {fsMethods} = require('./constants');
@@ -14,18 +22,20 @@ const getRecordRate = require('./get_record_rate');
 const guardChainRate = require('./guard_chain_rate');
 const {homePath} = require('./../storage');
 const {mainnet} = require('./constants');
-const {monthNumbers} = require('./constants');
-const {monthOffset} = require('./constants');
-const {notFoundIndex} = require('./constants');
 const rangeForDate = require('./range_for_date');
+const {sortBy} = require('./../arrays');
+const {summaryHeadings} = require('./constants');
 const tableRowsFromCsv = require('./table_rows_from_csv');
 
 const assetType = 'BTC';
-const currentDate = new Date().toISOString();
+const dayOf = n => moment.utc(n).format(dayFormat);
 const empty = '';
+const flatten = arr => [].concat(...arr);
+const {isArray} = Array;
+const {keys} = Object;
+const lastDayBefore = n => moment.utc(n).subtract(1, 'ms').format(dayFormat);
 const round = n => parseFloat(n).toFixed(2);
 const sumOf = arr => arr.reduce((sum, n) => sum + n, 0);
-const summaryHeadings = ['Total', 'Asset', 'Report Date', 'Total Fiat'];
 
 /** Get an accounting report
 
@@ -42,10 +52,9 @@ const summaryHeadings = ['Total', 'Asset', 'Report Date', 'Total Fiat'];
     }
     [is_csv]: <Return CSV Output Bool>
     [is_fiat_disabled]: <Omit Fiat Conversion Bool>
-    lnd: <Authenticated LND API Object>
+    lnds: [<Authenticated LND API Object>]
     [logger]: <Winston Logger Object>
     [month]: <Month for Report String>
-    [node]: <Node Name String>
     [rate_provider]: <Rate Provider String>
     request: <Request Function>
     [year]: <Year for Report String>
@@ -65,7 +74,9 @@ module.exports = (args, cbk) => {
         const {fs} = args;
 
         if (!args.category || !categories[args.category]) {
-          return cbk([400, 'ExpectedKnownAccountingRecordsCategory']);
+          return cbk([400, 'ExpectedKnownAccountingRecordsCategory', {
+            categories: keys(categories),
+          }]);
         }
 
         if (!!args.date && !args.month) {
@@ -76,8 +87,8 @@ module.exports = (args, cbk) => {
           return cbk([400, 'ExpectedFileSystemMethodsToGetAccountingReport']);
         }
 
-        if (!args.lnd) {
-          return cbk([400, 'ExpectedAuthenticatedLndToGetAccountingReport']);
+        if (!isArray(args.lnds) || !args.lnds.length) {
+          return cbk([400, 'ExpectedAuthenticatedLndsToGetAccountingReport']);
         }
 
         if (!args.request) {
@@ -88,34 +99,37 @@ module.exports = (args, cbk) => {
       },
 
       // Get date range
-      dateRange: ['validate', ({}, cbk) => {
-        const range = (() => {
-          try {
-            return {
-              value: rangeForDate({
-                date: args.date,
-                month: args.month,
-                year: args.year,
-              }),
-            };
-          } catch (err) {
-            return {err};
-          }
-        })();
-
-        // Exit early when the date range cannot be determined
-        if (!!range.err) {
-          return cbk([400, range.err.message]);
+      dateRange: ['validate', asyncify(() => {
+        try {
+          return rangeForDate({
+            date: args.date,
+            month: args.month,
+            year: args.year,
+          });
+        } catch (err) {
+          throw [400, err.message];
         }
+      })],
 
-        return cbk(null, range.value);
+      // Get the network names of the nodes
+      getNetworks: ['validate', ({}, cbk) => {
+        return asyncMap(args.lnds, (lnd, cbk) => getNetwork({lnd}, cbk), cbk);
       }],
 
-      // Get the network name
-      getNetwork: ['validate', ({}, cbk) => getNetwork({lnd: args.lnd}, cbk)],
+      // The network that the nodes are on
+      network: ['getNetworks', ({getNetworks}, cbk) => {
+        const [{network}] = getNetworks;
+
+        // Exit early when the nodes are not all on the same network
+        if (getNetworks.some(n => n.network !== network)) {
+          return cbk([400, 'ExpectedSameNetworkNodesForAccountingReport']);
+        }
+
+        return cbk(null, network);
+      }],
 
       // Make the method that gets chain prices, used without a rate provider
-      chain: ['getNetwork', ({getNetwork}, cbk) => {
+      chain: ['network', ({network}, cbk) => {
         // Exit early when fiat is disabled or a rate provider is specified
         if (!!args.is_fiat_disabled || !!args.rate_provider) {
           return cbk();
@@ -123,17 +137,23 @@ module.exports = (args, cbk) => {
 
         // Exit early when there are no chain prices for the network or no
         // file system to keep them in
-        if (!args.fs || getNetwork.network !== mainnet) {
+        if (!args.fs || network !== mainnet) {
           return cbk();
         }
 
         // Progress is only logged when it cannot end up in CSV output
-        const logger = !args.is_csv ? args.logger : undefined;
+        const {logger} = chainPricesLogger({
+          logger: !args.is_csv ? args.logger : undefined,
+        });
+
+        // Chain prices are derived through the first node
+        const [lnd] = args.lnds;
 
         const {getChainRate} = chainRates({
+          lnd,
           logger,
+          blocks_path: homePath({file: chainBlocksFile}).path,
           fs: args.fs,
-          lnd: args.lnd,
           path: homePath({file: chainPricesFile}).path,
         });
 
@@ -165,51 +185,63 @@ module.exports = (args, cbk) => {
         });
       }],
 
-      // Get accounting info
+      // Get the accounting records of each node, one node after another
       getAccounting: [
         'dateRange',
-        'getNetwork',
+        'network',
         'rate',
-        ({dateRange, getNetwork, rate}, cbk) =>
+        ({dateRange, network, rate}, cbk) =>
       {
-        return getAccountingReport({
-          after: dateRange.after,
-          before: dateRange.before,
-          category: categories[args.category],
-          currency: args.currency || defaultCurrency,
-          fiat: !args.is_fiat_disabled ? (args.fiat || defaultFiat) : null,
-          lnd: args.lnd,
-          network: getNetwork.network,
-          rate: !!rate ? rate.rate : undefined,
-          rate_provider: !rate ? args.rate_provider : undefined,
-          request: args.request,
+        return asyncMapSeries(args.lnds, (lnd, cbk) => {
+          return getAccountingReport({
+            lnd,
+            network,
+            after: dateRange.after,
+            before: dateRange.before,
+            category: categories[args.category],
+            currency: args.currency || defaultCurrency,
+            fiat: !args.is_fiat_disabled ? (args.fiat || defaultFiat) : null,
+            rate: !!rate ? rate.rate : undefined,
+            rate_provider: !rate ? args.rate_provider : undefined,
+            request: args.request,
+          },
+          cbk);
         },
         cbk);
       }],
 
-      // Convert the accounting CSV into rows for table display output
-      accounting: ['getAccounting', ({getAccounting}, cbk) => {
-        const csvType = `${categories[args.category]}_csv`;
+      // The records of all of the nodes in the category, in order of date
+      records: ['getAccounting', ({getAccounting}, cbk) => {
+        const category = categories[args.category];
+
+        const records = flatten(getAccounting.map(report => report[category]));
+
+        const {sorted} = sortBy({array: records, attribute: 'created_at'});
+
+        return cbk(null, sorted);
+      }],
+
+      // Convert the records into a CSV, or into rows for table display output
+      accounting: ['records', ({records}, cbk) => {
+        const {csv} = harmonize({records});
 
         // Exit early when a CSV dump is requested
         if (!!args.is_csv) {
-          return cbk(null, getAccounting[csvType]);
+          return cbk(null, csv);
         }
 
-        return tableRowsFromCsv({csv: getAccounting[csvType]}, cbk);
+        return tableRowsFromCsv({csv}, cbk);
       }],
 
       // Calculate total amounts
-      total: ['getAccounting', ({getAccounting}, cbk) => {
+      total: ['records', ({records}, cbk) => {
         // Exit early when a CSV dump is requested
         if (!!args.is_csv) {
           return cbk();
         }
 
-        const rows = getAccounting[categories[args.category]];
-
         // Token values are represented as amounts
-        const tokens = sumOf(rows.map(n => n.amount));
+        const tokens = sumOf(records.map(n => n.amount));
 
         // Exit early when there is no fiat data
         if (!!args.is_fiat_disabled) {
@@ -217,13 +249,18 @@ module.exports = (args, cbk) => {
         }
 
         // Fiat values are represented as fiat amounts
-        const fiat = round(sumOf(rows.map(n => n.fiat_amount)));
+        const fiat = round(sumOf(records.map(n => n.fiat_amount)));
 
         return cbk(null, {fiat, tokens});
       }],
 
       // Clean rows for display if necessary
-      report: ['accounting', 'total',({accounting, total}, cbk) => {
+      report: [
+        'accounting',
+        'dateRange',
+        'total',
+        ({accounting, dateRange, total}, cbk) =>
+      {
         // Exit early when there is no cleaning necessary
         if (!!args.is_csv) {
           return cbk(null, accounting);
@@ -249,7 +286,15 @@ module.exports = (args, cbk) => {
 
         const summary = [
           summaryHeadings,
-          [total.tokens, assetType, currentDate, total.fiat],
+          [
+            total.tokens,
+            assetType,
+            new Date().toISOString(),
+            total.fiat,
+            args.category,
+            !dateRange.after ? empty : dayOf(dateRange.after),
+            !dateRange.before ? empty : lastDayBefore(dateRange.before),
+          ],
         ];
 
         return cbk(null, {rows, rows_summary: summary});

package/balances/range_for_date.js

@@ -4,7 +4,9 @@ const {monthNumbers} = require('./constants');
 const {monthOffset} = require('./constants');
 const {notFoundIndex} = require('./constants');
 
-/** Get a before and after range 
+const isDay = n => /^\d+$/.test(n) && Number(n) > 0;
+
+/** Get a before and after range
 
   {
     [date]: <Day of Month String>
@@ -17,8 +19,8 @@ const {notFoundIndex} = require('./constants');
 
   @returns
   {
-    [after]: <After ISO 8601 Date String>
-    [before]: <Before ISO 8601 Date String>
+    [after]: <Records Created On or After ISO 8601 Date String>
+    [before]: <Records Created Before ISO 8601 Date String>
   }
 */
 module.exports = ({date, month, year}) => {
@@ -32,12 +34,6 @@ module.exports = ({date, month, year}) => {
     after.year(year);
   }
 
-  try {
-    after.toISOString();
-  } catch (err) {
-    throw new Error('UnrecognizedFormatForAccountingYear');
-  }
-
   const end = after.clone();
 
   if (!!month && monthNumbers.indexOf(month) !== notFoundIndex) {
@@ -46,8 +42,12 @@ module.exports = ({date, month, year}) => {
     [after, end].forEach(n => n.month(month));
   }
 
+  if (!!date && (!isDay(date) || Number(date) > after.daysInMonth())) {
+    throw new Error('ExpectedDayInMonthForAccountingDate');
+  }
+
   if (!!date) {
-    [after, end].forEach(n => n.date(date));
+    [after, end].forEach(n => n.date(Number(date)));
   }
 
   if (!!date) {
@@ -58,13 +58,5 @@ module.exports = ({date, month, year}) => {
     end.add([after].length, 'years');
   }
 
-  try {
-    after.toISOString();
-  } catch (err) {
-    throw new Error('UnrecognizedFormatForAccountingMonth');
-  }
-
-  after.subtract([after].length, 'millisecond');
-
   return {after: after.toISOString(), before: end.toISOString()};
 };