balanceofsatoshis 25.3.1 → 25.5.0

package/CHANGELOG.md

@@ -1,5 +1,15 @@
 # Versions
 
+## 25.5.0
+
+- `probe`: Add `--ignore-expiry` to probe expired payment requests
+
+## 25.4.0
+
+- `accounting`: After 2021-10-19 derive USD prices from chain payments data
+- `accounting`: Add `--rate-provider coinbase`
+- `telegram`: Reduce telegram linkification in messages
+
 ## 25.3.1
 
 - `closed`: Avoid using external chain data lookups to derive resolution info

package/README.md

@@ -7,12 +7,12 @@ Commands for working with LND balances.
 
 Supported LND versions:
 
-- v0.21.0-beta to v0.21.4-beta
-- v0.20.0-beta to v0.20.5-beta
+- v0.21.0-beta to v0.21.5-beta
+- v0.20.0-beta to v0.20.6-beta
 
 ## Install
 
-- Requires an [installation of Node v20+][nodejs-install-guide]
+- Requires an [installation of Node.js v22+][nodejs-install-guide]
 
 ```shell
 npm install -g balanceofsatoshis
@@ -298,6 +298,20 @@ If that is set, it will use that node if no node is specified.
 You can also add a JSON file to your .bos directory: config.json, add
 `{"default_saved_node": "nodename"}` to set the default via a file instead
 
+## Files in `~/.bos`
+
+`bos` keeps its files in `~/.bos`, or in `BOS_DATA_PATH` when that is set:
+
+- `chain_prices.json`: USD prices of BTC that `bos accounting` derived from
+    chain data, kept so that a day is only derived once. Prices are derived
+    this way when no `--rate-provider` is given, and the prices of the days
+    up to the release of `bos` come with it rather than being written here.
+    The file can be deleted, prices are then derived again when needed
+- `config.json`: settings you add, like a default saved node
+- `tags.json`: the tags made with `bos tags`
+- `telegram_bot_api_key`: the bot API key that `bos telegram` asked for
+- `YOUR_NODE_NAME/credentials.json`: the credentials of a saved node
+
 ## Linux Fu
 
 Some commands are designed to return outputs that can be piped or used in other CLI programs.

package/balances/constants.json

@@ -1,6 +1,11 @@
 {
+  "chainPricesFile": "chain_prices.json",
+  "coinbaseProvider": "coinbase",
   "defaultCurrency": "BTC",
   "defaultFiat": "USD",
+  "failedRequestErrors": ["UnexpectedErrorGettingCoinbaseCandles"],
+  "fsMethods": ["getFile", "makeDirectory", "renameFile", "writeFile"],
+  "mainnet": "btc",
   "notFoundIndex": -1,
   "monthNumbers": [
     "01",
@@ -25,5 +30,13 @@
     "11",
     "12"
   ],
-  "monthOffset": 1
+  "monthOffset": 1,
+  "pricesFileErrors": [
+    "ExpectedPricesObjectInPricesFile",
+    "FailedToParsePricesFile",
+    "FailedToReadPricesFile",
+    "FailedToReplacePricesFile",
+    "FailedToWritePricesFile",
+    "UnsupportedPricesFileVersion"
+  ]
 }

package/balances/get_accounting_report.js

@@ -1,11 +1,19 @@
 const asyncAuto = require('async/auto');
+const {chainRates} = require('@alexbosworth/fiat');
 const {getAccountingReport} = require('ln-accounting');
 const {getNetwork} = require('ln-sync');
 const {returnResult} = require('asyncjs-util');
 
 const categories = require('./accounting_categories');
+const {chainPricesFile} = require('./constants');
+const {coinbaseProvider} = require('./constants');
 const {defaultCurrency} = require('./constants');
 const {defaultFiat} = require('./constants');
+const {fsMethods} = require('./constants');
+const getRecordRate = require('./get_record_rate');
+const guardChainRate = require('./guard_chain_rate');
+const {homePath} = require('./../storage');
+const {mainnet} = require('./constants');
 const {monthNumbers} = require('./constants');
 const {monthOffset} = require('./constants');
 const {notFoundIndex} = require('./constants');
@@ -26,9 +34,16 @@ const summaryHeadings = ['Total', 'Asset', 'Report Date', 'Total Fiat'];
     [currency]: <Currency Label String>
     [date]: <Day of Month String>
     [fiat]: <Fiat Type String>
+    [fs]: {
+      getFile: <Read File Contents Function> (path, cbk) => {}
+      makeDirectory: <Make Directory Function> (path, cbk) => {}
+      renameFile: <Rename File Function> (from, to, cbk) => {}
+      writeFile: <Write File Contents Function> (path, contents, cbk) => {}
+    }
     [is_csv]: <Return CSV Output Bool>
     [is_fiat_disabled]: <Omit Fiat Conversion Bool>
     lnd: <Authenticated LND API Object>
+    [logger]: <Winston Logger Object>
     [month]: <Month for Report String>
     [node]: <Node Name String>
     [rate_provider]: <Rate Provider String>
@@ -47,6 +62,8 @@ module.exports = (args, cbk) => {
     return asyncAuto({
       // Validate
       validate: cbk => {
+        const {fs} = args;
+
         if (!args.category || !categories[args.category]) {
           return cbk([400, 'ExpectedKnownAccountingRecordsCategory']);
         }
@@ -55,6 +72,10 @@ module.exports = (args, cbk) => {
           return cbk([400, 'ExpectedMonthForDateToGetAccountingReport']);
         }
 
+        if (!!fs && !fsMethods.every(n => typeof fs[n] === 'function')) {
+          return cbk([400, 'ExpectedFileSystemMethodsToGetAccountingReport']);
+        }
+
         if (!args.lnd) {
           return cbk([400, 'ExpectedAuthenticatedLndToGetAccountingReport']);
         }
@@ -68,25 +89,88 @@ module.exports = (args, cbk) => {
 
       // Get date range
       dateRange: ['validate', ({}, cbk) => {
-        try {
-          return cbk(null, rangeForDate({
-            date: args.date,
-            month: args.month,
-            year: args.year,
-          }));
-        } catch (err) {
-          return cbk([400, err.message]);
+        const range = (() => {
+          try {
+            return {
+              value: rangeForDate({
+                date: args.date,
+                month: args.month,
+                year: args.year,
+              }),
+            };
+          } catch (err) {
+            return {err};
+          }
+        })();
+
+        // Exit early when the date range cannot be determined
+        if (!!range.err) {
+          return cbk([400, range.err.message]);
         }
+
+        return cbk(null, range.value);
       }],
 
       // Get the network name
       getNetwork: ['validate', ({}, cbk) => getNetwork({lnd: args.lnd}, cbk)],
 
+      // Make the method that gets chain prices, used without a rate provider
+      chain: ['getNetwork', ({getNetwork}, cbk) => {
+        // Exit early when fiat is disabled or a rate provider is specified
+        if (!!args.is_fiat_disabled || !!args.rate_provider) {
+          return cbk();
+        }
+
+        // Exit early when there are no chain prices for the network or no
+        // file system to keep them in
+        if (!args.fs || getNetwork.network !== mainnet) {
+          return cbk();
+        }
+
+        // Progress is only logged when it cannot end up in CSV output
+        const logger = !args.is_csv ? args.logger : undefined;
+
+        const {getChainRate} = chainRates({
+          logger,
+          fs: args.fs,
+          lnd: args.lnd,
+          path: homePath({file: chainPricesFile}).path,
+        });
+
+        return cbk(null, guardChainRate({getChainRate}));
+      }],
+
+      // Without a rate provider, records are valued with chain prices, and
+      // at their hour on Coinbase Exchange when chain prices cannot value them
+      rate: ['chain', ({chain}, cbk) => {
+        // Exit early when fiat is disabled
+        if (!!args.is_fiat_disabled) {
+          return cbk();
+        }
+
+        // Exit early when a rate provider other than Coinbase is specified
+        if (!!args.rate_provider && args.rate_provider !== coinbaseProvider) {
+          return cbk();
+        }
+
+        return cbk(null, {
+          rate: (record, cbk) => getRecordRate({
+            currency: record.currency,
+            date: record.date,
+            fiat: record.fiat,
+            get_chain_rate: !!chain ? chain.getChainRate : undefined,
+            request: args.request,
+          },
+          cbk),
+        });
+      }],
+
       // Get accounting info
       getAccounting: [
         'dateRange',
         'getNetwork',
-        ({dateRange, getNetwork}, cbk) =>
+        'rate',
+        ({dateRange, getNetwork, rate}, cbk) =>
       {
         return getAccountingReport({
           after: dateRange.after,
@@ -96,7 +180,8 @@ module.exports = (args, cbk) => {
           fiat: !args.is_fiat_disabled ? (args.fiat || defaultFiat) : null,
           lnd: args.lnd,
           network: getNetwork.network,
-          rate_provider: args.rate_provider || undefined,
+          rate: !!rate ? rate.rate : undefined,
+          rate_provider: !rate ? args.rate_provider : undefined,
           request: args.request,
         },
         cbk);

package/balances/get_historic_rate.js

@@ -0,0 +1,75 @@
+const asyncAuto = require('async/auto');
+const asyncRetry = require('async/retry');
+const {getPrices} = require('@alexbosworth/fiat');
+const {returnResult} = require('asyncjs-util');
+
+const {failedRequestErrors} = require('./constants');
+
+const isFailedRequest = err => !!err && failedRequestErrors.includes(err[1]);
+const retryDelaysMs = [1000, 2000];
+const supportedCurrency = 'BTC';
+
+/** Get the fiat rate of BTC at the hour of a date from a rate provider
+
+  {
+    currency: <Currency Type String>
+    date: <ISO 8601 Date String>
+    fiat: <Fiat Type String>
+    from: <Rate Provider String>
+    request: <Request Function>
+  }
+
+  @returns via cbk or Promise
+  {
+    cents: <Cents Per BTC Number>
+  }
+*/
+module.exports = ({currency, date, fiat, from, request}, cbk) => {
+  return new Promise((resolve, reject) => {
+    return asyncAuto({
+      // Check arguments
+      validate: cbk => {
+        if (currency !== supportedCurrency) {
+          return cbk([400, 'UnsupportedCurrencyToGetHistoricRate']);
+        }
+
+        if (!date) {
+          return cbk([400, 'ExpectedDateToGetHistoricRate']);
+        }
+
+        if (!fiat) {
+          return cbk([400, 'ExpectedFiatToGetHistoricRate']);
+        }
+
+        if (!from) {
+          return cbk([400, 'ExpectedRateProviderToGetHistoricRate']);
+        }
+
+        if (!request) {
+          return cbk([400, 'ExpectedRequestFunctionToGetHistoricRate']);
+        }
+
+        return cbk();
+      },
+
+      // Get the price at the hour of the date, trying a failed request again
+      getPrice: ['validate', ({}, cbk) => {
+        return asyncRetry({
+          errorFilter: isFailedRequest,
+          interval: count => retryDelaysMs[count - 1],
+          times: retryDelaysMs.length + 1,
+        },
+        cbk => getPrices({date, from, request, symbols: [fiat]}, cbk),
+        cbk);
+      }],
+
+      // The rate of the fiat
+      rate: ['getPrice', ({getPrice}, cbk) => {
+        const [ticker] = getPrice.tickers;
+
+        return cbk(null, {cents: ticker.rate});
+      }],
+    },
+    returnResult({reject, resolve, of: 'rate'}, cbk));
+  });
+};

package/balances/get_record_rate.js

@@ -0,0 +1,77 @@
+const asyncAuto = require('async/auto');
+const {returnResult} = require('asyncjs-util');
+
+const {coinbaseProvider} = require('./constants');
+const getHistoricRate = require('./get_historic_rate');
+
+/** Get the fiat rate of BTC for an accounting record
+
+  {
+    currency: <Currency Type String>
+    date: <Record ISO 8601 Date String>
+    fiat: <Fiat Type String>
+    [get_chain_rate]: <Get Chain Rate Function>
+    request: <Request Function>
+  }
+
+  @returns via cbk or Promise
+  {
+    cents: <Cents Per BTC Number>
+  }
+*/
+module.exports = (args, cbk) => {
+  return new Promise((resolve, reject) => {
+    return asyncAuto({
+      // Check arguments
+      validate: cbk => {
+        if (!args.request) {
+          return cbk([400, 'ExpectedRequestFunctionToGetRecordRate']);
+        }
+
+        return cbk();
+      },
+
+      // Get the price derived from chain data
+      getChainRate: ['validate', ({}, cbk) => {
+        // Exit early when there are no chain prices
+        if (!args.get_chain_rate) {
+          return cbk();
+        }
+
+        return args.get_chain_rate({
+          currency: args.currency,
+          date: args.date,
+          fiat: args.fiat,
+        },
+        cbk);
+      }],
+
+      // Get the price at the hour of the record on Coinbase Exchange
+      getCoinbaseRate: ['getChainRate', ({getChainRate}, cbk) => {
+        // Exit early when the record has a chain price
+        if (!!getChainRate) {
+          return cbk();
+        }
+
+        return getHistoricRate({
+          currency: args.currency,
+          date: args.date,
+          fiat: args.fiat,
+          from: coinbaseProvider,
+          request: args.request,
+        },
+        cbk);
+      }],
+
+      // The rate of the record
+      rate: [
+        'getChainRate',
+        'getCoinbaseRate',
+        ({getChainRate, getCoinbaseRate}, cbk) =>
+      {
+        return cbk(null, {cents: (getChainRate || getCoinbaseRate).cents});
+      }],
+    },
+    returnResult({reject, resolve, of: 'rate'}, cbk));
+  });
+};

package/balances/guard_chain_rate.js

@@ -0,0 +1,50 @@
+const {pricesFileErrors} = require('./constants');
+
+const isFileError = err => !!err && pricesFileErrors.includes(err[1]);
+
+/** Make a chain rate lookup that only fails on prices file errors
+
+  {
+    getChainRate: <Get Chain Rate Function>
+  }
+
+  @returns
+  {
+    getChainRate: <Get Chain Rate Method> ({
+      currency: <Currency To Price String>
+      date: <Date To Price ISO 8601 Date String>
+      fiat: <Fiat Currency To Price In String>
+    }, cbk) => (err, [{
+      cents: <US Cents Per BTC Number>
+    }])
+  }
+*/
+module.exports = ({getChainRate}) => {
+  const failure = {};
+
+  return {
+    getChainRate: (args, cbk) => {
+      // Exit early when the prices file failed, since it would fail again
+      if (!!failure.err) {
+        return cbk(failure.err);
+      }
+
+      return getChainRate(args, (err, res) => {
+        // Exit early when there is a chain price
+        if (!err) {
+          return cbk(null, res);
+        }
+
+        // Exit early without a price when chain prices cannot value the date
+        if (!isFileError(err)) {
+          return cbk();
+        }
+
+        // A prices file error is kept, as trying again cannot fix the file
+        failure.err = err;
+
+        return cbk(err);
+      });
+    },
+  };
+};

package/bos

@@ -70,6 +70,7 @@ prog
   .argument('<category>', 'Report category', keys(accountingCategories))
   .help(`Categories: ${keys(accountingCategories).join(', ')}`)
   .help(`Rate providers: ${rateProviders.join(', ')}`)
+  .help('By default fiat rates after 2021-10-19 use chain data derived rates')
   .help('Privacy note: this requests tx related data from third parties')
   .option('--csv', 'Output a CSV')
   .option('--date <day>', 'Show only records for specific date')
@@ -84,8 +85,15 @@ prog
     return new Promise(async (resolve, reject) => {
       try {
         return balances.getAccountingReport({
+          logger,
           category: args.category,
           date: options.date,
+          fs: {
+            writeFile,
+            getFile: readFile,
+            makeDirectory: mkdir,
+            renameFile: rename,
+          },
           is_csv: !!options.csv,
           is_fiat_disabled: options.disableFiat,
           lnd: (await lnd.authenticatedLnd({logger, node: options.node})).lnd,
@@ -1649,6 +1657,7 @@ prog
   .option('--avoid <avoid>', 'Avoid forwarding via node/chan/tag', REPEATABLE)
   .option('--avoid-high-fee-routes', 'Avoid evaluating too-high fee routes')
   .option('--find-max', 'Find the maximum routeable amount on success route')
+  .option('--ignore-expiry', 'Probe even when the request has expired')
   .option('--in <public_key>', 'Route through specific peer of destination')
   .option('--max-fee <max_fee>', 'Max expected fee for route', INT, 1337)
   .option('--max-paths <max>', 'Maximum paths to use for find-max', INT, 1)
@@ -1671,6 +1680,7 @@ prog
           find_max: !!options.findMax ? 16777215 : undefined,
           fs: {getFile: readFile},
           in_through: inThrough.public_key,
+          is_ignoring_expiry: !!options.ignoreExpiry,
           is_strict_max_fee: !!options.avoidHighFeeRoutes,
           max_fee: options.maxFee,
           max_paths: options.maxPaths,

package/chain/fund_transaction.js

@@ -1,5 +1,6 @@
 const asyncAuto = require('async/auto');
 const asyncEach = require('async/each');
+const asyncify = require('async/asyncify');
 const {broadcastTransaction} = require('ln-sync');
 const {componentsOfTransaction} = require('@alexbosworth/blockchain');
 const {formatTokens} = require('ln-sync');
@@ -112,10 +113,10 @@ module.exports = (args, cbk) => {
       getNetwork: ['validate', ({}, cbk) => getNetwork({lnd: args.lnd}, cbk)],
 
       // Derive a list of outputs to guide input selection
-      outputs: ['validate', ({}, cbk) => {
+      outputs: ['validate', asyncify(() => {
         // Exit early when the amount is open ended and thus depends on inputs
         if (hasMaxAmount(args.amounts)) {
-          return cbk();
+          return;
         }
 
         try {
@@ -125,11 +126,11 @@ module.exports = (args, cbk) => {
             return {address, tokens};
           });
 
-          return cbk(null, outputs);
+          return outputs;
         } catch (err) {
-          return cbk([400, err.message]);
+          throw [400, err.message];
         }
-      }],
+      })],
 
       // Get UTXOs to use for input selection and final fee rate calculation
       getUtxos: ['validate', ({}, cbk) => getUtxos({lnd: args.lnd}, cbk)],
@@ -238,7 +239,7 @@ module.exports = (args, cbk) => {
       }],
 
       // Parse amounts and put together the final set of outputs
-      finalOutputs: ['getMax', ({getMax}, cbk) => {
+      finalOutputs: ['getMax', asyncify(({getMax}) => {
         try {
           const outputs = args.addresses.map((address, i) => {
             const amount = args.amounts[i];
@@ -252,11 +253,11 @@ module.exports = (args, cbk) => {
             return {address, tokens: parseAmount({amount, variables}).tokens};
           });
 
-          return cbk(null, outputs);
+          return outputs;
         } catch (err) {
-          return cbk([400, err.message]);
+          throw [400, err.message];
         }
-      }],
+      })],
 
       // Create a funded PSBT
       fund: [

package/chain/get_block_transactions.js

@@ -1,4 +1,5 @@
 const asyncAuto = require('async/auto');
+const asyncify = require('async/asyncify');
 const {getBlock} = require('ln-service');
 const {queryTransactions} = require('@alexbosworth/blockchain');
 const {returnResult} = require('asyncjs-util');
@@ -77,10 +78,10 @@ module.exports = (args, cbk) => {
       }],
 
       // Find the transactions in the block
-      transactions: ['getBlock', ({getBlock}, cbk) => {
+      transactions: ['getBlock', asyncify(({getBlock}) => {
         // Exit early when there is no block to look through
         if (!getBlock.block) {
-          return cbk(null, {transactions: []});
+          return {transactions: []};
         }
 
         try {
@@ -89,11 +90,11 @@ module.exports = (args, cbk) => {
             ids: args.ids,
           });
 
-          return cbk(null, {transactions});
+          return {transactions};
         } catch (err) {
-          return cbk([503, 'UnexpectedErrorParsingBlockTransactions', {err}]);
+          throw [503, 'UnexpectedErrorParsingBlockTransactions', {err}];
         }
-      }],
+      })],
     },
     returnResult({reject, resolve, of: 'transactions'}, cbk));
   });

package/chain/recover_p2pk.js

@@ -1,6 +1,7 @@
 const {createHash} = require('crypto');
 
 const asyncAuto = require('async/auto');
+const asyncify = require('async/asyncify');
 const {broadcastChainTransaction} = require('ln-service');
 const {componentsOfTransaction} = require('@alexbosworth/blockchain');
 const {createChainAddress} = require('ln-service');
@@ -135,7 +136,7 @@ module.exports = ({id, lnd, request, vout}, cbk) => {
       recoveryScript: [
         'createAddress',
         'getNetwork',
-        ({createAddress, getNetwork}, cbk) =>
+        asyncify(({createAddress, getNetwork}) =>
       {
         try {
           const {script} = outputScriptForAddress({
@@ -143,11 +144,11 @@ module.exports = ({id, lnd, request, vout}, cbk) => {
             network: getNetwork.network,
           });
 
-          return cbk(null, {script});
+          return {script};
         } catch (err) {
-          return cbk([503, 'FailedToParseRecoveryAddress', {err}]);
+          throw [503, 'FailedToParseRecoveryAddress', {err}];
         }
-      }],
+      })],
 
       // Derive the sweep output, less the amount needed to pay for chain fees
       sweep: [

package/chain/split_utxos.js

@@ -1,5 +1,6 @@
 const asyncAuto = require('async/auto');
 const asyncEach = require('async/each');
+const asyncify = require('async/asyncify');
 const asyncMapSeries = require('async/mapSeries');
 const asyncReflect = require('async/reflect');
 const {broadcastChainTransaction} = require('ln-service');
@@ -85,7 +86,7 @@ module.exports = (args, cbk) => {
     getUtxos: ['validate', ({}, cbk) => getUtxos({lnd: args.lnd}, cbk)],
 
     // Select the inputs and the count of outputs that reach the count
-    plan: ['feeRate', 'getUtxos', ({feeRate, getUtxos}, cbk) => {
+    plan: ['feeRate', 'getUtxos', asyncify(({feeRate, getUtxos}) => {
       try {
         const plan = utxoSplitPlan({
           count: args.count,
@@ -95,11 +96,11 @@ module.exports = (args, cbk) => {
           utxos: getUtxos.utxos,
         });
 
-        return cbk(null, plan);
+        return plan;
       } catch (err) {
-        return cbk([400, err.message]);
+        throw [400, err.message];
       }
-    }],
+    })],
 
     // Generate addresses
     getAddresses: ['plan', ({plan}, cbk) => {

package/commands/constants.json

@@ -25,6 +25,6 @@
   ],
   "priceProviders": ["coinbase", "coindesk", "coingecko"],
   "process": "bos",
-  "rateProviders": ["coindesk", "coingecko"],
+  "rateProviders": ["coinbase", "coindesk", "coingecko"],
   "swapTypes": ["inbound", "outbound"]
 }