balanceofsatoshis 24.0.1 → 24.2.0

package/CHANGELOG.md

@@ -1,5 +1,18 @@
 # Versions
 
+## 24.2.0
+
+- Add support for LND 0.21.4 and LND 0.20.5
+- `pay`, `probe`: Add support for tags in `--out`, `--out-filter`
+
+## 24.1.0
+
+- `fees`: Add `--repeat-interval-ms` to show or set the fees again each time an
+    interval elapses, such as `5*min` to keep formula fee rates up to date
+- `rebalance`: Add `--repeat-interval-ms` to run the rebalance again each time
+    an interval elapses, such as `30*min`, `1*h * (0.5 + RAND())` for a random
+    interval, or `1*min * 2^FAILURES_COUNT` to back off on failures in a row
+
 ## 24.0.1
 
 ### Breaking Changes

package/README.md

@@ -7,8 +7,8 @@ Commands for working with LND balances.
 
 Supported LND versions:
 
-- v0.21.0-beta to v0.21.3-beta
-- v0.20.0-beta to v0.20.4-beta
+- v0.21.0-beta to v0.21.4-beta
+- v0.20.0-beta to v0.20.5-beta
 
 ## Install
 
@@ -327,18 +327,17 @@ expr $(bos balance --node=savedNode1) + $(bos balance --node=savedNode2)
 
 ### Auto Adjust Fees
 
-```
-# Cron every 5 minutes adjust fees
-*/5 * * * * /bin/timeout -s 2 30 /home/ubuntu/update-fees.sh
-```
-
-update-fees.sh:
+Repeat the `fees` command on an interval to keep a fee rate formula up to date
 
+```shell
+# Every 5 minutes, raise the outbound fees to a public key when inbound increases
+bos fees --to PUBLIC_KEY --set-fee-rate="IF(INBOUND>10000000,1000,500)" --repeat-interval-ms="5*min"
 ```
-#!/bin/bash
-# Raise the outbound fees to a public key when inbound increases
-/home/ubuntu/.npm-global/bin/bos fees --to PUBLIC_KEY --set-fee-rate="IF(INBOUND>10000000,1000,500)"
-```
+
+The interval is a formula in milliseconds that can use `s`, `min`, `h`, `d` and
+`w` units, so `"5*min * (0.5 + RAND())"` repeats at a random interval between
+2.5 and 7.5 minutes. See "Persist Long-Running Commands" below to keep the
+command running.
 
 ### Auto Balance Liquidity Between Two Nodes
 
@@ -431,13 +430,11 @@ Docker example:
 docker run -d --restart always -v $HOME/.bos:/home/node/.bos alexbosworth/balanceofsatoshis telegram --connect CONNECT_CODE
 ```
 
-You can also create a shell-script.sh to run a command repeatedly, with a delay
+Commands like `fees` and `rebalance` can also repeat themselves, with a delay
 
-```bash
-while true;
-do bos rebalance;
-sleep 2000;
-done
+```shell
+# Rebalance again 30 minutes after each attempt, or 1 hour when it failed
+bos rebalance --repeat-interval-ms="IF(FAILURES_COUNT > 0, 1*h, 30*min)"
 ```
 
 ## Docker

package/bos

@@ -14,7 +14,6 @@ const {writeFile} = require('fs');
 const importLazy = require('import-lazy')(require);
 
 const accounting = importLazy('ln-accounting');
-const fetch = importLazy('@alexbosworth/node-fetch');
 const fiat = importLazy('@alexbosworth/fiat');
 const lnService = importLazy('ln-service');
 const lnSync = importLazy('ln-sync');
@@ -731,7 +730,9 @@ prog
   .help('You can use INBOUND_FEE_RATE to mirror an inbound fee')
   .help('You can use FEE_RATE_OF_<PUBKEY> to reference other node rates')
   .help('--set-inbound-discount-rate is total discount on traffic from peer')
+  .help('--repeat-interval-ms vars: s, min, h, d, w: 1*h repeats hourly')
   .option('--node <node_name>', 'Saved node (not peer to set fees on)')
+  .option('--repeat-interval-ms <interval>', 'Repeat fees after interval')
   .option('--set-cltv-delta <count>', 'Set the number of blocks for CLTV', INT)
   .option('--set-fee-rate <rate>', 'Fee in parts per million or use a formula')
   .option('--set-inbound-rate-discount <rate>', 'Source discount rate in PPM')
@@ -739,13 +740,14 @@ prog
   .action((args, options, logger) => {
     return new Promise(async (resolve, reject) => {
       try {
-        return routing.adjustFees({
+        return routing.manageFees({
           logger,
           cltv_delta: options.setCltvDelta,
           fee_rate: options.setFeeRate,
           fs: {getFile: readFile},
           inbound_rate_discount: options.setInboundRateDiscount,
           lnd: (await lnd.authenticatedLnd({logger, node: options.node})).lnd,
+          repeat_interval_ms: options.repeatIntervalMs || undefined,
           to: flatten([options.to].filter(n => !!n)),
         },
         responses.returnObject({logger, reject, resolve, table: 'rows'}));
@@ -1514,6 +1516,10 @@ prog
   // Pay a payment request, probing first
   .command('pay', 'Pay a payment request, probing first')
   .help('Blinded path requests are probed to the introduction node, 1 path')
+  .help('--out can take a tag to make the first hop through any tagged peer')
+  .help('--out-filter vars: CAPACITY/HEIGHTS/INBOUND_LIQUIDITY')
+  .help('--out-filter vars: OUTBOUND_LIQUIDITY/PENDING_PAYMENTS')
+  .help('--out-filter vars: INBOUND_BASE_FEE/INBOUND_FEE_RATE')
   .argument('<request>', 'Payment Request')
   .option('--avoid <avoid>', 'Avoid forwarding via node/chan/tag', REPEATABLE)
   .option('--avoid-high-fee-routes', 'Avoid evaluating too-high fee routes')
@@ -1523,7 +1529,8 @@ prog
   .option('--message <message>', 'Attach text message to payment')
   .option('--no-color', 'Mute all colors')
   .option('--node <node_name>', 'Node to use to pay payment request')
-  .option('--out <public_key>', 'Make first hop through peer', REPEATABLE)
+  .option('--out <peer_or_tag>', 'Make first hop through peer/tag', REPEATABLE)
+  .option('--out-filter <out_filter>', 'Filter outbound peer/tag', REPEATABLE)
   .action((args, options, logger) => {
     return new Promise(async (resolve, reject) => {
       try {
@@ -1542,6 +1549,7 @@ prog
           max_paths: options.maxPaths,
           message: options.message,
           out: flatten([options.out].filter(n => !!n)),
+          out_filters: flatten([options.outFilter].filter(n => !!n)),
           request: args.request,
         },
         responses.returnObject({logger, reject, resolve}));
@@ -1634,6 +1642,10 @@ prog
   .command('probe', 'Check if a payment request is sendable')
   .help('Simulate paying a payment request without actually paying it')
   .help('Blinded path requests are probed to the introduction node, 1 path')
+  .help('--out can take a tag to make the first hop through any tagged peer')
+  .help('--out-filter vars: CAPACITY/HEIGHTS/INBOUND_LIQUIDITY')
+  .help('--out-filter vars: OUTBOUND_LIQUIDITY/PENDING_PAYMENTS')
+  .help('--out-filter vars: INBOUND_BASE_FEE/INBOUND_FEE_RATE')
   .argument('<to>', 'Payment request or node public key')
   .argument('[amount]', 'Amount to probe, default: request amount')
   .option('--avoid <avoid>', 'Avoid forwarding via node/chan/tag', REPEATABLE)
@@ -1644,7 +1656,8 @@ prog
   .option('--max-paths <max>', 'Maximum paths to use for find-max', INT, 1)
   .option('--no-color', 'Mute all colors')
   .option('--node <node_name>', 'Node to use for payment request check')
-  .option('--out <public_key>', 'Make first hop through peer', REPEATABLE)
+  .option('--out <peer_or_tag>', 'Make first hop through peer/tag', REPEATABLE)
+  .option('--out-filter <out_filter>', 'Filter outbound peer/tag', REPEATABLE)
   .action((args, options, logger) => {
     return new Promise(async (resolve, reject) => {
       try {
@@ -1664,6 +1677,7 @@ prog
           max_fee: options.maxFee,
           max_paths: options.maxPaths,
           out: flatten([options.out].filter(n => !!n)),
+          out_filters: flatten([options.outFilter].filter(n => !!n)),
           request: args.to.length !== 66 ? args.to : undefined,
           tokens: args.amount || undefined,
         },
@@ -1693,6 +1707,8 @@ prog
   .help('--in-filter/--out-filter vars: OUTBOUND_LIQUIDITY/PENDING_PAYMENTS')
   .help('--in-filter/--out-filter vars: INBOUND_BASE_FEE/INBOUND_FEE_RATE')
   .help('--out increases the inbound liquidity with a specific peer/tag')
+  .help('--repeat-interval-ms vars: s, min, h, d, w: 1*h repeats hourly')
+  .help('--repeat-interval-ms var FAILURES_COUNT: count of failures in a row')
   .option('--amount <amount>', 'Maximum amount to rebalance')
   .option('--avoid <pubkey_or_chanid>', 'Avoid forwarding through', REPEATABLE)
   .option('--avoid-append', 'Formula for appending failing edges to list')
@@ -1709,6 +1725,7 @@ prog
   .option('--out <pubkey_or_alias>', 'Route out through a specific peer')
   .option('--out-filter <out_filter>', 'Filter outbound peer/tag', REPEATABLE)
   .option('--out-target-inbound <amount>', 'Balance up to inbound amount')
+  .option('--repeat-interval-ms <interval>', 'Repeat rebalance after interval')
   .action((args, options, logger) => {
     return new Promise(async (resolve, reject) => {
       try {
@@ -1729,6 +1746,7 @@ prog
           out_filters: flatten([options.outFilter].filter(n => !!n)),
           out_inbound: options.outTargetInbound,
           out_through: options.out || undefined,
+          repeat_interval_ms: options.repeatIntervalMs || undefined,
           timeout_minutes: options.minutes || undefined,
         },
         responses.returnObject({exit, logger, reject, resolve}));

package/chain/index.js

@@ -6,7 +6,6 @@ const getDepositAddress = require('./get_deposit_address');
 const getMempoolSize = require('./get_mempool_size');
 const getRawTransaction = require('./get_raw_transaction');
 const getUtxos = require('./get_utxos');
-const outputScriptForAddress = require('./output_script_for_address');
 const recoverP2pk = require('./recover_p2pk');
 const splitUtxos = require('./split_utxos');
 
@@ -19,7 +18,6 @@ module.exports = {
   getMempoolSize,
   getRawTransaction,
   getUtxos,
-  outputScriptForAddress,
   recoverP2pk,
   splitUtxos,
 };

package/chain/recover_p2pk.js

@@ -1,40 +1,39 @@
 const {createHash} = require('crypto');
 
 const asyncAuto = require('async/auto');
-const bitcoinjsLib = require('bitcoinjs-lib');
 const {broadcastChainTransaction} = require('ln-service');
+const {componentsOfTransaction} = require('@alexbosworth/blockchain');
 const {createChainAddress} = require('ln-service');
-const {decode} = require('bip66');
 const {getChainFeeRate} = require('ln-service');
 const {getIdentity} = require('ln-service');
 const {getNetwork} = require('ln-sync');
+const {idForTransaction} = require('@alexbosworth/blockchain');
+const {outputScriptForAddress} = require('@alexbosworth/blockchain');
 const {returnResult} = require('asyncjs-util');
+const {scriptElementsAsScript} = require('@alexbosworth/blockchain');
 const {signBytes} = require('ln-service');
+const {sizeOfTransaction} = require('@alexbosworth/blockchain');
+const {transactionFromComponents} = require('@alexbosworth/blockchain');
 
 const getRawTransaction = require('./get_raw_transaction');
 
 const {ceil} = Math;
-const {compile} = bitcoinjsLib.script;
 const {concat} = Buffer;
 const description = 'bos recover p2pk node identity key funds';
-const {encode} = bitcoinjsLib.script.signature;
 const estimatedSignatureSize = 73;
 const format = 'p2wpkh';
-const {fromHex} = bitcoinjsLib.Transaction;
 const hashFlag = Buffer.from('01000000', 'hex');
 const hexAsBuffer = hex => Buffer.from(hex, 'hex');
-const inputIndex = 0;
 const inputSequence = 0;
 const isHash = n => !!n && /^[0-9A-F]{64}$/i.test(n);
-const networkNames = {btc: 'bitcoin', btctestnet: 'testnet'};
-const {networks} = bitcoinjsLib;
+const locktime = 0;
 const nodeIdentityKeyFamily = 6;
 const nodeIdentityKeyIndex = 0;
 const OP_CHECKSIG = 172;
 const sha256 = n => createHash('sha256').update(n).digest().toString('hex');
-const slicePoint = r => r.length === 33 ? r.slice(1) : r;
-const {toOutputScript} = bitcoinjsLib.address;
-const {Transaction} = bitcoinjsLib;
+const sigHashType = Buffer.from('01', 'hex');
+const unsignedScript = '';
+const version = 1;
 
 /** Recover funds sent to a P2PK using the node identity key
 
@@ -97,14 +96,16 @@ module.exports = ({id, lnd, request, vout}, cbk) => {
 
       // Derive the transaction details
       output: ['getIdentity', 'getTx', ({getIdentity, getTx}, cbk) => {
-        const tx = fromHex(getTx.transaction);
+        const {transaction} = getTx;
 
         // Make sure the tx data matches the input id
-        if (tx.getId() !== id) {
+        if (idForTransaction({transaction}).id !== id) {
           return cbk([503, 'ExpectedTransactionIdToMatchTxDataHashToRecover']);
         }
 
-        const output = tx.outs[vout];
+        const {outputs} = componentsOfTransaction({transaction});
+
+        const output = outputs[vout];
 
         // Make sure the output exists
         if (!output) {
@@ -113,14 +114,16 @@ module.exports = ({id, lnd, request, vout}, cbk) => {
 
         const identityKey = hexAsBuffer(getIdentity.public_key);
 
-        const expectedScript = compile([identityKey, OP_CHECKSIG]);
+        const expected = scriptElementsAsScript({
+          elements: [identityKey, OP_CHECKSIG],
+        });
 
         // Make sure that the output pays to the node identity key
-        if (!output.script.equals(expectedScript)) {
+        if (output.script !== expected.script) {
           return cbk([503, 'ExpectedOutputPayingToNodeIdentityPublicKey']);
         }
 
-        return cbk(null, {script: output.script, tokens: output.value});
+        return cbk(null, {script: output.script, tokens: output.tokens});
       }],
 
       // Create a recovery address
@@ -128,45 +131,73 @@ module.exports = ({id, lnd, request, vout}, cbk) => {
         return createChainAddress({format, lnd}, cbk);
       }],
 
-      // Make the transaction to sign
-      txToSign: [
+      // Derive the output script of the recovery address
+      recoveryScript: [
         'createAddress',
-        'getFee',
         'getNetwork',
-        'output',
-        ({createAddress, getFee, getNetwork, output}, cbk) =>
+        ({createAddress, getNetwork}, cbk) =>
       {
-        const network = networks[networkNames[getNetwork.network]];
-        const tx = new Transaction();
-
-        const scriptPubKey = toOutputScript(createAddress.address, network);
+        try {
+          const {script} = outputScriptForAddress({
+            address: createAddress.address,
+            network: getNetwork.network,
+          });
+
+          return cbk(null, {script});
+        } catch (err) {
+          return cbk([503, 'FailedToParseRecoveryAddress', {err}]);
+        }
+      }],
 
-        tx.addInput(hexAsBuffer(id).reverse(), vout, inputSequence);
-        tx.addOutput(scriptPubKey, output.tokens);
+      // Derive the sweep output, less the amount needed to pay for chain fees
+      sweep: [
+        'getFee',
+        'output',
+        'recoveryScript',
+        ({getFee, output, recoveryScript}, cbk) =>
+      {
+        const {script} = recoveryScript;
+
+        const unsigned = transactionFromComponents({
+          locktime,
+          version,
+          inputs: [{
+            id,
+            vout,
+            script: unsignedScript,
+            sequence: inputSequence,
+          }],
+          outputs: [{script, tokens: output.tokens}],
+        });
 
-        // There is only one output on the sweep transaction
-        const [out] = tx.outs;
+        const {vsize} = sizeOfTransaction(unsigned);
 
         // Include the prospective signature in the tx total weight
-        const vbytes = tx.virtualSize() + estimatedSignatureSize;
+        const vbytes = vsize + estimatedSignatureSize;
 
         // Reduce the sweep value by the amount needed to pay for chain fees
-        out.value -= ceil(vbytes * getFee.tokens_per_vbyte);
+        const tokens = output.tokens - ceil(vbytes * getFee.tokens_per_vbyte);
 
-        return cbk(null, tx);
+        return cbk(null, {script, tokens});
       }],
 
       // Derive the preimage to use for signing
-      preimage: ['output', 'txToSign', ({output, txToSign}, cbk) => {
-        const cloneTx = txToSign.clone();
-
-        const [input] = cloneTx.ins;
-
+      preimage: ['output', 'sweep', ({output, sweep}, cbk) => {
         // When signing, the input to sign is set to the previous output script
-        input.script = output.script;
+        const {transaction} = transactionFromComponents({
+          locktime,
+          version,
+          inputs: [{
+            id,
+            vout,
+            script: output.script,
+            sequence: inputSequence,
+          }],
+          outputs: [sweep],
+        });
 
         // The bytes to sign are the tx itself plus the signature hash flag
-        return cbk(null, concat([cloneTx.toBuffer(), hashFlag]));
+        return cbk(null, concat([hexAsBuffer(transaction), hashFlag]));
       }],
 
       // Give the preimage to signer, hashed once - signBytes does 2nd SHA hash
@@ -181,17 +212,18 @@ module.exports = ({id, lnd, request, vout}, cbk) => {
       }],
 
       // Put together the signature with the transaction
-      signedTx: ['getSig', 'txToSign', ({getSig, txToSign}, cbk) => {
-        const {r, s} = decode(hexAsBuffer(getSig.signature));
-
-        const rValue = slicePoint(r);
-
-        // Convert the signature from sign bytes to a chain signature
-        const scriptSig = encode(concat([rValue, s]), Transaction.SIGHASH_ALL);
-
-        txToSign.setInputScript(inputIndex, compile([scriptSig]));
-
-        return cbk(null, txToSign);
+      signedTx: ['getSig', 'sweep', ({getSig, sweep}, cbk) => {
+        // A chain signature is the DER signature followed by the sighash type
+        const signature = concat([hexAsBuffer(getSig.signature), sigHashType]);
+
+        const {script} = scriptElementsAsScript({elements: [signature]});
+
+        return cbk(null, transactionFromComponents({
+          locktime,
+          version,
+          inputs: [{id, script, vout, sequence: inputSequence}],
+          outputs: [sweep],
+        }));
       }],
 
       // Broadcast the signed transaction
@@ -199,7 +231,7 @@ module.exports = ({id, lnd, request, vout}, cbk) => {
         return broadcastChainTransaction({
           description,
           lnd,
-          transaction: signedTx.toHex(),
+          transaction: signedTx.transaction,
         },
         cbk);
       }],
@@ -214,7 +246,7 @@ module.exports = ({id, lnd, request, vout}, cbk) => {
         return cbk(null, {
           recovering: output.tokens,
           recovering_to: createAddress.address,
-          transaction_id: signedTx.getId(),
+          transaction_id: idForTransaction(signedTx).id,
         });
       }],
     },

package/commands/fetch_request.js

@@ -1,14 +1,12 @@
 const {stringify} = require('querystring');
 
-const {AbortController} = require('abort-controller');
-
 const encodeQs = qs => !qs ? '' : '?' + stringify(qs);
 const timeoutSignals = new WeakMap();
 
 /** Derive a request function that uses fetch to simulate request behavior
 
   {
-    fetch: <Node Fetch Function>
+    fetch: <Fetch Function>
   }
 
   @returns

package/display/constants.json

@@ -1,4 +1,10 @@
 {
+  "durationUnits": [
+    {"ms": 86400000, "unit": "day"},
+    {"ms": 3600000, "unit": "hour"},
+    {"ms": 60000, "unit": "minute"},
+    {"ms": 1000, "unit": "second"}
+  ],
   "parseErrors": {
     "#DIV/0!": "CannotDivideByZeroInFormula",
     "#ERROR!":  "FailedToParseFormula",

package/display/format_duration.js

@@ -0,0 +1,34 @@
+const {durationUnits} = require('./constants');
+
+const {floor} = Math;
+const maxUnits = 2;
+const plural = (count, unit) => `${count} ${unit}${count === 1 ? '' : 's'}`;
+
+/** Format a duration for display, using the two largest units of time
+
+  {
+    ms: <Duration Milliseconds Number>
+  }
+
+  @returns
+  {
+    display: <Display Formatted Duration String>
+  }
+*/
+module.exports = ({ms}) => {
+  // Take whole counts of each unit out of the duration, largest unit first
+  const {parts, remaining} = durationUnits.reduce((sum, unit) => {
+    const count = floor(sum.remaining / unit.ms);
+
+    return {
+      parts: !count ? sum.parts : sum.parts.concat(plural(count, unit.unit)),
+      remaining: sum.remaining - count * unit.ms,
+    };
+  },
+  {parts: [], remaining: ms});
+
+  // Leftover milliseconds are shown when present or when there is nothing else
+  const leftover = !remaining && !!parts.length ? [] : [`${remaining} ms`];
+
+  return {display: parts.concat(leftover).slice(0, maxUnits).join(' ')};
+};

package/display/index.js

@@ -2,6 +2,7 @@ const chartAliasForPeer = require('./chart_alias_for_peer');
 const describeConfidence = require('./describe_confidence');
 const describeRoute = require('./describe_route');
 const describeRoutingFailure = require('./describe_routing_failure');
+const formatDuration = require('./format_duration');
 const formatFeeRate = require('./format_fee_rate');
 const getIcons = require('./get_icons');
 const isMatchingFilters = require('./is_matching_filters');
@@ -13,6 +14,7 @@ module.exports = {
   describeConfidence,
   describeRoute,
   describeRoutingFailure,
+  formatDuration,
   formatFeeRate,
   getIcons,
   isMatchingFilters,

package/network/execute_probe.js

@@ -40,7 +40,7 @@ const tokensAsMillitokens = tok => (BigInt(tok) * BigInt(1e3)).toString();
       type: <Message To Final Destination Type Number String>
       value: <Message To Final Destination Raw Value Hex Encoded String>
     }]
-    [outgoing_channel]: <Outgoing Channel Id String>
+    [outgoing_channels]: [<Outgoing Channel Id String>]
     [paths]: [{
       base_fee_mtokens: <Accumulated Base Fee Millitokens String>
       cltv_delta: <Accumulated CLTV Expiry Delta Number>
@@ -176,7 +176,7 @@ module.exports = (args, cbk) => {
             max_fee_mtokens: strictMaxFee || undefined,
             max_timeout_height: args.max_timeout_height,
             messages: args.messages,
-            outgoing_channel: args.outgoing_channel,
+            outgoing_channels: args.outgoing_channels,
             path_timeout_ms: pathTimeoutMs,
             paths: args.paths,
             payment: args.payment,