balanceofsatoshis 23.1.7 → 23.1.8

package/CHANGELOG.md

@@ -1,6 +1,6 @@
 # Versions
 
-## 23.1.7
+## 23.1.8
 
 - Add support for LND 0.21.2
 - Add support for LND 0.20.3

package/bos

@@ -4,6 +4,7 @@ const {lstat} = require('fs');
 const {mkdir} = require('fs');
 const {readdir} = require('fs');
 const {readFile} = require('fs');
+const {rename} = require('fs');
 const {rmdir} = require('fs');
 const {spawn} = require('child_process');
 const {statSync} = require('fs');
@@ -1752,7 +1753,7 @@ prog
           avoid: flatten([options.avoid].filter(n => !!n)),
           avoid_append: options.avoidAppend || undefined,
           avoid_list: options.avoidList || undefined,
-          fs: {appendFile, writeFile, getFile: readFile},
+          fs: {appendFile, writeFile, getFile: readFile, renameFile: rename},
           in_filters: flatten([options.inFilter].filter(n => !!n)),
           in_outbound: options.inTargetOutbound || undefined,
           in_through: options.in || undefined,

package/package.json

@@ -83,5 +83,5 @@
     "postpublish": "docker buildx build --platform linux/amd64,linux/arm64 -t alexbosworth/balanceofsatoshis -t alexbosworth/balanceofsatoshis:$npm_package_version --push .",
     "test": "node --experimental-test-coverage --test test/arrays/*.js test/balances/*.js test/chain/*.js test/display/*.js test/encryption/*.js test/lnd/*.js test/network/*.js test/nodes/*.js test/peers/*.js test/responses/*.js test/routing/*.js test/services/*.js test/swaps/*.js test/tags/*.js test/telegram/*.js test/wallets/*.js"
   },
-  "version": "23.1.7"
+  "version": "23.1.8"
 }

package/swaps/get_avoid_list.js

@@ -0,0 +1,53 @@
+const asyncAuto = require('async/auto');
+const {returnResult} = require('asyncjs-util');
+
+const matchNewLines = /\r?\n/;
+const trim = n => n.trim();
+const uniq = arr => Array.from(new Set(arr));
+
+/** Get the list of avoid directives from an avoid list file
+
+  {
+    fs: {
+      getFile: <Read File Contents Function> (path, cbk) => {}
+    }
+    path: <Path To Avoid List File String>
+  }
+
+  @returns via cbk or Promise
+  {
+    lines: [<Avoid Directive Line String>]
+  }
+*/
+module.exports = ({fs, path}, cbk) => {
+  return new Promise((resolve, reject) => {
+    return asyncAuto({
+      // Check arguments
+      validate: cbk => {
+        if (!fs) {
+          return cbk([400, 'ExpectedFilesystemMethodsToGetAvoidList']);
+        }
+
+        if (!path) {
+          return cbk([400, 'ExpectedPathToAvoidListFileToGetAvoidList']);
+        }
+
+        return cbk();
+      },
+
+      // Get the lines of the avoid list file
+      getList: ['validate', ({}, cbk) => {
+        return fs.getFile(path, (err, res) => {
+          if (!!err) {
+            return cbk([503, 'UnexpectedErrorGettingAvoidListFile', {err}]);
+          }
+
+          const lines = res.toString().split(matchNewLines).map(trim);
+
+          return cbk(null, {lines: uniq(lines.filter(n => !!n))});
+        });
+      }],
+    },
+    returnResult({reject, resolve, of: 'getList'}, cbk));
+  });
+};

package/swaps/manage_rebalance.js

@@ -2,18 +2,18 @@ const asyncAuto = require('async/auto');
 const asyncFilter = require('async/filter');
 const asyncRetry = require('async/retry');
 const {getChannel} = require('ln-service');
+const {getWalletInfo} = require('ln-service');
 const {returnResult} = require('asyncjs-util');
 
 const appendFailingEdge = require('./append_failing_edge');
+const getAvoidList = require('./get_avoid_list');
 const rebalance = require('./rebalance');
+const writeAvoidList = require('./write_avoid_list');
 
 const channelFromEdge = edge => edge.slice(0, -2);
 const codeMissingChannel = 404;
 const {isArray} = Array;
 const isEdge = n => /^\d*x\d*x\d*x(0|1)*$/.test(n);
-const joinWithNewLines = lines => lines.join('\n');
-const matchNewLines = /\r?\n/;
-const uniq = arr => Array.from(new Set(arr));
 
 /** Manage rebalance attempts
 
@@ -24,6 +24,7 @@ const uniq = arr => Array.from(new Set(arr));
     fs: {
       appendFile: <Append to File Function> (path, content, cbk) => {}
       getFile: <Read File Contents Function> (path, cbk) => {}
+      renameFile: <Rename File Function> (from, to, cbk) => {}
       writeFile: <Write File Contents Function> (path, contents, cbk) => {}
     }
     [in_filters]: [<Inbound Filter Formula String>]
@@ -76,31 +77,28 @@ module.exports = (args, cbk) => {
         return cbk();
       },
 
-      // Get the avoid list and add it to the total avoids
+      // Get the avoid directives from the avoid list file
       getAvoids: ['validate', ({}, cbk) => {
         // Exit early when there is no ignore list
         if (!args.avoid_list) {
-          return cbk(null, {avoid: args.avoid, file: []});
+          return cbk(null, {lines: []});
         }
 
-        return args.fs.getFile(args.avoid_list, (err, res) => {
-          if (!!err) {
-            return cbk([500, 'UnexpectedErrorFetchingAvoidList', {err}]);
-          }
-
-          const original = res.toString();
+        return getAvoidList({
+          fs: {getFile: args.fs.getFile},
+          path: args.avoid_list,
+        },
+        cbk);
+      }],
 
-          const file = uniq(original
-            .split(matchNewLines)
-            .map(line => line.trim())
-            .filter(line => !!line.length));
+      // Get the graph sync status to know if missing channels are reliable
+      getGraphSyncStatus: ['validate', ({}, cbk) => {
+        // Exit early when there is no ignore list to clean
+        if (!args.avoid_list) {
+          return cbk();
+        }
 
-          return cbk(null, {
-            file,
-            original,
-            avoid: file.concat(args.avoid || []),
-          });
-        });
+        return getWalletInfo({lnd: args.lnd}, cbk);
       }],
 
       // Create failing edge logger for avoid appending
@@ -133,8 +131,17 @@ module.exports = (args, cbk) => {
       }],
 
       // Look at all of the lines in the file and clean them up
-      getCleanAvoids: ['getAvoids', ({getAvoids}, cbk) => {
-        return asyncFilter(getAvoids.file, (line, cbk) => {
+      getCleanAvoids: [
+        'getAvoids',
+        'getGraphSyncStatus',
+        ({getAvoids, getGraphSyncStatus}, cbk) =>
+      {
+        // Exit early with no info when missing channels may be resurrected
+        if (!!getGraphSyncStatus && !getGraphSyncStatus.is_synced_to_graph) {
+          return cbk();
+        }
+
+        return asyncFilter(getAvoids.lines, (line, cbk) => {
           // Exit early when not looking at an edge
           if (!isEdge(line)) {
             return cbk(null, true);
@@ -186,7 +193,7 @@ module.exports = (args, cbk) => {
         cbk => {
           return rebalance({
             start,
-            avoid: getAvoids.avoid,
+            avoid: getAvoids.lines.concat(args.avoid || []),
             fs: args.fs,
             in_filters: args.in_filters,
             in_outbound: args.in_outbound,
@@ -208,31 +215,58 @@ module.exports = (args, cbk) => {
         cbk);
       }],
 
-      // Write a cleaned up avoid list
-      writeCleanAvoidList: [
+      // Determine which lines should be removed from the avoid list
+      removals: [
         'getAvoids',
         'getCleanAvoids',
         ({getAvoids, getCleanAvoids}, cbk) =>
       {
-        // Exit early when there is no ignore list
-        if (!args.avoid_list) {
-          return cbk(null, {avoid: args.avoid, file: []});
+        // Exit early with no removals when the clean check had no answer
+        if (!getCleanAvoids) {
+          return cbk(null, []);
         }
 
-        const cleaned = joinWithNewLines(getCleanAvoids);
+        // The clean avoids are the original lines minus missing channels
+        const keeping = new Set(getCleanAvoids);
 
-        // Exit early when the clean file is identical to the original
-        if (getAvoids.original === cleaned) {
+        // A line that was not kept refers to a channel absent from the graph
+        return cbk(null, getAvoids.lines.filter(n => !keeping.has(n)));
+      }],
+
+      // Get the avoid list again to preserve concurrently appended lines
+      getCurrentAvoids: ['removals', ({removals}, cbk) => {
+        // Exit early when no lines were cleaned out of the avoid list
+        if (!removals.length) {
           return cbk();
         }
 
-        return args.fs.writeFile(args.avoid_list, cleaned, err => {
-          if (!!err) {
-            return cbk([503, 'UnexpectedErrorWritingCleanedAvoidList', {err}]);
-          }
+        return getAvoidList({
+          fs: {getFile: args.fs.getFile},
+          path: args.avoid_list,
+        },
+        cbk);
+      }],
 
+      // Write a cleaned up avoid list
+      writeCleanAvoidList: [
+        'getCurrentAvoids',
+        'removals',
+        ({getCurrentAvoids, removals}, cbk) =>
+      {
+        // Exit early when there is no cleaned avoid list to write out
+        if (!getCurrentAvoids) {
           return cbk();
-        });
+        }
+
+        const removing = new Set(removals);
+
+        // Atomically write the avoid list without the cleaned out lines
+        return writeAvoidList({
+          fs: {renameFile: args.fs.renameFile, writeFile: args.fs.writeFile},
+          lines: getCurrentAvoids.lines.filter(n => !removing.has(n)),
+          path: args.avoid_list,
+        },
+        cbk);
       }],
     },
     returnResult({reject, resolve, of: 'rebalance'}, cbk));

package/swaps/write_avoid_list.js

@@ -0,0 +1,68 @@
+const asyncAuto = require('async/auto');
+const {returnResult} = require('asyncjs-util');
+
+const {isArray} = Array;
+const joinWithNewLines = lines => lines.join('\n');
+const tempFile = path => `${path}.${process.pid}`;
+
+/** Atomically write the lines of an avoid list file
+
+  The list is written to a temporary file that is swapped into place so that
+  concurrent readers of the list never see a partially written file.
+
+  {
+    fs: {
+      renameFile: <Rename File Function> (from, to, cbk) => {}
+      writeFile: <Write File Contents Function> (path, contents, cbk) => {}
+    }
+    lines: [<Avoid Directive Line String>]
+    path: <Path To Avoid List File String>
+  }
+
+  @returns via cbk or Promise
+*/
+module.exports = ({fs, lines, path}, cbk) => {
+  return new Promise((resolve, reject) => {
+    return asyncAuto({
+      // Check arguments
+      validate: cbk => {
+        if (!fs) {
+          return cbk([400, 'ExpectedFilesystemMethodsToWriteAvoidList']);
+        }
+
+        if (!isArray(lines)) {
+          return cbk([400, 'ExpectedAvoidLinesToWriteAvoidList']);
+        }
+
+        if (!path) {
+          return cbk([400, 'ExpectedPathToAvoidListFileToWriteAvoidList']);
+        }
+
+        return cbk();
+      },
+
+      // Write the list to a temporary file next to the avoid list
+      write: ['validate', ({}, cbk) => {
+        return fs.writeFile(tempFile(path), joinWithNewLines(lines), err => {
+          if (!!err) {
+            return cbk([503, 'UnexpectedErrorWritingAvoidList', {err}]);
+          }
+
+          return cbk();
+        });
+      }],
+
+      // Atomically swap the temporary file in place of the avoid list
+      swap: ['write', ({}, cbk) => {
+        return fs.renameFile(tempFile(path), path, err => {
+          if (!!err) {
+            return cbk([503, 'UnexpectedErrorReplacingAvoidList', {err}]);
+          }
+
+          return cbk();
+        });
+      }],
+    },
+    returnResult({reject, resolve}, cbk));
+  });
+};

package/test/swaps/test_get_avoid_list.js

@@ -0,0 +1,62 @@
+const {deepEqual} = require('node:assert').strict;
+const {rejects} = require('node:assert').strict;
+const test = require('node:test');
+
+const getAvoidList = require('./../../swaps/get_avoid_list.js');
+
+const makeFs = ({content, err} = {}) => {
+  return {getFile: (path, cbk) => cbk(err, Buffer.from(content || ''))};
+};
+
+const makeArgs = override => {
+  const args = {
+    fs: makeFs({content: '\n1x1x1x0\n b \n\na\r\n1x1x1x0\nc'}),
+    path: '/tmp/avoid',
+  };
+
+  Object.keys(override).forEach(key => args[key] = override[key]);
+
+  return args;
+};
+
+const tests = [
+  {
+    args: makeArgs({fs: undefined}),
+    description: 'Filesystem methods are required',
+    error: [400, 'ExpectedFilesystemMethodsToGetAvoidList'],
+  },
+  {
+    args: makeArgs({path: undefined}),
+    description: 'An avoid list path is required',
+    error: [400, 'ExpectedPathToAvoidListFileToGetAvoidList'],
+  },
+  {
+    args: makeArgs({fs: makeFs({err: new Error('Failed to read')})}),
+    description: 'A read error is returned',
+    error: [503, 'UnexpectedErrorGettingAvoidListFile'],
+  },
+  {
+    args: makeArgs({fs: makeFs({})}),
+    description: 'An empty file has no lines',
+    expected: {lines: []},
+  },
+  {
+    args: makeArgs({}),
+    description: 'Lines are trimmed, deduplicated, and blanks are removed',
+    expected: {lines: ['1x1x1x0', 'b', 'a', 'c']},
+  },
+];
+
+tests.forEach(({args, description, error, expected}) => {
+  return test(description, async () => {
+    if (!!error) {
+      await rejects(getAvoidList(args), error, 'Got expected error');
+    } else {
+      const result = await getAvoidList(args);
+
+      deepEqual(result, expected, 'Got expected result');
+    }
+
+    return;
+  });
+});

package/test/swaps/test_write_avoid_list.js

@@ -0,0 +1,83 @@
+const {deepEqual} = require('node:assert').strict;
+const {rejects} = require('node:assert').strict;
+const test = require('node:test');
+
+const writeAvoidList = require('./../../swaps/write_avoid_list.js');
+
+const temp = `/tmp/avoid.${process.pid}`;
+
+const makeFs = ({renameErr, writeErr} = {}) => {
+  const fs = {
+    calls: [],
+    renameFile: (from, to, cbk) => {
+      fs.calls.push({from, to});
+
+      return cbk(renameErr);
+    },
+    writeFile: (path, contents, cbk) => {
+      fs.calls.push({contents, path});
+
+      return cbk(writeErr);
+    },
+  };
+
+  return fs;
+};
+
+const makeArgs = override => {
+  const args = {fs: makeFs({}), lines: ['a', 'b'], path: '/tmp/avoid'};
+
+  Object.keys(override).forEach(key => args[key] = override[key]);
+
+  return args;
+};
+
+const tests = [
+  {
+    args: makeArgs({fs: undefined}),
+    description: 'Filesystem methods are required',
+    error: [400, 'ExpectedFilesystemMethodsToWriteAvoidList'],
+  },
+  {
+    args: makeArgs({lines: undefined}),
+    description: 'Avoid lines are required',
+    error: [400, 'ExpectedAvoidLinesToWriteAvoidList'],
+  },
+  {
+    args: makeArgs({path: undefined}),
+    description: 'An avoid list path is required',
+    error: [400, 'ExpectedPathToAvoidListFileToWriteAvoidList'],
+  },
+  {
+    args: makeArgs({fs: makeFs({writeErr: new Error('WriteFailed')})}),
+    description: 'A write error is returned',
+    error: [503, 'UnexpectedErrorWritingAvoidList'],
+  },
+  {
+    args: makeArgs({fs: makeFs({renameErr: new Error('RenameFailed')})}),
+    description: 'A rename error is returned',
+    error: [503, 'UnexpectedErrorReplacingAvoidList'],
+  },
+  {
+    args: makeArgs({}),
+    calls: [
+      {contents: 'a\nb', path: temp},
+      {from: temp, to: '/tmp/avoid'},
+    ],
+    description: 'The list is written to a temp file and swapped in place',
+  },
+];
+
+tests.forEach(({args, calls, description, error}) => {
+  return test(description, async () => {
+    if (!!error) {
+      await rejects(writeAvoidList(args), error, 'Got expected error');
+    } else {
+      await writeAvoidList(args);
+
+      deepEqual(args.fs.calls, calls, 'Got expected filesystem calls');
+    }
+
+    return;
+  });
+});