@bitpay-labs/bitcore-wallet-service 11.5.2

package/scripts/walletStats.js

@@ -0,0 +1,290 @@
+#!/usr/bin/env node
+
+import fs from 'fs';
+import os from 'os';
+import readline from 'readline';
+import * as CWC from '@bitpay-labs/crypto-wallet-core';
+import Moralis from 'moralis';
+import request from 'request-promise-native';
+import { Storage } from '../../ts_build/src';
+import config from '../../ts_build/src/config';
+import { WalletService } from '../../ts_build/src/lib/server';
+
+const rl = readline.createInterface({ input: process.stdin, output: process.stdout });
+
+let shuttingDown = false;
+process.on('SIGINT', function() {
+  if (shuttingDown) {
+    console.log('Forced exit');
+    process.exit(1);
+  }
+  shuttingDown = true;
+});
+
+function usage(errMsg) {
+  console.log('USAGE: ./walletStats.js [options]');
+  console.log('OPTIONS:');
+  console.log('  --start <value>         Start month (MM/YYYY) (default: last month)');
+  console.log('  --end <value>           End month (MM/YYYY) - inclusive (default: last month)');
+  console.log('  --chain <value>         Chain (default: all)');
+  console.log('  --network <value>       Network (default: livenet)');
+  console.log('  --out <value>           Output file (default: stats-<date>.json)');
+  console.log('  --config                Only change the sleep config file (walletStats.config.json) and exit');
+  console.log('  --sleep <value>         Sleep time in milliseconds between iterations. Default: 250');
+  console.log('  --every <value>         Sleep for --sleep milliseconds every --every loop iteration. Default: 100');
+  console.log('  ');
+  if (errMsg) {
+    console.log('\nERROR: ' + errMsg);
+  }
+  process.exit();
+};
+
+const args = process.argv.slice(2);
+
+if (args.includes('--help') || args.includes('-h')) {
+  usage();
+}
+
+
+const sleepIdx = args.indexOf('--sleep');
+let sleepMs = sleepIdx > -1 ? parseInt(args[sleepIdx + 1]) : 250;
+const everyIdx = args.indexOf('--every');
+let nSleep = everyIdx > -1 ? parseInt(args[everyIdx + 1]) : 100;
+fs.writeFileSync('./walletStats.config.json', JSON.stringify({ sleepMs, nSleep }, null, 2));
+if (args.includes('--config')) {
+  console.log('Config updated.');
+  process.exit(0);
+}
+
+
+const yestermonth = new Date();
+yestermonth.setUTCMonth(yestermonth.getUTCMonth() - 1); // Default to last month
+const lastMonth = (yestermonth.getUTCMonth() + 1).toString().padStart(2, '0');
+const year = yestermonth.getUTCFullYear();
+
+const startIdx = args.indexOf('--start');
+if (startIdx > -1 && (!args[startIdx + 1] || !args[startIdx + 1].match(/^(0[1-9]|1[0-2])\/20[0-9]{2}$/))) {
+  usage('Invalid start date format. Use MM/YYYY');
+}
+const start = startIdx > -1 ? args[startIdx + 1] : `${lastMonth}/${year}`;
+
+const endIdx = args.indexOf('--end');
+if (endIdx > -1 && (!args[endIdx + 1] || !args[endIdx + 1].match(/^(0[1-9]|1[0-2])\/20[0-9]{2}$/))) {
+  usage('Invalid end date format. Use MM/YYYY');
+}
+const end = endIdx > -1 ? args[endIdx + 1] : `${lastMonth}/${year}`;
+
+const chainIdx = args.indexOf('--chain');
+const chain = (chainIdx > -1 && args[chainIdx + 1]?.toLowerCase()) || null;
+
+const networkIdx = args.indexOf('--network');
+const network = (networkIdx > -1 && args[networkIdx + 1]?.toLowerCase()) || 'livenet';
+
+const outIdx = args.indexOf('--out');
+let outFile = (outIdx > -1 && args[outIdx + 1]) || `stats-${new Date().toISOString()}.json`;
+
+if (outFile.startsWith('~')) {
+  outFile = outFile.replace('~', os.homedir());
+}
+if (outFile.startsWith('$HOME')) {
+  outFile = outFile.replace('$HOME', os.homedir());
+}
+if (!outFile.startsWith('/') && !outFile.startsWith('./') && !outFile.startsWith('../')) {
+  outFile = __dirname + '/' + outFile;
+}
+
+const tokenChainMap = {
+  WBTC: 'btc',
+  WETH: 'eth',
+  MATIC: 'matic'
+};
+
+const storage = new Storage();
+storage.connect(config.storageOpts, async (err) => {
+  if (err) {
+    console.log(err);
+    return;
+  }
+  await new Promise((r, j) => WalletService.initialize(
+    {
+      ...config,
+      storage,
+      messageBroker: { onMessage: () => {} },
+      fiatRateService: {}
+    },
+    (err) => {
+      if (err) return j(err);
+      return r();
+    }
+  ));
+  const walletService = new WalletService();
+  
+  function done(err) {
+    if (err) { console.log(err); }
+    rl.close();
+    WalletService.shutDown((err) => {
+      if (err) { console.log(err); }
+      fs.rmSync('./walletStats.config.json');
+      console.log('done');
+    });
+    // storage.disconnect(() => { console.log('done'); });
+  }
+
+  try {
+    // await Moralis.start({
+    //   apiKey: config.moralis.apiKey
+    // });
+    // Get all wallets
+    const walletCnt = await storage.db.collection(Storage.collections.WALLETS).count({});
+    const walletStream = storage.db.collection(Storage.collections.WALLETS).find({});
+
+    console.log(`  Found ${Intl.NumberFormat().format(walletCnt)} total wallets to scan`);
+    console.log(`  Output file: ${outFile}`);
+    const ans = await new Promise(r => rl.question('Would you like to continue? (Y/n): ', r));
+    if (ans?.toLowerCase() === 'n') {
+      return done('Good bye.');
+    }
+    
+    const retval = {};
+    let count = 0;
+    let relevantWallets = 0;
+    let perfTime = Date.now();
+    let perfCnt = 0;
+
+
+    for await (const wallet of walletStream) {
+      if (shuttingDown) {
+        throw new Error('Interrupted by user');
+      }
+
+      count++;
+      if (count % nSleep === 0) {
+        const cntDiff = count - perfCnt;
+        const timeDiff = Date.now() - perfTime;
+        const rate = (cntDiff / timeDiff * 1000).toFixed(2); // wallets per second
+        process.stdout.write(`Processed ${(count / walletCnt * 100).toFixed(4)}% wallets (${count} | ${relevantWallets}) - ${rate} wallets/sec\r`); // shows how fast things are working
+        await new Promise(resolve => setTimeout(resolve, sleepMs)); // cooldown
+        ({ nSleep, sleepMs } = JSON.parse(fs.readFileSync('./walletStats.config.json', 'utf8'))); // reload config
+        perfTime = Date.now();
+        perfCnt = count;
+      }
+
+      const wChain = wallet.chain || wallet.coin || 'btc';
+      const [smonth, syear] = start.split('/');
+      const startDate = new Date(`${syear}-${smonth}-01T00:00:00Z`);
+
+
+      if (wallet.status !== 'complete') continue; // skip incomplete wallets
+      if (!wallet.beAuthPrivateKey2) continue; // skip too old wallets
+      if (chain && (
+        (wallet.chain?.toLowerCase() !== chain && wallet.coin?.toLowerCase() !== chain) ||
+        (!wallet.coin && chain !== 'btc') // legacy wallets default to btc
+      )) continue; // skip non-matching chain
+      if (wallet.network?.toLowerCase() !== network) continue;
+
+      relevantWallets++;
+
+      const [emonth, eyear] = end.split('/');
+      const endDate = new Date(`${eyear}-${emonth}-01T00:00:00Z`);
+
+      const tokenOpts = CWC.Constants[`${wChain.toUpperCase()}_TOKEN_OPTS`];
+      const dateKey = startDate.toISOString().split('T')[0];
+      if (!retval[dateKey]) {
+        retval[dateKey] = {};
+      }
+      if (!retval[dateKey][wChain]) {
+        retval[dateKey][wChain] = { wallets: 0n, totalBalance: 0n };
+      }
+
+      try {
+        const bc = walletService._getBlockchainExplorer(wChain, wallet.network);
+        const client = bc._getAuthClient(wallet);
+        // todo: is there a quick check to see if the wallet has _any_ activity?
+        while (startDate <= endDate) {
+          let balance = 0;
+          if (['btc', 'bch', 'ltc', 'doge', 'xrp'].includes(wChain)) {
+            const query = '';
+            const apiUrl = `${client.baseUrl}/wallet/${wallet.beAuthPublicKey2}/balance/${startDate.toISOString()}`;
+            const url = apiUrl + query;
+            const signature = client.sign({ method: 'GET', url });
+            const response = await request.get(url, {
+              headers: { 'x-signature': signature },
+              json: true
+            });
+
+            if (!response.balance) {
+              startDate.setUTCMonth(startDate.getUTCMonth() + 1);
+              continue;
+            }
+            balance = response.balance;
+          } else if (wChain === 'sol') {
+            // todo
+            continue;
+          } else { // EVM
+            const CHAIN_IDS = CWC.Constants.EVM_CHAIN_NETWORK_TO_CHAIN_ID;
+            const chainKey = `${wChain.toUpperCase()}_${wallet.network === 'livenet' ? 'mainnet' : wallet.network}`;
+            if (!CHAIN_IDS[chainKey]) {
+              throw new Error(`Unsupported chain/network: ${chainKey}`);
+            }
+            const chainId = CHAIN_IDS[chainKey];
+
+            const addresses = await storage.db.collection(Storage.collections.ADDRESSES).find({ walletId: wallet.id, createdOn: { $exists: true } }).toArray();
+            if (!addresses[0]) {
+              break; // no addresses, skip
+            }
+
+            const blkResponse = await Moralis.EvmApi.block.getDateToBlock({
+              chain: chainId,
+              date: startDate.toISOString()
+            });
+
+            const balResponse = await Moralis.EvmApi.wallets.getWalletTokenBalancesPrice({
+              chain: chainId,
+              toBlock: blkResponse.raw.block,
+              excludeSpam: true,
+              address: addresses[0].address // assume first address is representative of whole wallet
+            });
+
+            for (const token of balResponse.json.result) {
+              if (token.native_token) {
+                balance = BigInt(token.balance);
+              } else {
+                const tokenOpt = tokenOpts[token.token_address.toLowerCase()];
+                const tChain = tokenChainMap[tokenOpt?.symbol];
+                if (tChain) {
+                  retval[dateKey][tChain] = retval[dateKey][tChain] || { wallets: 0n, totalBalance: 0n };
+                  retval[dateKey][tChain].wallets++;
+                  retval[dateKey][tChain].totalBalance += BigInt(token.balance);
+                } else {
+                  // record unknown/stablecoin token's USD values
+                  if (!retval[dateKey]['USD']) {
+                    retval[dateKey]['USD'] = { wallets: 0n, totalBalance: 0n };
+                  }
+                  retval[dateKey]['USD'].wallets++;
+                  retval[dateKey]['USD'].totalBalance += BigInt(Math.round(token.usd_value));
+                }
+              }
+            }
+          }
+
+          retval[dateKey][wChain].wallets++;
+          retval[dateKey][wChain].totalBalance += BigInt(balance);
+
+          startDate.setUTCMonth(startDate.getUTCMonth() + 1);
+        }
+
+        fs.writeFileSync(outFile, JSON.stringify(retval, (k, v) => typeof v === 'bigint' ? v.toString() : v, 2));
+      } catch (e) {
+        console.log(`\nError fetching data for wallet ${wallet.id} (${wChain}/${wallet.network}): ${e.stack}`);
+      }
+    }
+
+    process.stdout.write('\n');
+
+    console.log(`Wrote stats for ${relevantWallets} wallets to ${outFile}`);
+
+    return done();
+  } catch (err) {
+    return done(err);
+  }
+});

package/src/app.ts

@@ -0,0 +1,28 @@
+#!/usr/bin/env node
+
+import { spawn } from 'child_process';
+import async from 'async';
+
+const scripts = [
+  'locker/locker.js',
+  'messagebroker/messagebroker.js',
+  'bcmonitor/bcmonitor.js',
+  'emailservice/emailservice.js',
+  'pushnotificationsservice/pushnotificationsservice.js',
+  'fiatrateservice/fiatrateservice.js',
+  'bws.js'
+];
+
+async.eachSeries(scripts, function(script, callback) {
+  console.log(`Spawning ${script}`);
+
+  const node = spawn('node', [script]);
+  node.stdout.on('data', data => {
+    console.log(`${data}`);
+  });
+  node.stderr.on('data', data => {
+    console.error(`${data}`);
+  });
+
+  callback();
+});

package/src/bcmonitor/bcmonitor.ts

@@ -0,0 +1,11 @@
+#!/usr/bin/env node
+import config from '../config';
+import { BlockchainMonitor } from '../lib/blockchainmonitor';
+import logger from '../lib/logger';
+
+const bcm = new BlockchainMonitor();
+bcm.start(config, err => {
+  if (err) throw err;
+
+  logger.info('Blockchain monitor started');
+});

package/src/bws.ts

@@ -0,0 +1,85 @@
+#!/usr/bin/env node
+import cluster from 'cluster';
+import * as fs from 'fs';
+import 'source-map-support/register';
+import os from 'os';
+import config from './config';
+import { ExpressApp } from './lib/expressapp';
+import logger from './lib/logger';
+
+const port = process.env.BWS_PORT || config.port || 3232;
+// eslint-disable-next-line @typescript-eslint/no-require-imports
+const serverModule = config.https ? require('https') : require('http');
+
+const serverOpts: {
+  key?: Buffer;
+  cert?: Buffer;
+  ciphers?: string[];
+  honorCipherOrder?: boolean;
+  ca?: Buffer[];
+} = {};
+
+if (config.https) {
+  serverOpts.key = fs.readFileSync(config.privateKeyFile || './ssl/privatekey.pem');
+  serverOpts.cert = fs.readFileSync(config.certificateFile || './ssl/certificate.pem');
+  if (config.ciphers) {
+    serverOpts.ciphers = config.ciphers;
+    serverOpts.honorCipherOrder = true;
+  }
+
+  // This sets the intermediate CA certs only if they have all been designated in the config.js
+  if (config.CAinter1 && config.CAinter2 && config.CAroot) {
+    serverOpts.ca = [
+      fs.readFileSync(config.CAinter1),
+      fs.readFileSync(config.CAinter2),
+      fs.readFileSync(config.CAroot)
+    ];
+  }
+}
+
+if (config.cluster && !config.messageBrokerOpts.messageBrokerServer)
+  throw new Error('When running in cluster mode, message broker server need to be configured');
+
+const expressApp = new ExpressApp();
+
+function startInstance() {
+  const server = config.https
+    ? serverModule.createServer(serverOpts, expressApp.app)
+    : serverModule.Server(expressApp.app);
+
+  expressApp.start(config, err => {
+    if (err) {
+      logger.error('Could not start BWS instance: %o', err);
+      return;
+    }
+
+    server.listen(port);
+
+    const instanceInfo = cluster.worker ? ' [Instance:' + cluster.worker.id + ']' : '';
+    logger.info('BWS running ' + instanceInfo);
+    return;
+  });
+}
+
+if (config.cluster && cluster.isMaster) {
+  // Count the machine's CPUs
+  const instances = config.clusterInstances || os.cpus().length;
+
+  logger.info('Starting ' + instances + ' instances');
+
+  // Create a worker for each CPU
+  for (let i = 0; i < instances; i += 1) {
+    cluster.fork();
+  }
+
+  // Listen for dying workers
+  cluster.on('exit', worker => {
+    // Replace the dead worker,
+    logger.error('Worker ' + worker.id + ' died :(');
+    cluster.fork();
+  });
+  // Code to run if we're in a worker process
+} else {
+  logger.info('Listening on port: ' + port);
+  startInstance();
+}

package/src/cleanfiatrates/cleanfiatrates.ts

@@ -0,0 +1,9 @@
+#!/usr/bin/env node
+
+import { CleanFiatRates } from '../lib/cleanfiatrates';
+
+const cleanFiatRatesScript = new CleanFiatRates();
+cleanFiatRatesScript.run(err => {
+  if (err) throw err;
+  console.log('Clean fiat rates script finished');
+});