@bitpay-labs/bitcore-wallet-service 11.5.2

package/scripts/getSolStats2.js

@@ -0,0 +1,81 @@
+#!/usr/bin/env node
+import fs from 'fs';
+import request from 'request-promise-native';
+import { Storage } from '../../ts_build/src';
+import config from '../../ts_build/src/config';
+
+const format = (bal, num) => Intl.NumberFormat('en-US', { minimumFractionDigits: num }).format(bal / Math.pow(10, num));
+
+const storage = new Storage();
+storage.connect(config.storageOpts, async (err) => {
+  if (err) {
+    console.log(err);
+    return;
+  }
+  
+  function done(err) {
+    if (err) { console.log(err); }
+    storage.disconnect(() => { console.log('done'); });
+  }
+
+  try {
+    const walletIds = fs.readFileSync('./output-getSolStats.txt', 'utf8').split('\n').filter(Boolean);
+    const walletCnt = walletIds.length;
+
+    let count = 0;
+    let countNoAddy = 0;
+    let totalCount = 0;
+    let balanceSOL = 0;
+    let balanceUSDC = 0;
+    let balanceUSDT = 0;
+    for (const walletId of walletIds) {
+      await new Promise(resolve => setTimeout(resolve, 80)); // sleep 80ms
+
+      totalCount++;
+      if (totalCount % 100 === 0) {
+        process.stdout.write(`Processed ${count}/${totalCount}/${walletCnt} wallets (${(totalCount / walletCnt * 100).toFixed(1)}%)` +
+          ` -- ${format(balanceSOL, 9)}/${format(balanceUSDC, 6)}/${format(balanceUSDT, 6)}\r`); // shows how fast things are working
+      }
+
+      count++;
+      const address = await storage.db.collection(Storage.collections.ADDRESSES).findOne({ walletId: walletId });
+      if (!address) {
+        countNoAddy++;
+        continue;
+      }
+ 
+      const [res, resUSDC, resUSDT] = await Promise.all([
+        request(`https://api-sol.bitcore.io/api/SOL/mainnet/address/${address.address}/balance`, { json: true }),
+        request(`https://api-sol.bitcore.io/api/SOL/mainnet/address/${address.address}/balance?tokenAddress=EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v`, { json: true })
+          .catch(err => {
+            if (err.message.includes('ATA not initialized')) {
+              return { balance: 0 };
+            }
+            throw err;
+          }),
+        request(`https://api-sol.bitcore.io/api/SOL/mainnet/address/${address.address}/balance?tokenAddress=Es9vMFrzaCERmJfrF4H2FYD4KCoNkY11McCe8BenwNYB`, { json: true })
+          .catch(err => {
+            if (err.message.includes('ATA not initialized')) {
+              return { balance: 0 };
+            }
+            throw err;
+          })
+      ]);
+      balanceSOL += parseInt(res.balance);
+      balanceUSDC += parseInt(resUSDC.balance);
+      balanceUSDT += parseInt(resUSDT.balance);
+    }
+    process.stdout.write('\n');
+
+    console.log('Wallets with no address: ' + countNoAddy);
+    console.log('Wallets: ' + count);
+    console.log('Balance: ' + format(balanceSOL, 9) + ' SOL');
+    console.log('Balance: ' + format(balanceUSDC, 6) + ' USDC');
+    console.log('Balance: ' + format(balanceUSDT, 6) + ' USDT');
+
+
+    return done();
+  } catch (err) {
+    return done(err);
+  }
+});

package/scripts/migrateToNamedTestnets.js

@@ -0,0 +1,191 @@
+#!/usr/bin/env node
+import pkg from '../../ts_build/src';
+import config from '../../ts_build/src/config';
+
+const { Storage } = pkg;
+
+const startDate = new Date('2011-01-01T00:00:00.000Z');
+const endDate = new Date();
+
+const networkMapping = { 
+  eth: {
+    testnet: 'sepolia'
+  },
+  btc: {
+    testnet: 'testnet3'
+  },
+  bch: {
+    testnet: 'testnet3'
+  },
+  doge: {
+    testnet: 'testnet3'
+  },
+  ltc: {
+    testnet: 'testnet4',
+  },
+  matic: {
+    testnet: 'amoy'
+  },
+  arb: {
+    testnet: 'sepolia'
+  },
+  base: {
+    testnet: 'sepolia'
+  },
+  op: {
+    testnet: 'sepolia'
+  }
+
+};
+
+const doit = process.argv[2] === '--doit';
+
+if (!doit) {
+  console.log('Dry run (pass --doit to actually do it)');
+} else {
+  console.log('LET\'S DO IT FOR REAL');
+}
+
+const storage = new Storage();
+storage.connect(config.storageOpts, async (err) => {
+  if (err) {
+    console.log(err);
+    return;
+  }
+  
+  function done(err) {
+    if (err) { console.log(err); }
+    storage.disconnect(() => { console.log('done'); });
+  }
+
+  try {
+    // Get all wallets
+    const walletCnt = await storage.db.collection(Storage.collections.WALLETS).count({});
+    const walletStream = storage.db.collection(Storage.collections.WALLETS).find({});
+
+    let fixAddressCount = 0;
+    let fixWalletsCount = 0;
+    let fixTxsCount = 0;
+    let skipCountMainnet = 0;
+    let skipCountOther = 0;
+    let count = 0;
+
+    console.log(`  ${doit ? 'REAL:' : 'DRY RUN:'} Found ${Intl.NumberFormat().format(walletCnt)} total wallets to scan`);
+    console.log('  Pausing 10s for effect...');
+    await new Promise(resolve => setTimeout(resolve, 10000)); // sleep 10s
+
+    for await (const wallet of walletStream) {
+      count++;
+      if (count % 100 === 0) {
+        console.log(`Processed ${(count / walletCnt * 100).toFixed(4)}% wallets`); // shows how fast things are working
+        await new Promise(resolve => setTimeout(resolve, 250)); // cooldown
+      }
+      // if wallet chain is not covered or if network isn't testnet then skip
+      if (!(Object.keys(networkMapping).includes(wallet.chain || wallet.coin) && wallet.network == 'testnet')) {
+        if (wallet.network === 'livenet' || wallet.network === 'mainnet') {
+          skipCountMainnet++;
+        } else {
+          skipCountOther++;
+        }
+        continue;
+      }
+
+      if (doit) {
+        // Update Wallets collection
+        const resWallet = await storage.db.collection(Storage.collections.WALLETS).updateMany({
+          id: wallet.id,
+          network: 'testnet'
+        }, {
+          $set: { network: networkMapping[wallet.chain || wallet.coin].testnet }
+        });
+
+        if (resWallet?.result?.nModified > 0) {
+          fixWalletsCount++;
+        } else if (!resWallet?.result?.ok) {
+          console.log(JSON.stringify(resWallet));
+        }
+
+        // Update Addresses collection
+        const resAddress = await storage.db.collection(Storage.collections.ADDRESSES).updateMany({
+          walletId: wallet.id,
+          createdOn: { $gte: startDate.getTime() / 1000, $lte: endDate.getTime() / 1000 }, // Included only to use index
+          network: 'testnet'
+        }, {
+          $set: { network: networkMapping[wallet.chain || wallet.coin].testnet }
+        });
+
+        if (resAddress?.result?.nModified > 0) {
+          fixAddressCount++;
+        } else if (!resAddress?.result?.ok) {
+          console.log(JSON.stringify(resAddress));
+        }
+
+        // Update Wallets collection
+        const resTxs = await storage.db.collection(Storage.collections.TXS).updateMany({
+          walletId: wallet.id,
+          createdOn: { $gte: startDate.getTime() / 1000, $lte: endDate.getTime() / 1000 }, // Included only to use index
+          network: 'testnet'
+        }, {
+          $set: { network: networkMapping[wallet.chain || wallet.coin].testnet }
+        });
+
+        // Also update changeAddress and escrowAddress network if they exist
+        await storage.db.collection(Storage.collections.TXS).updateMany({
+          walletId: wallet.id,
+          createdOn: { $gte: startDate.getTime() / 1000, $lte: endDate.getTime() / 1000 }, // Included only to use index
+          'changeAddress.network': 'testnet'
+        }, {
+          $set: { 'changeAddress.network': networkMapping[wallet.chain || wallet.coin].testnet }
+        });
+        await storage.db.collection(Storage.collections.TXS).updateMany({
+          walletId: wallet.id,
+          createdOn: { $gte: startDate.getTime() / 1000, $lte: endDate.getTime() / 1000 }, // Included only to use index
+          'escrowAddress.network': 'testnet'
+        }, {
+          $set: { 'escrowAddress.network': networkMapping[wallet.chain || wallet.coin].testnet }
+        });
+
+        if (resTxs?.result?.nModified > 0) {
+          fixTxsCount++;
+        } else if (!resTxs?.result?.ok) {
+          console.log(JSON.stringify(resTxs));
+        }
+      } else {
+        // Update Wallets collection
+        const walletCount = await storage.db.collection(Storage.collections.WALLETS).countDocuments({
+          id: wallet.id,
+          network: 'testnet'
+        });
+        fixWalletsCount += walletCount;
+
+        // Update Addresses collection
+        const addressCount = await storage.db.collection(Storage.collections.ADDRESSES).countDocuments({
+          walletId: wallet.id,
+          createdOn: { $gte: startDate.getTime() / 1000, $lte: endDate.getTime() / 1000 }, // Included only to use index
+          network: 'testnet'
+        });
+
+        fixAddressCount+= addressCount;
+
+        // Update Wallets collection
+        const txsCount = await storage.db.collection(Storage.collections.TXS).countDocuments({
+          walletId: wallet.id,
+          createdOn: { $gte: startDate.getTime() / 1000, $lte: endDate.getTime() / 1000 }, // Included only to use index
+          network: 'testnet'
+        });
+
+        fixTxsCount += txsCount;
+      }
+    }
+
+    console.log(`Fixed ${fixWalletsCount} wallets`);
+    console.log(`Fixed ${fixAddressCount} Addresses`);
+    console.log(`Fixed ${fixTxsCount} Txs`);
+    console.log('Mainnet wallets:', skipCountMainnet);
+    console.log('Other network wallets:', skipCountOther);
+
+    return done();
+  } catch (err) {
+    return done(err);
+  }
+});

package/scripts/migrateWalletsNetwork.js

@@ -0,0 +1,204 @@
+#!/usr/bin/env node
+
+import fs from 'fs';
+import os from 'os';
+import readline from 'readline';
+import config from '../ts_build/src/config';
+import { Storage } from '../ts_build/src/lib/storage';
+
+const rl = readline.createInterface({ input: process.stdin, output: process.stdout });
+
+const startDate = new Date('2011-01-01T00:00:00.000Z');
+const endDate = new Date();
+
+function usage(errMsg) {
+  console.log('USAGE: ./migrateWalletsNetwork.js [options]');
+  console.log('OPTIONS:');
+  console.log('  --chain <value>         REQUIRED - e.g. BTC, BCH, DOGE, LTC...');
+  console.log('  --oldNetwork <value>    REQUIRED - e.g. testnet3');
+  console.log('  --newNetwork <value>    REQUIRED - e.g. testnet4');
+  console.log('  --doit                  Save the migration to the db');
+  console.log('  --out <value>           Output file (default: <chain>-<oldNetwork>-<newNetwork>-migrate.json)');
+  if (errMsg) {
+    console.log('\nERROR: ' + errMsg);
+  }
+  process.exit();
+}
+
+const args = process.argv.slice(2);
+
+if (args.includes('--help') || args.includes('-h')) {
+  usage();
+}
+
+const chainIdx = args.indexOf('--chain');
+const oldNetworkIdx = args.indexOf('--oldNetwork');
+const newNetworkIdx = args.indexOf('--newNetwork');
+const chain = args[chainIdx + 1]?.toLowerCase(); // wallet.chain/coin is lowercase
+const oldNetwork = args[oldNetworkIdx + 1]?.toLowerCase();
+const newNetwork = args[newNetworkIdx + 1]?.toLowerCase();
+
+if (
+  chainIdx === -1 ||
+  oldNetworkIdx === -1 ||
+  newNetworkIdx === -1 ||
+  !chain ||
+  !oldNetwork ||
+  !newNetwork
+) {
+  usage('Missing required options.');
+}
+
+const outIdx = args.indexOf('--out');
+let outFile = (outIdx > -1 && args[outIdx + 1]) || `${chain}-${oldNetwork}-${newNetwork}-migrate.json`;
+
+if (outFile.startsWith('~')) {
+  outFile = outFile.replace('~', os.homedir());
+}
+if (outFile.startsWith('$HOME')) {
+  outFile = outFile.replace('$HOME', os.homedir());
+}
+if (!outFile.startsWith('/') && !outFile.startsWith('./') && !outFile.startsWith('../')) {
+  outFile = __dirname + '/' + outFile;
+}
+
+const doit = args.includes('--doit');
+
+if (!doit) {
+  console.log('Dry run (pass --doit to actually do it)');
+} else {
+  console.log('LET\'S DO IT FOR REAL');
+}
+
+const storage = new Storage();
+storage.connect(config.storageOpts, async (err) => {
+  if (err) {
+    console.log(err);
+    return;
+  }
+  
+  function done(err) {
+    if (err) { console.log(err); }
+    rl.close();
+    storage.disconnect(() => { console.log('done'); });
+  }
+
+  try {
+    // Get all wallets
+    const walletCnt = await storage.db.collection(Storage.collections.WALLETS).count({});
+    const walletStream = storage.db.collection(Storage.collections.WALLETS).find({});
+
+    let fixAddressCount = 0;
+    let fixWalletsCount = 0;
+    let fixTxsCount = 0;
+    let count = 0;
+
+    console.log(`  ${doit ? 'REAL:' : 'DRY RUN:'} Found ${Intl.NumberFormat().format(walletCnt)} total wallets to scan`);
+    console.log(`  Migrating ${chain}:${oldNetwork}->${newNetwork}`);
+    console.log(`  Output file: ${outFile}`);
+    const ans = await new Promise(r => rl.question('Would you like to continue? (y/N): ', r));
+    if (ans?.toLowerCase() !== 'y') {
+      return done('Good bye.');
+    }
+
+    for await (const wallet of walletStream) {
+      count++;
+      if (count % 100 === 0) {
+        process.stdout.write(`Processed ${(count / walletCnt * 100).toFixed(4)}% wallets (${count}) - fixed ${fixWalletsCount}\r`); // shows how fast things are working
+        await new Promise(resolve => setTimeout(resolve, 250)); // cooldown
+      }
+      // if wallet chain is not covered or if network isn't testnet then skip
+      if (wallet.chain !== chain || (!wallet.chain && wallet.coin !== chain) || wallet.network !== oldNetwork) {
+        continue;
+      }
+
+      fs.appendFileSync(outFile, wallet.id + '\n');
+
+      if (doit) {
+        // Update Wallets collection
+        const resWallet = await storage.db.collection(Storage.collections.WALLETS).updateMany({
+          id: wallet.id,
+          network: oldNetwork
+        }, {
+          $set: { network: newNetwork }
+        });
+
+        if (resWallet?.result?.nModified > 0) {
+          fixWalletsCount++;
+        } else if (!resWallet?.result?.ok) {
+          console.log(JSON.stringify(resWallet));
+        }
+
+        // Update Addresses collection
+        const resAddress = await storage.db.collection(Storage.collections.ADDRESSES).updateMany({
+          walletId: wallet.id,
+          createdOn: { $gte: startDate.getTime() / 1000, $lte: endDate.getTime() / 1000 }, // Included only to use index
+          network: oldNetwork
+        }, {
+          $set: { network: newNetwork }
+        });
+
+        if (resAddress?.result?.nModified > 0) {
+          fixAddressCount++;
+        } else if (!resAddress?.result?.ok) {
+          console.log(JSON.stringify(resAddress));
+        }
+
+        // Delete Txs for old network
+        const resTxs = await storage.db.collection(Storage.collections.TXS).deleteMany({
+          walletId: wallet.id,
+          createdOn: { $gte: startDate.getTime() / 1000, $lte: endDate.getTime() / 1000 }, // Included only to use index
+          network: oldNetwork
+        });
+
+        if (resTxs?.result?.n > 0) {
+          fixTxsCount++;
+        } else if (!resTxs?.result?.ok) {
+          console.log(JSON.stringify(resTxs));
+        }
+
+        const resCache = await storage.db.collection(Storage.collections.CACHE).deleteMany({
+          walletId: wallet.id
+        });
+        if (!resCache?.result?.ok) {
+          console.log(JSON.stringify(resTxs));
+        }
+      } else {
+        // Update Wallets collection
+        const walletCount = await storage.db.collection(Storage.collections.WALLETS).countDocuments({
+          id: wallet.id,
+          network: oldNetwork
+        });
+
+        fixWalletsCount += walletCount;
+
+        // Update Addresses collection
+        const addressCount = await storage.db.collection(Storage.collections.ADDRESSES).countDocuments({
+          walletId: wallet.id,
+          createdOn: { $gte: startDate.getTime() / 1000, $lte: endDate.getTime() / 1000 }, // Included only to use index
+          network: oldNetwork
+        });
+
+        fixAddressCount += addressCount;
+
+        // Delete Txs collection
+        const txsCount = await storage.db.collection(Storage.collections.TXS).countDocuments({
+          walletId: wallet.id,
+          createdOn: { $gte: startDate.getTime() / 1000, $lte: endDate.getTime() / 1000 }, // Included only to use index
+          network: oldNetwork
+        });
+
+        fixTxsCount += txsCount;
+      }
+    }
+    process.stdout.write('\n');
+
+    console.log(`Fixed ${fixWalletsCount} wallets`);
+    console.log(`Fixed ${fixAddressCount} Addresses`);
+    console.log(`Fixed ${fixTxsCount} Txs`);
+
+    return done();
+  } catch (err) {
+    return done(err);
+  }
+});