@gooddollar/goodprotocol 2.0.25-beta.8 → 2.0.25

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@gooddollar/goodprotocol",
-  "version": "2.0.25-beta.8",
+  "version": "2.0.25",
   "description": "GoodDollar Protocol",
   "engines": {
     "node": ">=16.x"
@@ -61,22 +61,6 @@
     "url": "https://github.com/GoodDollar/GoodProtocol/issues"
   },
   "homepage": "https://gooddollar.org",
-  "dependencies": {
-    "@gooddollar/goodcontracts": "^2.6.6",
-    "@jsier/retrier": "^1.2.4",
-    "@openzeppelin/contracts": "^4.8.0",
-    "@openzeppelin/contracts-upgradeable": "^4.8.0",
-    "@openzeppelin/hardhat-upgrades": "^1.22.1",
-    "@superfluid-finance/ethereum-contracts": "^1.8.1",
-    "@superfluid-finance/sdk-core": "^0.5.8",
-    "@typechain/hardhat": "^6.1.5",
-    "async-promise-pool": "^1.0.4",
-    "openzeppelin-solidity": "^4.3.2",
-    "patch-package": "latest",
-    "press-any-key": "^0.1.1",
-    "truffle-plugin-verify": "^0.6.1",
-    "truffle-source-verify": "^0.0.6"
-  },
   "devDependencies": {
     "@babel/core": "*",
     "@babel/polyfill": "*",
@@ -88,19 +72,28 @@
     "@gnosis.pm/safe-ethers-lib": "^1.7.0",
     "@gnosis.pm/safe-service-client": "^1.4.0",
     "@gooddollar/bridge-contracts": "1.0.14-beta.7",
+    "@gooddollar/goodcontracts": "^2.6.6",
+    "@jsier/retrier": "^1.2.4",
     "@mean-finance/uniswap-v3-oracle": "^1.0.3",
     "@nomicfoundation/hardhat-chai-matchers": "1",
     "@nomicfoundation/hardhat-network-helpers": "^1.0.8",
     "@nomicfoundation/hardhat-verify": "^2.0.1",
     "@nomiclabs/hardhat-ethers": "^2.2.1",
     "@nomiclabs/hardhat-waffle": "^2.0.6",
+    "@openzeppelin/contracts": "^4.8.0",
+    "@openzeppelin/contracts-upgradeable": "^4.8.0",
+    "@openzeppelin/hardhat-upgrades": "^1.22.1",
+    "@superfluid-finance/ethereum-contracts": "^1.8.1",
+    "@superfluid-finance/sdk-core": "^0.5.8",
     "@swc/core": "1.3.96",
     "@swc/helpers": "^0.3.16",
     "@typechain/ethers-v5": "^11.1.1",
+    "@typechain/hardhat": "^6.1.5",
     "@types/mocha": "*",
     "@types/node": "*",
     "@uniswap/v2-core": "*",
     "@uniswap/v2-periphery": "^1.1.0-beta.0",
+    "async-promise-pool": "^1.0.4",
     "bignumber.js": "*",
     "bindings": "*",
     "chai": "*",
@@ -114,7 +107,7 @@
     "fs-extra": "9.0.0",
     "graphql": "^15.5.0",
     "graphql-request": "^3.4.0",
-    "hardhat": "^2.17.3",
+    "hardhat": "^2.*",
     "hardhat-contract-sizer": "^2.6.1",
     "hardhat-gas-reporter": "^1.0.8",
     "hardhat-storage-layout": "^0.1.7",
@@ -124,18 +117,23 @@
     "mocha-lcov-reporter": "*",
     "node-fetch": "*",
     "node-jq": "*",
+    "openzeppelin-solidity": "^4.3.2",
+    "patch-package": "latest",
+    "press-any-key": "^0.1.1",
     "prompt": "^1.3.0",
     "regenerator-runtime": "^0.13.9",
     "solhint": "*",
     "solidity-coverage": "^0.8.4",
     "truffle": "^5.11.5",
     "truffle-hdwallet-provider": "*",
+    "truffle-plugin-verify": "^0.6.1",
+    "truffle-source-verify": "^0.0.6",
     "ts-generator": "^0.1.1",
     "ts-node": "^10.9.1",
     "typechain": "^8.3.1",
     "typescript": "^5.3.3"
   },
-  "packageManager": "yarn@3.6.0",
+  "packageManager": "yarn@3.6.1",
   "resolutions": {
     "@swc/core": "1.3.96"
   },

package/releases/deployment.json

@@ -434,11 +434,10 @@
     "StaticOracle": "0x00851A91a3c4E9a4c1B48df827Bacc1f884bdE28",
     "BuyGDFactory": "0x00e533B7d6255D05b7f15034B1c989c21F51b91C",
     "BuyGDFactoryV2": "0x1F60C4C7037C6766924A43666B781ED1479587a2",
+    "CUSD": "0x765DE816845861e75A25fCA122bb6898B8B1282a",
     "BulkWhitelist": "0x30Afd0534dAA29135308d048103b40dc1BA4518d"
   },
-  "gnosis": {
-    "GuardiansSafe": "0x84c10b45fe51bfb4f86c19a47fdbc187d4572fb8"
-  },
+  "gnosis": { "GuardiansSafe": "0x84c10b45fe51bfb4f86c19a47fdbc187d4572fb8" },
   "development-celo": {
     "network": "development-celo",
     "networkId": 42220,
@@ -464,7 +463,13 @@
     "OneTimePaymentsV2": "0x2cb1c7d50371e277d7a36B171c0ACe9B157518EE",
     "UniswapV3Router": "0x5615CDAb10dc425a742d643d949a7F474C01abc4",
     "BuyGDFactory": "0x00e533B7d6255D05b7f15034B1c989c21F51b91C",
-    "BuyGDFactoryV2": "0x1F60C4C7037C6766924A43666B781ED1479587a2"
+    "BuyGDFactoryV2": "0x1F60C4C7037C6766924A43666B781ED1479587a2",
+    "MentoExchangeProvider": "0x02C5e6FfeC49Dca92af50A0718d2c4944DAaCb05",
+    "MentoExpansionController": "0xA354fE15A08318912D594E94DC7622C7D23Bb11A",
+    "CUSD": "0x2FFc031e855fE7C36669DC95CE00356D52195999",
+    "MentoReserve": "0x3f13b9FdB9ca8CDDa1c1E125296C8101B170C37C",
+    "MentoBroker": "0x43bC03995090E7B4E74cE70deF6bfA27D62049dC",
+    "CeloDistributionHelper": "0x4FE2400B80376a34e6a9011988d9D0D921EEf059"
   },
   "staging-celo": {
     "network": "staging-celo",
@@ -491,7 +496,13 @@
     "OneTimePaymentsV2": "0x963E7B4c970626Bb103c61fA16b002AFf6E6c2C7",
     "UniswapV3Router": "0x5615CDAb10dc425a742d643d949a7F474C01abc4",
     "BuyGDFactory": "0x00e533B7d6255D05b7f15034B1c989c21F51b91C",
-    "BuyGDFactoryV2": "0x1F60C4C7037C6766924A43666B781ED1479587a2"
+    "BuyGDFactoryV2": "0x1F60C4C7037C6766924A43666B781ED1479587a2",
+    "MentoExchangeProvider": "0x02C5e6FfeC49Dca92af50A0718d2c4944DAaCb05",
+    "MentoExpansionController": "0xA354fE15A08318912D594E94DC7622C7D23Bb11A",
+    "CUSD": "0x2FFc031e855fE7C36669DC95CE00356D52195999",
+    "MentoReserve": "0x3f13b9FdB9ca8CDDa1c1E125296C8101B170C37C",
+    "MentoBroker": "0x43bC03995090E7B4E74cE70deF6bfA27D62049dC",
+    "CeloDistributionHelper": "0x4FE2400B80376a34e6a9011988d9D0D921EEf059"
   },
   "development-goerli": {
     "ProxyFactory": "0x65D8eACBCd9618b8780C4b444081915a5D54D611",
@@ -541,7 +552,67 @@
     "GoodDollarMintBurnWrapper": "0x69d9c8d240E282a4ec0058CF0AC4E9D8ac7a11ac",
     "GoodDollarStaking": "0xC9a3DdadC753e6a7675Aa125991096623d06c437"
   },
-  "development-mainnet": {
-    "networkId": 1
+  "development-mainnet": { "networkId": 1 },
+  "test": {
+    "ProxyFactory": "0xCd7c00Ac6dc51e8dCc773971Ac9221cC582F3b1b",
+    "NameService": "0xB3d91a2650C8dB4456e96e06Bb147799338c5739",
+    "GReputation": "0xD759c30ADea571d35159302515b81473f7F0B5b3",
+    "CompoundVotingMachine": "0x07d17EcCcEE4568DE2b40cD1CB99b9984DF4519c",
+    "ClaimersDistribution": "0xA6D3dC26fC2f258d06B54EB6634077835C3C73Aa",
+    "GovernanceStaking": "0xA9e6Bfa2BF53dE88FEb19761D9b2eE2e821bF1Bf",
+    "UBIScheme": "0x16d5EF010758a483f94F3A001DeFf0ac9efF6352",
+    "ProtocolUpgradeFuse": "0x286B8DecD5ED79c962b2d8F4346CD97FF0E2C352",
+    "network": "test",
+    "networkId": 4447,
+    "HomeBridge": "0xcE0066b1008237625dDDBE4a751827de037E53D2",
+    "OneTimePayments": "0xcC4c41415fc68B2fBf70102742A83cDe435e0Ca7",
+    "AdminWallet": "0xAe120F0df055428E45b264E7794A18c54a2a3fAF",
+    "Identity": "0x87006e75a5B6bE9D1bbF61AC8Cd84f05D9140589",
+    "GoodDollar": "0x609a4341d11C29F8F5cDeF00931AdB96f76EF017",
+    "Controller": "0xd1f8C3b3C9C63a7a7f0dCFd20ae9053A04182db1",
+    "Avatar": "0xcafCfdF4517F504a473469F3723e674413EE9bce",
+    "FirstClaimPool": "0x0aec7c174554AF8aEc3680BB58431F6618311510",
+    "BancorFormula": "0x7B4f352Cd40114f12e82fC675b5BA8C7582FC513",
+    "DAI": "0xD6b040736e948621c5b6E0a494473c47a6113eA8",
+    "cDAI": "0xAdE429ba898c34722e722415D722A70a297cE3a2",
+    "COMP": "0x139e1D41943ee15dDe4DF876f9d0E7F85e26660A"
+  },
+  "test-mainnet": {
+    "ProxyFactory": "0x70E5370b8981Abc6e14C91F4AcE823954EFC8eA3",
+    "NameService": "0x50f2b544FD7af66D8a9dCF9a8d28F60c4955468D",
+    "GReputation": "0x08BC83e7cfe7DfEacAbCdC2432Cc6281ee678B46",
+    "CompoundVotingMachine": "0x6B2A2eA3b4965B2069f7112DFc5e766b10f52Bf4",
+    "GoodMarketMaker": "0x6E16fE6B78F307d78142A65CdFc3fB1c01Ed5416",
+    "GoodReserveCDai": "0xEF4c7EAD8C1aE06E76BD792785c9bed03811173B",
+    "ExchangeHelper": "0x39acC72200A9a847c2B6EFe514Db9eeE429e4B29",
+    "GoodFundManager": "0x68CC3Dc71a215bbbbc5fFeD38bc92F12d56eA835",
+    "StakersDistribution": "0x3957892fB94Ef52aB441e2BA77777F9dC8F7C501",
+    "ProtocolUpgrade": "0xa62835D1A6bf5f521C4e2746E1F51c923b8f3483",
+    "UniswapV2SwapHelper": "0x8E45C0936fa1a65bDaD3222bEFeC6a03C83372cE",
+    "CompoundStakingFactory": "0xBEe6FFc1E8627F51CcDF0b4399a1e1abc5165f15",
+    "AaveStakingFactory": "0xC32609C91d6B6b51D48f2611308FEf121B02041f",
+    "StakingContracts": [
+      ["0x14aA87DEed1d13012F1C9C93ef660D88a594D289", 13888],
+      ["0x47c7D4b2A310E651f846d7E8717F44414F59a7Da", "6944"]
+    ],
+    "DonationsStaking": "0x0Af313Cb7928CC29Db382575b0d96969C00414EC",
+    "network": "test-mainnet",
+    "networkId": 4447,
+    "ForeignBridge": "0x02b0B4EFd909240FCB2Eb5FAe060dC60D112E3a4",
+    "Contribution": "0x40a42Baf86Fc821f972Ad2aC878729063CeEF403",
+    "Identity": "0x6C2d83262fF84cBaDb3e416D527403135D757892",
+    "GoodDollar": "0xDda88b7dF7C04D319c41B53305Cd443B1f975e29",
+    "Controller": "0xF2AdAad89d56D49C697B9907C7D66ef27d96f859",
+    "Avatar": "0xd1891dD9DFF0784baa1dEb361dDFCAa5aE49cc6F",
+    "FirstClaimPool": "0xA56F946D6398Dd7d9D4D9B337Cf9E0F68982ca5B",
+    "BancorFormula": "0x01c1DeF3b91672704716159C9041Aeca392DdFfb",
+    "DAI": "0xfcDB4564c18A9134002b9771816092C9693622e3",
+    "cDAI": "0x32EEce76C2C2e8758584A83Ee2F522D4788feA0f",
+    "COMP": "0x927b167526bAbB9be047421db732C663a0b77B11"
+  },
+  "development-base": {
+    "ProxyFactory": "0x5B22F5623ECB00E288539346F389E05f65e226ae",
+    "SuperfluidFaucet": "0x88d18B06E55b33B94578628fcB5a6e1D198b0e73",
+    "AdminWallet": "0x6672C998C49635aA6825Be355fF2e731f417B674"
   }
 }

package/scripts/analytics/activeStakers.ts

@@ -0,0 +1,25 @@
+import { uniq } from "lodash";
+import fs from "fs";
+import { network, ethers } from "hardhat";
+import { Contract, Provider, setMulticallAddress } from "ethers-multicall";
+import release from "../../releases/deployment.json"
+import { SimpleStakingV2 } from "../../types";
+
+setMulticallAddress(122, "0x3CE6158b7278Bf6792e014FA7B4f3c6c46fe9410");
+
+setMulticallAddress(42220, "0x188C1bf697B66474dC3eaa119Ae691a8352537e3");
+
+const main = async () => {
+
+    const c1 = await ethers.getContractAt("SimpleStakingV2", release["production-mainnet"].StakingContractsV3[0][0]) as SimpleStakingV2
+    const c2 = await ethers.getContractAt("SimpleStakingV2", release["production-mainnet"].StakingContractsV3[1][0]) as SimpleStakingV2
+    const f = c1.filters.Staked()
+    const events = await c1.queryFilter(f, 14338550)
+    const events2 = await c2.queryFilter(f, 14338550)
+    const stakers = uniq(events.concat(events2).map(_ => _.args[0]))
+    console.log(stakers)
+    const res = (await Promise.all(stakers.map(async s => [s, await c1.balanceOf(s), await c2.balanceOf(s)]))).filter(_ => _[1].gt(0) || _[2].gt(0))
+    console.log(events.length)
+    console.log(res)
+}
+main()

package/scripts/analytics/activeUsersCount.ts

@@ -1,5 +1,7 @@
-import { range, chunk, uniq } from "lodash";
-import { ethers as Ethers } from "hardhat";
+import { range, chunk, uniq, mapValues, groupBy } from "lodash";
+import { ethers, ethers as Ethers } from "hardhat";
+import fs from "fs";
+import { bulkGetLastAuth } from "../utils";
 
 const ONE_DAY = 24 * 60 * 60;
 const ONE_HOUR = 60 * 60;
@@ -19,28 +21,27 @@ const main = async () => {
   for (let day of daysAgo) {
     const query = `
       { 
-        walletStats(first:1000, where: { lastClaimed_lte: ${curDay},lastClaimed_gt: ${
-      curDay - ONE_DAY
-    } isActiveUser: true }) {
+        walletStats(first:1000, where: { lastClaimed_lte: ${curDay},lastClaimed_gt: ${curDay - ONE_DAY
+      } isActiveUser: true }) {
         id        
       }
     }
   `;
 
     console.log("fetching inactive users since:", { curDay, day }, JSON.stringify({ query }));
-    const {
-      data: { walletStats }
-    } = await fetch("https://api.thegraph.com/subgraphs/name/gooddollar/gooddollarfuse", {
+    const result = await fetch("https://api.thegraph.com/subgraphs/name/gooddollar/gooddollarfuse", {
       method: "post",
       body: JSON.stringify({ query }),
       headers: { "Content-Type": "application/json" }
     }).then(_ => _.json());
 
-    console.log("got inactive wallets:", walletStats.length);
-    if (walletStats) {
-      const accounts = walletStats.map(_ => _.id);
-      allActive.push(...accounts);
-    }
+
+    console.log("got inactive wallets:", result);
+    // if (walletStats) {
+    //   const accounts = walletStats.map(_ => _.id);
+    //   allActive.push(...accounts);
+    // }
+
     // for (let tofish of chunk(accounts, 50)) {
     //   const tx = await ubiScheme.fishMulti(tofish, { gasLimit: 2000000 });
     //   console.log("fishing tx:", tx, tofish);
@@ -54,32 +55,83 @@ const main = async () => {
   console.log("active claimers:", unique.length);
 };
 
+const countClaimersUsingExplorer = async () => {
+  let claimers = [] //JSON.parse(fs.readFileSync("claimers.json").toString())
+  const curBlock = await ethers.provider.getBlockNumber()
+  const DAYS = 1
+  let fromBlock = curBlock - 60 * 60 * 24 * DAYS / 5 //roughly DAYS of blocks
+  const querySize = 90// 15 min of blocks
+  const blockRanges = range(fromBlock, curBlock, querySize)
+  const chunks = chunk(blockRanges, 10)
+  for (let idx in chunks) {
+    const toFetch = chunks[idx]
+    const ps = toFetch.map(async fromBlock => {
+      let toBlock = fromBlock + querySize
+      toBlock = toBlock > curBlock ? curBlock : toBlock
+      const result = await fetch(`https://explorer.celo.org/mainnet/api?module=logs&action=getLogs&fromBlock=${fromBlock}&toBlock=${toBlock}&address=0x43d72ff17701b2da814620735c39c620ce0ea4a1&topic0=0x89ed24731df6b066e4c5186901fffdba18cd9a10f07494aff900bdee260d1304`).then(_ => _.json())
+      fromBlock = toBlock
+      const addrs = result.result.map(_ => ethers.utils.defaultAbiCoder.decode(["address"], _.topics[1])[0])
+      claimers.push(...addrs);
+      console.log("claimers:", addrs.length, "block range:", { fromBlock, toBlock }, addrs[0])
+    })
+    await Promise.all(ps)
+    console.log(`${idx} out of ${chunks.length}`)
+  }
+
+  claimers = uniq(claimers)
+  fs.writeFileSync("claimers.json", JSON.stringify(claimers))
+}
+
+const countLastWhitelisted = async () => {
+  const claimers = JSON.parse(fs.readFileSync("claimers.json").toString())
+  const DAY = 60 * 60 * 24
+  let total = 0
+  let howOld = []
+  console.log("unique claimers:", claimers.length)
+  for (let bulk of chunk(claimers, 1000)) {
+    const results = await bulkGetLastAuth(bulk)
+    const older = results.filter(_ => _.toNumber() < Date.now() / 1000 - DAY * 180).length
+    bulk.forEach((_, i) => howOld.push([_, Math.ceil((Date.now() / 1000 - DAY * 180 - results[i].toNumber()) / (30 * DAY)).toFixed(0)]))
+    console.log(older, "out of", results.length)
+    total += older
+  }
+
+  howOld = howOld.filter(_ => Number(_[1]) > 0)
+  console.log({ total })
+  howOld = mapValues(groupBy(howOld, _ => _[1]), _ => _.length)
+  console.log(howOld)
+}
+
 const countActive = async () => {
-  const twoWeeksAgo = parseInt((Date.now() / 1000).toFixed(0)); //
 
-  const hoursAgo: number[] = range(0, 24 * 14, 1);
-  let curDay = twoWeeksAgo;
+  const hoursAgo: number[] = range(0, 24 * 30, 1);
+  let curDay = parseInt((Date.now() / 1000).toFixed(0));
   const allActive = [];
   for (let hour of hoursAgo) {
     const query = `
       { 
-        walletStats(first:1000, where: { lastClaimed_lte: ${curDay},lastClaimed_gt: ${
-      curDay - ONE_HOUR
-    } isActiveUser: true }) {
+        walletStats(first:1000, where: { lastClaimed_lte: ${curDay},lastClaimed_gt: ${curDay - ONE_HOUR
+      } isActiveUser: true }) {
         id        
       }
     }
   `;
 
     console.log("fetching active users since:", { curDay, hour }, JSON.stringify({ query }));
+    const result =
+
+      await fetch("https://gateway-arbitrum.network.thegraph.com/api/3c409250a317ce7b0c6a7a8b9a409ef8/subgraphs/id/F7314rxGdcpKPC1nN5KCoFW84EGRoUyzseY2sAT9PEkw", {
+        // await fetch("https://gateway.thegraph.com/api/3c409250a317ce7b0c6a7a8b9a409ef8/subgraphs/id/5cAhhzm7LSqGiFibV1odbbgZWiRmZsYjYrmaoj87UxFd", {
+        method: "post",
+        body: JSON.stringify({ query }),
+        headers: { "Content-Type": "application/json", "origin": "https://wallet.gooddollar.org" }
+
+      }).then(_ => _.json());
+
+    console.log(result)
     const {
       data: { walletStats }
-    } = await fetch("https://api.thegraph.com/subgraphs/name/gooddollar/gooddollarfuse", {
-      method: "post",
-      body: JSON.stringify({ query }),
-      headers: { "Content-Type": "application/json" }
-    }).then(_ => _.json());
-
+    } = result
     console.log("got active wallets:", walletStats.length);
     if (walletStats) {
       const accounts = walletStats.map(_ => _.id);
@@ -99,4 +151,6 @@ const countActive = async () => {
 };
 
 //main();
-countActive();
+// countActive();
+// countClaimersUsingExplorer()
+countLastWhitelisted()

package/scripts/analytics/activeWalletsStats.ts

@@ -1,24 +1,88 @@
-import { maxBy, range, sortBy } from "lodash";
+import { maxBy, range, sortBy, flatten } from "lodash";
 import PromisePool from "async-promise-pool";
 import fs from "fs";
 import { ethers } from "hardhat";
 
+const today = new Date().toLocaleDateString().replace(/\//g, "");
+console.log({ today });
 /****
  * Fetch token holders and their last activity date
  * can be used to create stats about active users and how much G$ isnt active
  */
 const main = async (chain = "fuse") => {
-  const explorer = chain === "fuse" ? "https://explorer.fuse.io" : "https://explorer.celo.org";
-  const subgraph = chain === "fuse" ? "https://api.thegraph.com/subgraphs/name/gooddollar/gooddollarfuse" : "https://";
+  let result = [];
+  let balances = {};
+  let curPage = 1;
+  let maxResult;
+  const fuseExplorer = "https://explorer.fuse.io/api";
+  const celoExplorer = "https://explorer.celo.org/mainnet/api";
+  const fuseSubgraph =
+    "https://gateway.thegraph.com/api/9048669a7632776aae01a191c4939445/subgraphs/id/5cAhhzm7LSqGiFibV1odbbgZWiRmZsYjYrmaoj87UxFd";
+  const celoSubgraph =
+    "https://gateway.thegraph.com/api/9048669a7632776aae01a191c4939445/subgraphs/id/F7314rxGdcpKPC1nN5KCoFW84EGRoUyzseY2sAT9PEkw";
+  do {
+    const pages = range(curPage, curPage + 5, 1);
+    curPage += 5;
+    const ps = pages.map(p =>
+      fetch(
+        `${fuseExplorer}?module=token&action=getTokenHolders&contractaddress=0x495d133B938596C9984d462F007B676bDc57eCEC&page=${p}&offset=10000`
+      )
+        .then(_ => _.json())
+        .then(_ => _.result)
+    );
+    const results = await Promise.all(ps);
+    result = result.concat(...results);
+    maxResult = maxBy(results, "length");
+    console.log(maxResult.length, result.length);
+  } while (maxResult.length === 10000);
+  result.forEach(
+    r =>
+      (balances[r.address.toLowerCase()] = {
+        balance: Number(r.value) / 100,
+        fuseBalance: Number(r.value) / 100,
+        lastSeen: 0
+      })
+  );
+
+  console.log("fetching celo balances....");
+  curPage = 1;
+  result = [];
+  do {
+    const pages = range(curPage, curPage + 3, 1);
+    curPage += 3;
+    const ps = pages.map(p =>
+      fetch(
+        `${celoExplorer}?module=token&action=getTokenHolders&contractaddress=0x62B8B11039FcfE5aB0C56E502b1C372A3d2a9c7A&page=${p}&offset=10000`
+      )
+        .then(_ => _.json())
+        .then(_ => _.result)
+    );
+    const results = await Promise.all(ps);
+    result = result.concat(...results);
+    maxResult = maxBy(results, "length");
+    console.log(maxResult.length, result.length);
+  } while (maxResult.length === 10000);
+
+  result.forEach(
+    r =>
+      (balances[r.address.toLowerCase()] = {
+        ...balances[r.address.toLowerCase()],
+        celoBalance: Number(r.value) / 1e18,
+        balance: Number(balances[r.address.toLowerCase()]?.fuseBalance || 0) + Number(r.value) / 1e18,
+        lastSeen: 0
+      })
+  );
+
+  fs.writeFileSync(`activeWalletsBalances-${today}.json`, JSON.stringify(balances));
 
-  const balances = await getFuseBalances();
+  balances = JSON.parse(fs.readFileSync(`activeWalletsBalances-${today}.json`).toString());
 
   const EPOCH = 60 * 60 * 6;
-  const pool = new PromisePool({ concurrency: 30 });
+  const pool = new PromisePool({ concurrency: 10 });
   const lastUsed = {};
   const epochs = range(1596045730, (Date.now() / 1000).toFixed(0), EPOCH);
 
-  const graphQuery = async (start, skip) => {
+  const graphQuery = async (start, skip, subgraph, retry = 3) => {
     const query = `{
         walletStats(first: 1000 skip:${skip} where: { dateAppeared_gte: ${start} dateAppeared_lt:${start + EPOCH} }) {
           id
@@ -31,7 +95,7 @@ const main = async (chain = "fuse") => {
       }`;
     // console.log({ query });
     try {
-      const { data = {}, errors } = await fetch("https://api.thegraph.com/subgraphs/name/gooddollar/gooddollarfuse", {
+      const { data = {}, errors } = await fetch(subgraph, {
         method: "POST",
         headers: {
           "Content-Type": "application/json"
@@ -39,27 +103,36 @@ const main = async (chain = "fuse") => {
         body: JSON.stringify({ query })
       }).then(_ => _.json());
       errors && console.log({ errors });
+      if (errors) {
+        console.log("query failed:", { subgraph, start, skip, retrying: retry > 0 });
+        if (retry > 0) {
+          return graphQuery(start, skip, subgraph, retry - 1);
+        }
+        return [];
+      }
+      // console.log("query ok:", { subgraph, start, skip, records: data.walletStats.length })
       if (data?.walletStats?.length === 1000) {
-        return data.walletStats.concat(await graphQuery(start, skip + 1000));
+        return data.walletStats.concat(await graphQuery(start, skip + 1000, subgraph));
       }
       return data.walletStats || [];
     } catch (error) {
-      console.log({ query, error });
+      console.log({ query, error, subgraph });
       return [];
     }
   };
   epochs.forEach(e => {
     pool.add(async () => {
-      const walletStats = await graphQuery(e, 0);
+      const walletStats = flatten(await Promise.all([graphQuery(e, 0, fuseSubgraph), graphQuery(e, 0, celoSubgraph)]));
       walletStats.forEach(w => {
         balances[w.id.toLowerCase()] = {
+          ...balances[w.id.toLowerCase()],
           lastSeen: Math.max(
+            balances[w.id.toLowerCase()]?.lastSeen,
             Number(w.lastClaimed),
             Number(w.lastTransactionFrom),
             Number(w.lastTransactionTo),
             Number(w.dateAppeared)
-          ),
-          balance: balances[w.id.toLowerCase()]?.balance || w.balance
+          )
         };
       });
       console.log({ curDate: e, records: walletStats.length });
@@ -67,7 +140,7 @@ const main = async (chain = "fuse") => {
   });
 
   await pool.all();
-  fs.writeFileSync("activeWalletsLastUsed.json", JSON.stringify(balances));
+  fs.writeFileSync(`activeWalletsLastUsed-${today}.json`, JSON.stringify(balances));
   //   console.log({ lastUsed });
 };
 
@@ -85,8 +158,46 @@ function arrayToCsv(data) {
     )
     .join("\r\n"); // rows starting on new lines
 }
+
+const fix = async () => {
+  const balances = JSON.parse(fs.readFileSync(`activeWalletsLastUsed-${today}.json`).toString());
+
+  let result = [];
+
+  let missing = 0;
+  for (let addr in balances) {
+    const r = balances[addr];
+    if (!r.fuseBalance && !r.celoBalance) {
+      console.log("missing:", addr);
+      missing += 1;
+      continue;
+    }
+    r.celoBalance = (r.celoBalance || 0) * 100;
+    r.balance = r.celoBalance + r.fuseBalance;
+  }
+
+  console.log("missing balance", missing);
+  console.log(sortBy(Object.entries(balances), _ => -_.balance).slice(0, 10));
+
+  fs.writeFileSync(`activeWalletsLastUsed-${today}.json`, JSON.stringify(balances));
+};
 const etl = async () => {
-  const balances = JSON.parse(fs.readFileSync("activeWalletsLastUsed.json").toString());
+  /** Convert a 2D array into a CSV string
+   */
+  function arrayToCsv(data) {
+    return data
+      .map(
+        row =>
+          row
+            .map(String) // convert every value to String
+            .map(v => v.replaceAll('"', '""')) // escape double colons
+            .map(v => `"${v}"`) // quote it
+            .join(",") // comma-separated
+      )
+      .join("\r\n"); // rows starting on new lines
+  }
+
+  const balances = JSON.parse(fs.readFileSync(`activeWalletsLastUsed-${today}.json`).toString());
 
   let result = [];
 
@@ -95,22 +206,30 @@ const etl = async () => {
     if (!r.balance) {
       continue;
     }
-    result.push([addr, r.balance / 100, r.lastSeen, false]);
+    result.push([addr, r.balance, r.lastSeen, false, r.fuseBalance, r.celoBalance]);
   }
+  result = sortBy(result, _ => -_[1]);
   const top100 = result.slice(0, 100);
   const pool = new PromisePool({ concurrency: 30 });
   const provider = new ethers.providers.JsonRpcBatchProvider("https://rpc.fuse.io");
+  const celoprovider = new ethers.providers.JsonRpcBatchProvider("https://forno.celo.org");
 
   for (let idx in top100) {
     pool.add(async () => {
       const record = top100[idx];
-      let isContract = (await provider.getCode(record[0]).catch(e => "0x")) !== "0x";
+      let isContract =
+        (
+          await Promise.all([
+            provider.getCode(record[0]).catch(e => "0x"),
+            celoprovider.getCode(record[0]).catch(e => "0x")
+          ])
+        ).find(_ => _ !== "0x") !== undefined;
       record[3] = isContract;
     });
   }
   await pool.all();
   console.log({ top100 });
-  fs.writeFileSync("activeWalletsLastUsed.csv", arrayToCsv(sortBy(result, _ => -Number(_[1]))));
+  fs.writeFileSync(`activeWalletsLastUsed-${today}.csv`, arrayToCsv(result));
 };
 
 const getFuseBalances = async (refetch = true) => {
@@ -177,5 +296,6 @@ const fundsByLastSeen = async () => {
 };
 getFuseBalances(false).catch(e => console.log(e));
 // main().catch(e => console.log(e));
-// etl();
+// fix();
+etl();
 // fundsByLastSeen();

package/scripts/fv.ts

@@ -1,10 +1,11 @@
 import fs from "fs";
 import { chunk, uniqBy } from "lodash";
 import delay from "delay";
+import { bulkIsWhitelisted, bulkLastAuth } from "./utils";
 
-import { bulkIsWhitelisted, bulkIsWhitelistedFuse, bulkLastAuth } from "./utils";
+//create tunnel to fv server ssh -L 9090:server:8080 -N user@server -i sshkey
 
-const fetchImages = async a => {
+const saveImages = async a => {
   const i1 = await fetch("http://localhost:9090/enrollment-3d/" + a[1]).then(_ => _.json());
   const i2 = await fetch("http://localhost:9090/enrollment-3d/" + a[2]).then(_ => _.json());
   if (i1.auditTrailBase64 && i2.auditTrailBase64) {
@@ -16,7 +17,7 @@ const fetchImages = async a => {
     });
   } else console.log("not found", a, !!i1.auditTrailBase64, !!i2.auditTrailBase64);
 };
-//create tunnel to fv server ssh -L 9090:server:8080 -N user@server -i sshkey
+
 const main = async () => {
   const data = JSON.parse(fs.readFileSync("fvtriplets2.txt").toString());
   const triplets = uniqBy(chunk(data, 3), _ => _.join("_"));
@@ -33,7 +34,7 @@ const main = async () => {
     return fs.existsSync(key) === false;
   });
   console.log({ notfetched });
-  const ps = notfetched.map(fetchImages);
+  const ps = notfetched.map(saveImages);
 
   await Promise.all(ps);
 };
@@ -207,6 +208,7 @@ const deleteIdentifiers = async password => {
 };
 // checkIndexedOrDelete();
 fixInvalidIndexed();
-// main();
 // console.log(process.env.ADMIN_PASSWORD);
 // deleteIdentifiers(process.env.ADMIN_PASSWORD);
+// main();
+saveImages(["", "", ""]);