@gooddollar/goodprotocol 2.0.25-beta.0 → 2.0.25-beta.10

package/scripts/analytics/activeStakers.ts

@@ -0,0 +1,25 @@
+import { uniq } from "lodash";
+import fs from "fs";
+import { network, ethers } from "hardhat";
+import { Contract, Provider, setMulticallAddress } from "ethers-multicall";
+import release from "../../releases/deployment.json"
+import { SimpleStakingV2 } from "../../types";
+
+setMulticallAddress(122, "0x3CE6158b7278Bf6792e014FA7B4f3c6c46fe9410");
+
+setMulticallAddress(42220, "0x188C1bf697B66474dC3eaa119Ae691a8352537e3");
+
+const main = async () => {
+
+    const c1 = await ethers.getContractAt("SimpleStakingV2", release["production-mainnet"].StakingContractsV3[0][0]) as SimpleStakingV2
+    const c2 = await ethers.getContractAt("SimpleStakingV2", release["production-mainnet"].StakingContractsV3[1][0]) as SimpleStakingV2
+    const f = c1.filters.Staked()
+    const events = await c1.queryFilter(f, 14338550)
+    const events2 = await c2.queryFilter(f, 14338550)
+    const stakers = uniq(events.concat(events2).map(_ => _.args[0]))
+    console.log(stakers)
+    const res = (await Promise.all(stakers.map(async s => [s, await c1.balanceOf(s), await c2.balanceOf(s)]))).filter(_ => _[1].gt(0) || _[2].gt(0))
+    console.log(events.length)
+    console.log(res)
+}
+main()

package/scripts/analytics/activeUsersCount.ts

@@ -1,5 +1,7 @@
-import { range, chunk, uniq } from "lodash";
-import { ethers as Ethers } from "hardhat";
+import { range, chunk, uniq, mapValues, groupBy } from "lodash";
+import { ethers, ethers as Ethers } from "hardhat";
+import fs from "fs";
+import { bulkGetLastAuth } from "../utils";
 
 const ONE_DAY = 24 * 60 * 60;
 const ONE_HOUR = 60 * 60;
@@ -19,28 +21,27 @@ const main = async () => {
   for (let day of daysAgo) {
     const query = `
       { 
-        walletStats(first:1000, where: { lastClaimed_lte: ${curDay},lastClaimed_gt: ${
-      curDay - ONE_DAY
-    } isActiveUser: true }) {
+        walletStats(first:1000, where: { lastClaimed_lte: ${curDay},lastClaimed_gt: ${curDay - ONE_DAY
+      } isActiveUser: true }) {
         id        
       }
     }
   `;
 
     console.log("fetching inactive users since:", { curDay, day }, JSON.stringify({ query }));
-    const {
-      data: { walletStats }
-    } = await fetch("https://api.thegraph.com/subgraphs/name/gooddollar/gooddollarfuse", {
+    const result = await fetch("https://api.thegraph.com/subgraphs/name/gooddollar/gooddollarfuse", {
       method: "post",
       body: JSON.stringify({ query }),
       headers: { "Content-Type": "application/json" }
     }).then(_ => _.json());
 
-    console.log("got inactive wallets:", walletStats.length);
-    if (walletStats) {
-      const accounts = walletStats.map(_ => _.id);
-      allActive.push(...accounts);
-    }
+
+    console.log("got inactive wallets:", result);
+    // if (walletStats) {
+    //   const accounts = walletStats.map(_ => _.id);
+    //   allActive.push(...accounts);
+    // }
+
     // for (let tofish of chunk(accounts, 50)) {
     //   const tx = await ubiScheme.fishMulti(tofish, { gasLimit: 2000000 });
     //   console.log("fishing tx:", tx, tofish);
@@ -54,32 +55,83 @@ const main = async () => {
   console.log("active claimers:", unique.length);
 };
 
+const countClaimersUsingExplorer = async () => {
+  let claimers = [] //JSON.parse(fs.readFileSync("claimers.json").toString())
+  const curBlock = await ethers.provider.getBlockNumber()
+  const DAYS = 1
+  let fromBlock = curBlock - 60 * 60 * 24 * DAYS / 5 //roughly DAYS of blocks
+  const querySize = 90// 15 min of blocks
+  const blockRanges = range(fromBlock, curBlock, querySize)
+  const chunks = chunk(blockRanges, 10)
+  for (let idx in chunks) {
+    const toFetch = chunks[idx]
+    const ps = toFetch.map(async fromBlock => {
+      let toBlock = fromBlock + querySize
+      toBlock = toBlock > curBlock ? curBlock : toBlock
+      const result = await fetch(`https://explorer.celo.org/mainnet/api?module=logs&action=getLogs&fromBlock=${fromBlock}&toBlock=${toBlock}&address=0x43d72ff17701b2da814620735c39c620ce0ea4a1&topic0=0x89ed24731df6b066e4c5186901fffdba18cd9a10f07494aff900bdee260d1304`).then(_ => _.json())
+      fromBlock = toBlock
+      const addrs = result.result.map(_ => ethers.utils.defaultAbiCoder.decode(["address"], _.topics[1])[0])
+      claimers.push(...addrs);
+      console.log("claimers:", addrs.length, "block range:", { fromBlock, toBlock }, addrs[0])
+    })
+    await Promise.all(ps)
+    console.log(`${idx} out of ${chunks.length}`)
+  }
+
+  claimers = uniq(claimers)
+  fs.writeFileSync("claimers.json", JSON.stringify(claimers))
+}
+
+const countLastWhitelisted = async () => {
+  const claimers = JSON.parse(fs.readFileSync("claimers.json").toString())
+  const DAY = 60 * 60 * 24
+  let total = 0
+  let howOld = []
+  console.log("unique claimers:", claimers.length)
+  for (let bulk of chunk(claimers, 1000)) {
+    const results = await bulkGetLastAuth(bulk)
+    const older = results.filter(_ => _.toNumber() < Date.now() / 1000 - DAY * 180).length
+    bulk.forEach((_, i) => howOld.push([_, Math.ceil((Date.now() / 1000 - DAY * 180 - results[i].toNumber()) / (30 * DAY)).toFixed(0)]))
+    console.log(older, "out of", results.length)
+    total += older
+  }
+
+  howOld = howOld.filter(_ => Number(_[1]) > 0)
+  console.log({ total })
+  howOld = mapValues(groupBy(howOld, _ => _[1]), _ => _.length)
+  console.log(howOld)
+}
+
 const countActive = async () => {
-  const twoWeeksAgo = parseInt((Date.now() / 1000).toFixed(0)); //
 
-  const hoursAgo: number[] = range(0, 24 * 14, 1);
-  let curDay = twoWeeksAgo;
+  const hoursAgo: number[] = range(0, 24 * 30, 1);
+  let curDay = parseInt((Date.now() / 1000).toFixed(0));
   const allActive = [];
   for (let hour of hoursAgo) {
     const query = `
       { 
-        walletStats(first:1000, where: { lastClaimed_lte: ${curDay},lastClaimed_gt: ${
-      curDay - ONE_HOUR
-    } isActiveUser: true }) {
+        walletStats(first:1000, where: { lastClaimed_lte: ${curDay},lastClaimed_gt: ${curDay - ONE_HOUR
+      } isActiveUser: true }) {
         id        
       }
     }
   `;
 
     console.log("fetching active users since:", { curDay, hour }, JSON.stringify({ query }));
+    const result =
+
+      await fetch("https://gateway-arbitrum.network.thegraph.com/api/3c409250a317ce7b0c6a7a8b9a409ef8/subgraphs/id/F7314rxGdcpKPC1nN5KCoFW84EGRoUyzseY2sAT9PEkw", {
+        // await fetch("https://gateway.thegraph.com/api/3c409250a317ce7b0c6a7a8b9a409ef8/subgraphs/id/5cAhhzm7LSqGiFibV1odbbgZWiRmZsYjYrmaoj87UxFd", {
+        method: "post",
+        body: JSON.stringify({ query }),
+        headers: { "Content-Type": "application/json", "origin": "https://wallet.gooddollar.org" }
+
+      }).then(_ => _.json());
+
+    console.log(result)
     const {
       data: { walletStats }
-    } = await fetch("https://api.thegraph.com/subgraphs/name/gooddollar/gooddollarfuse", {
-      method: "post",
-      body: JSON.stringify({ query }),
-      headers: { "Content-Type": "application/json" }
-    }).then(_ => _.json());
-
+    } = result
     console.log("got active wallets:", walletStats.length);
     if (walletStats) {
       const accounts = walletStats.map(_ => _.id);
@@ -99,4 +151,6 @@ const countActive = async () => {
 };
 
 //main();
-countActive();
+// countActive();
+// countClaimersUsingExplorer()
+countLastWhitelisted()

package/scripts/analytics/activeWalletsStats.ts

@@ -1,8 +1,10 @@
-import { maxBy, range, sortBy } from "lodash";
+import { maxBy, range, sortBy, flatten } from "lodash";
 import PromisePool from "async-promise-pool";
 import fs from "fs";
 import { ethers } from "hardhat";
 
+const today = new Date().toLocaleDateString().replace(/\//g, "");
+console.log({ today });
 /****
  * Fetch token holders and their last activity date
  * can be used to create stats about active users and how much G$ isnt active
@@ -12,14 +14,18 @@ const main = async (chain = "fuse") => {
   let balances = {};
   let curPage = 1;
   let maxResult;
-  const explorer = chain === "fuse" ? "https://explorer.fuse.io" : "https://explorer.celo.org";
-  const subgraph = chain === "fuse" ? "https://api.thegraph.com/subgraphs/name/gooddollar/gooddollarfuse" : "https://";
+  const fuseExplorer = "https://explorer.fuse.io/api";
+  const celoExplorer = "https://explorer.celo.org/mainnet/api";
+  const fuseSubgraph =
+    "https://gateway.thegraph.com/api/9048669a7632776aae01a191c4939445/subgraphs/id/5cAhhzm7LSqGiFibV1odbbgZWiRmZsYjYrmaoj87UxFd";
+  const celoSubgraph =
+    "https://gateway.thegraph.com/api/9048669a7632776aae01a191c4939445/subgraphs/id/F7314rxGdcpKPC1nN5KCoFW84EGRoUyzseY2sAT9PEkw";
   do {
     const pages = range(curPage, curPage + 5, 1);
     curPage += 5;
     const ps = pages.map(p =>
       fetch(
-        `https://explorer.fuse.io/api?module=token&action=getTokenHolders&contractaddress=0x495d133B938596C9984d462F007B676bDc57eCEC&page=${p}&offset=10000`
+        `${fuseExplorer}?module=token&action=getTokenHolders&contractaddress=0x495d133B938596C9984d462F007B676bDc57eCEC&page=${p}&offset=10000`
       )
         .then(_ => _.json())
         .then(_ => _.result)
@@ -29,17 +35,54 @@ const main = async (chain = "fuse") => {
     maxResult = maxBy(results, "length");
     console.log(maxResult.length, result.length);
   } while (maxResult.length === 10000);
-  result.forEach(r => (balances[r.address.toLowerCase()] = { balance: r.value, lastSeen: 0 }));
-  fs.writeFileSync("activeWalletsBalances.json", JSON.stringify(balances));
+  result.forEach(
+    r =>
+      (balances[r.address.toLowerCase()] = {
+        balance: Number(r.value) / 100,
+        fuseBalance: Number(r.value) / 100,
+        lastSeen: 0
+      })
+  );
+
+  console.log("fetching celo balances....");
+  curPage = 1;
+  result = [];
+  do {
+    const pages = range(curPage, curPage + 3, 1);
+    curPage += 3;
+    const ps = pages.map(p =>
+      fetch(
+        `${celoExplorer}?module=token&action=getTokenHolders&contractaddress=0x62B8B11039FcfE5aB0C56E502b1C372A3d2a9c7A&page=${p}&offset=10000`
+      )
+        .then(_ => _.json())
+        .then(_ => _.result)
+    );
+    const results = await Promise.all(ps);
+    result = result.concat(...results);
+    maxResult = maxBy(results, "length");
+    console.log(maxResult.length, result.length);
+  } while (maxResult.length === 10000);
+
+  result.forEach(
+    r =>
+      (balances[r.address.toLowerCase()] = {
+        ...balances[r.address.toLowerCase()],
+        celoBalance: Number(r.value) / 1e18,
+        balance: Number(balances[r.address.toLowerCase()]?.fuseBalance || 0) + Number(r.value) / 1e18,
+        lastSeen: 0
+      })
+  );
 
-  // balances = JSON.parse(fs.readFileSync("activeWalletsBalances.json").toString());
+  fs.writeFileSync(`activeWalletsBalances-${today}.json`, JSON.stringify(balances));
+
+  balances = JSON.parse(fs.readFileSync(`activeWalletsBalances-${today}.json`).toString());
 
   const EPOCH = 60 * 60 * 6;
-  const pool = new PromisePool({ concurrency: 30 });
+  const pool = new PromisePool({ concurrency: 10 });
   const lastUsed = {};
   const epochs = range(1596045730, (Date.now() / 1000).toFixed(0), EPOCH);
 
-  const graphQuery = async (start, skip) => {
+  const graphQuery = async (start, skip, subgraph, retry = 3) => {
     const query = `{
         walletStats(first: 1000 skip:${skip} where: { dateAppeared_gte: ${start} dateAppeared_lt:${start + EPOCH} }) {
           id
@@ -52,7 +95,7 @@ const main = async (chain = "fuse") => {
       }`;
     // console.log({ query });
     try {
-      const { data = {}, errors } = await fetch("https://api.thegraph.com/subgraphs/name/gooddollar/gooddollarfuse", {
+      const { data = {}, errors } = await fetch(subgraph, {
         method: "POST",
         headers: {
           "Content-Type": "application/json"
@@ -60,27 +103,36 @@ const main = async (chain = "fuse") => {
         body: JSON.stringify({ query })
       }).then(_ => _.json());
       errors && console.log({ errors });
+      if (errors) {
+        console.log("query failed:", { subgraph, start, skip, retrying: retry > 0 });
+        if (retry > 0) {
+          return graphQuery(start, skip, subgraph, retry - 1);
+        }
+        return [];
+      }
+      // console.log("query ok:", { subgraph, start, skip, records: data.walletStats.length })
       if (data?.walletStats?.length === 1000) {
-        return data.walletStats.concat(await graphQuery(start, skip + 1000));
+        return data.walletStats.concat(await graphQuery(start, skip + 1000, subgraph));
       }
       return data.walletStats || [];
     } catch (error) {
-      console.log({ query, error });
+      console.log({ query, error, subgraph });
       return [];
     }
   };
   epochs.forEach(e => {
     pool.add(async () => {
-      const walletStats = await graphQuery(e, 0);
+      const walletStats = flatten(await Promise.all([graphQuery(e, 0, fuseSubgraph), graphQuery(e, 0, celoSubgraph)]));
       walletStats.forEach(w => {
         balances[w.id.toLowerCase()] = {
+          ...balances[w.id.toLowerCase()],
           lastSeen: Math.max(
+            balances[w.id.toLowerCase()]?.lastSeen,
             Number(w.lastClaimed),
             Number(w.lastTransactionFrom),
             Number(w.lastTransactionTo),
             Number(w.dateAppeared)
-          ),
-          balance: balances[w.id.toLowerCase()]?.balance || w.balance
+          )
         };
       });
       console.log({ curDate: e, records: walletStats.length });
@@ -88,10 +140,47 @@ const main = async (chain = "fuse") => {
   });
 
   await pool.all();
-  fs.writeFileSync("activeWalletsLastUsed.json", JSON.stringify(balances));
+  fs.writeFileSync(`activeWalletsLastUsed-${today}.json`, JSON.stringify(balances));
   //   console.log({ lastUsed });
 };
 
+/** Convert a 2D array into a CSV string
+ */
+function arrayToCsv(data) {
+  return data
+    .map(
+      row =>
+        row
+          .map(String) // convert every value to String
+          .map(v => v.replaceAll('"', '""')) // escape double colons
+          .map(v => `"${v}"`) // quote it
+          .join(",") // comma-separated
+    )
+    .join("\r\n"); // rows starting on new lines
+}
+
+const fix = async () => {
+  const balances = JSON.parse(fs.readFileSync(`activeWalletsLastUsed-${today}.json`).toString());
+
+  let result = [];
+
+  let missing = 0;
+  for (let addr in balances) {
+    const r = balances[addr];
+    if (!r.fuseBalance && !r.celoBalance) {
+      console.log("missing:", addr);
+      missing += 1;
+      continue;
+    }
+    r.celoBalance = (r.celoBalance || 0) * 100;
+    r.balance = r.celoBalance + r.fuseBalance;
+  }
+
+  console.log("missing balance", missing);
+  console.log(sortBy(Object.entries(balances), _ => -_.balance).slice(0, 10));
+
+  fs.writeFileSync(`activeWalletsLastUsed-${today}.json`, JSON.stringify(balances));
+};
 const etl = async () => {
   /** Convert a 2D array into a CSV string
    */
@@ -108,7 +197,7 @@ const etl = async () => {
       .join("\r\n"); // rows starting on new lines
   }
 
-  const balances = JSON.parse(fs.readFileSync("activeWalletsLastUsed.json").toString());
+  const balances = JSON.parse(fs.readFileSync(`activeWalletsLastUsed-${today}.json`).toString());
 
   let result = [];
 
@@ -117,22 +206,64 @@ const etl = async () => {
     if (!r.balance) {
       continue;
     }
-    result.push([addr, r.balance / 100, r.lastSeen, false]);
+    result.push([addr, r.balance, r.lastSeen, false, r.fuseBalance, r.celoBalance]);
   }
+  result = sortBy(result, _ => -_[1]);
   const top100 = result.slice(0, 100);
   const pool = new PromisePool({ concurrency: 30 });
   const provider = new ethers.providers.JsonRpcBatchProvider("https://rpc.fuse.io");
+  const celoprovider = new ethers.providers.JsonRpcBatchProvider("https://forno.celo.org");
 
   for (let idx in top100) {
     pool.add(async () => {
       const record = top100[idx];
-      let isContract = (await provider.getCode(record[0]).catch(e => "0x")) !== "0x";
+      let isContract =
+        (
+          await Promise.all([
+            provider.getCode(record[0]).catch(e => "0x"),
+            celoprovider.getCode(record[0]).catch(e => "0x")
+          ])
+        ).find(_ => _ !== "0x") !== undefined;
       record[3] = isContract;
     });
   }
   await pool.all();
   console.log({ top100 });
-  fs.writeFileSync("activeWalletsLastUsed.csv", arrayToCsv(sortBy(result, _ => -Number(_[1]))));
+  fs.writeFileSync(`activeWalletsLastUsed-${today}.csv`, arrayToCsv(result));
+};
+
+const getFuseBalances = async (refetch = true) => {
+  if (refetch === false) {
+    const balances = JSON.parse(fs.readFileSync("activeWalletsBalances.json").toString());
+    const rows = Object.entries(balances).filter(_ => Number(_[1].balance) > 10000000);
+    const sorted = sortBy(rows, _ => -Number(_[1].balance));
+    sorted.forEach(_ => (_[1].balance = Number(_[1].balance) / 100));
+    console.log(arrayToCsv(sorted.slice(0, 20).map(_ => [_[0], _[1].balance])));
+    return balances;
+  }
+  let result = [];
+  let balances = {};
+  let curPage = 1;
+  let maxResult;
+
+  do {
+    const pages = range(curPage, curPage + 5, 1);
+    curPage += 5;
+    const ps = pages.map(p =>
+      fetch(
+        `https://explorer.fuse.io/api?module=token&action=getTokenHolders&contractaddress=0x495d133B938596C9984d462F007B676bDc57eCEC&page=${p}&offset=10000`
+      )
+        .then(_ => _.json())
+        .then(_ => _.result)
+    );
+    const results = await Promise.all(ps);
+    result = result.concat(...results);
+    maxResult = maxBy(results, "length");
+    console.log(maxResult.length, result.length);
+  } while (maxResult.length === 10000);
+  result.forEach(r => (balances[r.address.toLowerCase()] = { balance: r.value, lastSeen: 0 }));
+  fs.writeFileSync("activeWalletsBalances.json", JSON.stringify(balances));
+  return balances;
 };
 
 const fundsByLastSeen = async () => {
@@ -163,6 +294,8 @@ const fundsByLastSeen = async () => {
 
   console.log({ total, total1Year, total2Year });
 };
+getFuseBalances(false).catch(e => console.log(e));
 // main().catch(e => console.log(e));
-// etl();
-fundsByLastSeen();
+// fix();
+etl();
+// fundsByLastSeen();