@gooddollar/goodprotocol 2.0.25-beta.7 → 2.0.25-beta.9

package/scripts/analytics/activeStakers.ts

@@ -0,0 +1,25 @@
+import { uniq } from "lodash";
+import fs from "fs";
+import { network, ethers } from "hardhat";
+import { Contract, Provider, setMulticallAddress } from "ethers-multicall";
+import release from "../../releases/deployment.json"
+import { SimpleStakingV2 } from "../../types";
+
+setMulticallAddress(122, "0x3CE6158b7278Bf6792e014FA7B4f3c6c46fe9410");
+
+setMulticallAddress(42220, "0x188C1bf697B66474dC3eaa119Ae691a8352537e3");
+
+const main = async () => {
+
+    const c1 = await ethers.getContractAt("SimpleStakingV2", release["production-mainnet"].StakingContractsV3[0][0]) as SimpleStakingV2
+    const c2 = await ethers.getContractAt("SimpleStakingV2", release["production-mainnet"].StakingContractsV3[1][0]) as SimpleStakingV2
+    const f = c1.filters.Staked()
+    const events = await c1.queryFilter(f, 14338550)
+    const events2 = await c2.queryFilter(f, 14338550)
+    const stakers = uniq(events.concat(events2).map(_ => _.args[0]))
+    console.log(stakers)
+    const res = (await Promise.all(stakers.map(async s => [s, await c1.balanceOf(s), await c2.balanceOf(s)]))).filter(_ => _[1].gt(0) || _[2].gt(0))
+    console.log(events.length)
+    console.log(res)
+}
+main()

package/scripts/analytics/activeUsersCount.ts

@@ -1,5 +1,7 @@
-import { range, chunk, uniq } from "lodash";
-import { ethers as Ethers } from "hardhat";
+import { range, chunk, uniq, mapValues, groupBy } from "lodash";
+import { ethers, ethers as Ethers } from "hardhat";
+import fs from "fs";
+import { bulkGetLastAuth } from "../utils";
 
 const ONE_DAY = 24 * 60 * 60;
 const ONE_HOUR = 60 * 60;
@@ -19,28 +21,27 @@ const main = async () => {
   for (let day of daysAgo) {
     const query = `
       { 
-        walletStats(first:1000, where: { lastClaimed_lte: ${curDay},lastClaimed_gt: ${
-      curDay - ONE_DAY
-    } isActiveUser: true }) {
+        walletStats(first:1000, where: { lastClaimed_lte: ${curDay},lastClaimed_gt: ${curDay - ONE_DAY
+      } isActiveUser: true }) {
         id        
       }
     }
   `;
 
     console.log("fetching inactive users since:", { curDay, day }, JSON.stringify({ query }));
-    const {
-      data: { walletStats }
-    } = await fetch("https://api.thegraph.com/subgraphs/name/gooddollar/gooddollarfuse", {
+    const result = await fetch("https://api.thegraph.com/subgraphs/name/gooddollar/gooddollarfuse", {
       method: "post",
       body: JSON.stringify({ query }),
       headers: { "Content-Type": "application/json" }
     }).then(_ => _.json());
 
-    console.log("got inactive wallets:", walletStats.length);
-    if (walletStats) {
-      const accounts = walletStats.map(_ => _.id);
-      allActive.push(...accounts);
-    }
+
+    console.log("got inactive wallets:", result);
+    // if (walletStats) {
+    //   const accounts = walletStats.map(_ => _.id);
+    //   allActive.push(...accounts);
+    // }
+
     // for (let tofish of chunk(accounts, 50)) {
     //   const tx = await ubiScheme.fishMulti(tofish, { gasLimit: 2000000 });
     //   console.log("fishing tx:", tx, tofish);
@@ -54,32 +55,83 @@ const main = async () => {
   console.log("active claimers:", unique.length);
 };
 
+const countClaimersUsingExplorer = async () => {
+  let claimers = [] //JSON.parse(fs.readFileSync("claimers.json").toString())
+  const curBlock = await ethers.provider.getBlockNumber()
+  const DAYS = 1
+  let fromBlock = curBlock - 60 * 60 * 24 * DAYS / 5 //roughly DAYS of blocks
+  const querySize = 90// 15 min of blocks
+  const blockRanges = range(fromBlock, curBlock, querySize)
+  const chunks = chunk(blockRanges, 10)
+  for (let idx in chunks) {
+    const toFetch = chunks[idx]
+    const ps = toFetch.map(async fromBlock => {
+      let toBlock = fromBlock + querySize
+      toBlock = toBlock > curBlock ? curBlock : toBlock
+      const result = await fetch(`https://explorer.celo.org/mainnet/api?module=logs&action=getLogs&fromBlock=${fromBlock}&toBlock=${toBlock}&address=0x43d72ff17701b2da814620735c39c620ce0ea4a1&topic0=0x89ed24731df6b066e4c5186901fffdba18cd9a10f07494aff900bdee260d1304`).then(_ => _.json())
+      fromBlock = toBlock
+      const addrs = result.result.map(_ => ethers.utils.defaultAbiCoder.decode(["address"], _.topics[1])[0])
+      claimers.push(...addrs);
+      console.log("claimers:", addrs.length, "block range:", { fromBlock, toBlock }, addrs[0])
+    })
+    await Promise.all(ps)
+    console.log(`${idx} out of ${chunks.length}`)
+  }
+
+  claimers = uniq(claimers)
+  fs.writeFileSync("claimers.json", JSON.stringify(claimers))
+}
+
+const countLastWhitelisted = async () => {
+  const claimers = JSON.parse(fs.readFileSync("claimers.json").toString())
+  const DAY = 60 * 60 * 24
+  let total = 0
+  let howOld = []
+  console.log("unique claimers:", claimers.length)
+  for (let bulk of chunk(claimers, 1000)) {
+    const results = await bulkGetLastAuth(bulk)
+    const older = results.filter(_ => _.toNumber() < Date.now() / 1000 - DAY * 180).length
+    bulk.forEach((_, i) => howOld.push([_, Math.ceil((Date.now() / 1000 - DAY * 180 - results[i].toNumber()) / (30 * DAY)).toFixed(0)]))
+    console.log(older, "out of", results.length)
+    total += older
+  }
+
+  howOld = howOld.filter(_ => Number(_[1]) > 0)
+  console.log({ total })
+  howOld = mapValues(groupBy(howOld, _ => _[1]), _ => _.length)
+  console.log(howOld)
+}
+
 const countActive = async () => {
-  const twoWeeksAgo = parseInt((Date.now() / 1000).toFixed(0)); //
 
-  const hoursAgo: number[] = range(0, 24 * 14, 1);
-  let curDay = twoWeeksAgo;
+  const hoursAgo: number[] = range(0, 24 * 30, 1);
+  let curDay = parseInt((Date.now() / 1000).toFixed(0));
   const allActive = [];
   for (let hour of hoursAgo) {
     const query = `
       { 
-        walletStats(first:1000, where: { lastClaimed_lte: ${curDay},lastClaimed_gt: ${
-      curDay - ONE_HOUR
-    } isActiveUser: true }) {
+        walletStats(first:1000, where: { lastClaimed_lte: ${curDay},lastClaimed_gt: ${curDay - ONE_HOUR
+      } isActiveUser: true }) {
         id        
       }
     }
   `;
 
     console.log("fetching active users since:", { curDay, hour }, JSON.stringify({ query }));
+    const result =
+
+      await fetch("https://gateway-arbitrum.network.thegraph.com/api/3c409250a317ce7b0c6a7a8b9a409ef8/subgraphs/id/F7314rxGdcpKPC1nN5KCoFW84EGRoUyzseY2sAT9PEkw", {
+        // await fetch("https://gateway.thegraph.com/api/3c409250a317ce7b0c6a7a8b9a409ef8/subgraphs/id/5cAhhzm7LSqGiFibV1odbbgZWiRmZsYjYrmaoj87UxFd", {
+        method: "post",
+        body: JSON.stringify({ query }),
+        headers: { "Content-Type": "application/json", "origin": "https://wallet.gooddollar.org" }
+
+      }).then(_ => _.json());
+
+    console.log(result)
     const {
       data: { walletStats }
-    } = await fetch("https://api.thegraph.com/subgraphs/name/gooddollar/gooddollarfuse", {
-      method: "post",
-      body: JSON.stringify({ query }),
-      headers: { "Content-Type": "application/json" }
-    }).then(_ => _.json());
-
+    } = result
     console.log("got active wallets:", walletStats.length);
     if (walletStats) {
       const accounts = walletStats.map(_ => _.id);
@@ -99,4 +151,6 @@ const countActive = async () => {
 };
 
 //main();
-countActive();
+// countActive();
+// countClaimersUsingExplorer()
+countLastWhitelisted()

package/scripts/analytics/activeWalletsStats.ts

@@ -1,24 +1,88 @@
-import { maxBy, range, sortBy } from "lodash";
+import { maxBy, range, sortBy, flatten } from "lodash";
 import PromisePool from "async-promise-pool";
 import fs from "fs";
 import { ethers } from "hardhat";
 
+const today = new Date().toLocaleDateString().replace(/\//g, "");
+console.log({ today });
 /****
  * Fetch token holders and their last activity date
  * can be used to create stats about active users and how much G$ isnt active
  */
 const main = async (chain = "fuse") => {
-  const explorer = chain === "fuse" ? "https://explorer.fuse.io" : "https://explorer.celo.org";
-  const subgraph = chain === "fuse" ? "https://api.thegraph.com/subgraphs/name/gooddollar/gooddollarfuse" : "https://";
+  let result = [];
+  let balances = {};
+  let curPage = 1;
+  let maxResult;
+  const fuseExplorer = "https://explorer.fuse.io/api";
+  const celoExplorer = "https://explorer.celo.org/mainnet/api";
+  const fuseSubgraph =
+    "https://gateway.thegraph.com/api/9048669a7632776aae01a191c4939445/subgraphs/id/5cAhhzm7LSqGiFibV1odbbgZWiRmZsYjYrmaoj87UxFd";
+  const celoSubgraph =
+    "https://gateway.thegraph.com/api/9048669a7632776aae01a191c4939445/subgraphs/id/F7314rxGdcpKPC1nN5KCoFW84EGRoUyzseY2sAT9PEkw";
+  do {
+    const pages = range(curPage, curPage + 5, 1);
+    curPage += 5;
+    const ps = pages.map(p =>
+      fetch(
+        `${fuseExplorer}?module=token&action=getTokenHolders&contractaddress=0x495d133B938596C9984d462F007B676bDc57eCEC&page=${p}&offset=10000`
+      )
+        .then(_ => _.json())
+        .then(_ => _.result)
+    );
+    const results = await Promise.all(ps);
+    result = result.concat(...results);
+    maxResult = maxBy(results, "length");
+    console.log(maxResult.length, result.length);
+  } while (maxResult.length === 10000);
+  result.forEach(
+    r =>
+      (balances[r.address.toLowerCase()] = {
+        balance: Number(r.value) / 100,
+        fuseBalance: Number(r.value) / 100,
+        lastSeen: 0
+      })
+  );
+
+  console.log("fetching celo balances....");
+  curPage = 1;
+  result = [];
+  do {
+    const pages = range(curPage, curPage + 3, 1);
+    curPage += 3;
+    const ps = pages.map(p =>
+      fetch(
+        `${celoExplorer}?module=token&action=getTokenHolders&contractaddress=0x62B8B11039FcfE5aB0C56E502b1C372A3d2a9c7A&page=${p}&offset=10000`
+      )
+        .then(_ => _.json())
+        .then(_ => _.result)
+    );
+    const results = await Promise.all(ps);
+    result = result.concat(...results);
+    maxResult = maxBy(results, "length");
+    console.log(maxResult.length, result.length);
+  } while (maxResult.length === 10000);
+
+  result.forEach(
+    r =>
+      (balances[r.address.toLowerCase()] = {
+        ...balances[r.address.toLowerCase()],
+        celoBalance: Number(r.value) / 1e18,
+        balance: Number(balances[r.address.toLowerCase()]?.fuseBalance || 0) + Number(r.value) / 1e18,
+        lastSeen: 0
+      })
+  );
+
+  fs.writeFileSync(`activeWalletsBalances-${today}.json`, JSON.stringify(balances));
 
-  const balances = await getFuseBalances();
+  balances = JSON.parse(fs.readFileSync(`activeWalletsBalances-${today}.json`).toString());
 
   const EPOCH = 60 * 60 * 6;
-  const pool = new PromisePool({ concurrency: 30 });
+  const pool = new PromisePool({ concurrency: 10 });
   const lastUsed = {};
   const epochs = range(1596045730, (Date.now() / 1000).toFixed(0), EPOCH);
 
-  const graphQuery = async (start, skip) => {
+  const graphQuery = async (start, skip, subgraph, retry = 3) => {
     const query = `{
         walletStats(first: 1000 skip:${skip} where: { dateAppeared_gte: ${start} dateAppeared_lt:${start + EPOCH} }) {
           id
@@ -31,7 +95,7 @@ const main = async (chain = "fuse") => {
       }`;
     // console.log({ query });
     try {
-      const { data = {}, errors } = await fetch("https://api.thegraph.com/subgraphs/name/gooddollar/gooddollarfuse", {
+      const { data = {}, errors } = await fetch(subgraph, {
         method: "POST",
         headers: {
           "Content-Type": "application/json"
@@ -39,27 +103,36 @@ const main = async (chain = "fuse") => {
         body: JSON.stringify({ query })
       }).then(_ => _.json());
       errors && console.log({ errors });
+      if (errors) {
+        console.log("query failed:", { subgraph, start, skip, retrying: retry > 0 });
+        if (retry > 0) {
+          return graphQuery(start, skip, subgraph, retry - 1);
+        }
+        return [];
+      }
+      // console.log("query ok:", { subgraph, start, skip, records: data.walletStats.length })
       if (data?.walletStats?.length === 1000) {
-        return data.walletStats.concat(await graphQuery(start, skip + 1000));
+        return data.walletStats.concat(await graphQuery(start, skip + 1000, subgraph));
       }
       return data.walletStats || [];
     } catch (error) {
-      console.log({ query, error });
+      console.log({ query, error, subgraph });
       return [];
     }
   };
   epochs.forEach(e => {
     pool.add(async () => {
-      const walletStats = await graphQuery(e, 0);
+      const walletStats = flatten(await Promise.all([graphQuery(e, 0, fuseSubgraph), graphQuery(e, 0, celoSubgraph)]));
       walletStats.forEach(w => {
         balances[w.id.toLowerCase()] = {
+          ...balances[w.id.toLowerCase()],
           lastSeen: Math.max(
+            balances[w.id.toLowerCase()]?.lastSeen,
             Number(w.lastClaimed),
             Number(w.lastTransactionFrom),
             Number(w.lastTransactionTo),
             Number(w.dateAppeared)
-          ),
-          balance: balances[w.id.toLowerCase()]?.balance || w.balance
+          )
         };
       });
       console.log({ curDate: e, records: walletStats.length });
@@ -67,7 +140,7 @@ const main = async (chain = "fuse") => {
   });
 
   await pool.all();
-  fs.writeFileSync("activeWalletsLastUsed.json", JSON.stringify(balances));
+  fs.writeFileSync(`activeWalletsLastUsed-${today}.json`, JSON.stringify(balances));
   //   console.log({ lastUsed });
 };
 
@@ -85,8 +158,46 @@ function arrayToCsv(data) {
     )
     .join("\r\n"); // rows starting on new lines
 }
+
+const fix = async () => {
+  const balances = JSON.parse(fs.readFileSync(`activeWalletsLastUsed-${today}.json`).toString());
+
+  let result = [];
+
+  let missing = 0;
+  for (let addr in balances) {
+    const r = balances[addr];
+    if (!r.fuseBalance && !r.celoBalance) {
+      console.log("missing:", addr);
+      missing += 1;
+      continue;
+    }
+    r.celoBalance = (r.celoBalance || 0) * 100;
+    r.balance = r.celoBalance + r.fuseBalance;
+  }
+
+  console.log("missing balance", missing);
+  console.log(sortBy(Object.entries(balances), _ => -_.balance).slice(0, 10));
+
+  fs.writeFileSync(`activeWalletsLastUsed-${today}.json`, JSON.stringify(balances));
+};
 const etl = async () => {
-  const balances = JSON.parse(fs.readFileSync("activeWalletsLastUsed.json").toString());
+  /** Convert a 2D array into a CSV string
+   */
+  function arrayToCsv(data) {
+    return data
+      .map(
+        row =>
+          row
+            .map(String) // convert every value to String
+            .map(v => v.replaceAll('"', '""')) // escape double colons
+            .map(v => `"${v}"`) // quote it
+            .join(",") // comma-separated
+      )
+      .join("\r\n"); // rows starting on new lines
+  }
+
+  const balances = JSON.parse(fs.readFileSync(`activeWalletsLastUsed-${today}.json`).toString());
 
   let result = [];
 
@@ -95,22 +206,30 @@ const etl = async () => {
     if (!r.balance) {
       continue;
     }
-    result.push([addr, r.balance / 100, r.lastSeen, false]);
+    result.push([addr, r.balance, r.lastSeen, false, r.fuseBalance, r.celoBalance]);
   }
+  result = sortBy(result, _ => -_[1]);
   const top100 = result.slice(0, 100);
   const pool = new PromisePool({ concurrency: 30 });
   const provider = new ethers.providers.JsonRpcBatchProvider("https://rpc.fuse.io");
+  const celoprovider = new ethers.providers.JsonRpcBatchProvider("https://forno.celo.org");
 
   for (let idx in top100) {
     pool.add(async () => {
       const record = top100[idx];
-      let isContract = (await provider.getCode(record[0]).catch(e => "0x")) !== "0x";
+      let isContract =
+        (
+          await Promise.all([
+            provider.getCode(record[0]).catch(e => "0x"),
+            celoprovider.getCode(record[0]).catch(e => "0x")
+          ])
+        ).find(_ => _ !== "0x") !== undefined;
       record[3] = isContract;
     });
   }
   await pool.all();
   console.log({ top100 });
-  fs.writeFileSync("activeWalletsLastUsed.csv", arrayToCsv(sortBy(result, _ => -Number(_[1]))));
+  fs.writeFileSync(`activeWalletsLastUsed-${today}.csv`, arrayToCsv(result));
 };
 
 const getFuseBalances = async (refetch = true) => {
@@ -177,5 +296,6 @@ const fundsByLastSeen = async () => {
 };
 getFuseBalances(false).catch(e => console.log(e));
 // main().catch(e => console.log(e));
-// etl();
+// fix();
+etl();
 // fundsByLastSeen();

package/scripts/fv.ts

@@ -1,9 +1,23 @@
-import fetch from "cross-fetch";
 import fs from "fs";
 import { chunk, uniqBy } from "lodash";
+import delay from "delay";
 import { bulkIsWhitelisted, bulkLastAuth } from "./utils";
 
 //create tunnel to fv server ssh -L 9090:server:8080 -N user@server -i sshkey
+
+const saveImages = async a => {
+  const i1 = await fetch("http://localhost:9090/enrollment-3d/" + a[1]).then(_ => _.json());
+  const i2 = await fetch("http://localhost:9090/enrollment-3d/" + a[2]).then(_ => _.json());
+  if (i1.auditTrailBase64 && i2.auditTrailBase64) {
+    fs.writeFileSync("fvimages/" + a.join("_") + "-a.jpg", i1.auditTrailBase64, {
+      encoding: "base64"
+    });
+    fs.writeFileSync("fvimages/" + a.join("_") + "-b.jpg", i2.auditTrailBase64, {
+      encoding: "base64"
+    });
+  } else console.log("not found", a, !!i1.auditTrailBase64, !!i2.auditTrailBase64);
+};
+
 const main = async () => {
   const data = JSON.parse(fs.readFileSync("fvtriplets2.txt").toString());
   const triplets = uniqBy(chunk(data, 3), _ => _.join("_"));
@@ -20,22 +34,125 @@ const main = async () => {
     return fs.existsSync(key) === false;
   });
   console.log({ notfetched });
-  const ps = notfetched.map(async a => {
-    const i1 = await fetch("http://localhost:9090/enrollment-3d/" + a[1]).then(_ => _.json());
-    const i2 = await fetch("http://localhost:9090/enrollment-3d/" + a[2]).then(_ => _.json());
-    if (i1.auditTrailBase64 && i2.auditTrailBase64) {
-      fs.writeFileSync("fvimages/" + a.join("_") + "-a.jpg", i1.auditTrailBase64, {
-        encoding: "base64"
-      });
-      fs.writeFileSync("fvimages/" + a.join("_") + "-b.jpg", i2.auditTrailBase64, {
-        encoding: "base64"
-      });
-    } else console.log("not found", a, !!i1.auditTrailBase64, !!i2.auditTrailBase64);
-  });
+  const ps = notfetched.map(saveImages);
 
   await Promise.all(ps);
 };
 
+// check which ids have been indexed, if not then make sure they are whitelisted
+const checkIndexedOrDelete = async () => {
+  const data = JSON.parse(
+    "[" + fs.readFileSync("fvids.json").toString().replace(/\n$/, "").replaceAll(/\n/g, ",\n") + "]"
+  );
+  const triplets = uniqBy(data, _ => _["wallet"]);
+
+  const accounts = triplets.map(_ => _["wallet"]);
+  console.log("Total accounts:", accounts.length, accounts[0]);
+  let whitelisted = [];
+  for (let batch of chunk(accounts, 200)) {
+    whitelisted.push(...(await bulkIsWhitelistedFuse(batch)));
+  }
+  const failed = triplets.filter(_ => whitelisted.includes(_["wallet"]) === false);
+
+  console.log("Total whitelisted:", whitelisted.length);
+  console.log("Total not whitelisted:", failed.length);
+  let ps = [];
+
+  ps = [];
+  for (let record of triplets) {
+    const isEnrolled = fetch("http://localhost:9090/enrollment-3d/" + record["identifier"].slice(0, 42))
+      .then(_ => _.json())
+      .then(_ => _.success);
+    // const isEnrolled = "unknown";
+    const isIndexed = fetch("http://localhost:9090/3d-db/get", {
+      method: "POST",
+      body: JSON.stringify({ identifier: record["identifier"].slice(0, 42), groupName: "GoodDollar" })
+    })
+      .then(_ => _.json())
+      .then(_ => _.success);
+
+    ps.push(
+      Promise.all([isEnrolled, isIndexed]).then(_ => ({
+        ...record,
+        isEnrolled: _[0],
+        isIndexed: _[1],
+        isWhitelisted: whitelisted.includes(record.wallet)
+      }))
+    );
+    if (ps.length % 50 === 0) {
+      console.log("waiting...", ps.length);
+      await Promise.all(ps);
+    }
+  }
+  console.log("waiting...", ps.length);
+  const r = await Promise.all(ps);
+  const toDelete = r.filter(
+    _ => (_.isWhitelisted && (!_.isEnrolled || !_.isIndexed)) || (!_.isWhitelisted && _.isIndexed && !_.isEnrolled)
+  );
+  const toWhitelist = r.filter(_ => !_.isWhitelisted && _.isEnrolled && _.isIndexed);
+
+  console.log("toDelete:", toDelete.length);
+
+  console.log("toWhitelist:", toWhitelist.length);
+  fs.writeFileSync("fvIndexedOrEnrolled.json", JSON.stringify(r));
+};
+
+const fixInvalidIndexed = async () => {
+  const indexedOrEnrolled = JSON.parse(fs.readFileSync("fvIndexedOrEnrolled.json").toString());
+  const toDelete = indexedOrEnrolled.filter(
+    _ => (_.isWhitelisted && (!_.isEnrolled || !_.isIndexed)) || (!_.isWhitelisted && _.isIndexed && !_.isEnrolled)
+  );
+  const toWhitelist = indexedOrEnrolled.filter(_ => !_.isWhitelisted && _.isEnrolled && _.isIndexed);
+
+  console.log("toDelete:", toDelete.length);
+
+  console.log("toWhitelist:", toWhitelist.length);
+
+  console.log(toWhitelist);
+  return;
+  let ps = [];
+  for (let record of toWhitelist) {
+    ps.push(fetch(`https://goodserver.gooddollar.org/syncWhitelist/${record.wallet}`).then(_ => _.json()));
+    if (ps.length > 0 && ps.length % 3 === 0) {
+      console.log("waiting...", ps.length);
+      await Promise.all(ps).catch(e => console.log(e));
+      console.log("waiting 60 seconds for rate limit");
+      await delay(50000);
+    }
+  }
+  if (ps.length > 0) {
+    console.log("waiting...", ps.length);
+    await Promise.all(ps).catch(e => console.log(e));
+  }
+
+  const removeWhitelist = toDelete.filter(_ => _.isWhitelisted).map(_ => _.wallet);
+  // removeWhitelist.push(...toWhitelist.map(_ => _.wallet));
+  ps = [];
+  for (let record of toDelete) {
+    if (record.isIndexed) {
+      // console.log("removing record from index and queuing for whitelist removal", record.wallet);
+      const isIndexed = fetch("http://localhost:9090/3d-db/delete", {
+        method: "POST",
+        body: JSON.stringify({ identifier: record["identifier"].slice(0, 42), groupName: "GoodDollar" })
+      }).then(_ => _.json());
+
+      ps.push(
+        isIndexed.then(_ => {
+          if (!_.success) throw new Error(`delete index failed: ${record.wallet}`);
+        })
+      );
+    }
+    if (ps.length > 0 && ps.length % 10 === 0) {
+      console.log("waiting...", ps.length);
+      await Promise.all(ps).catch(e => console.log(e));
+    }
+  }
+  console.log("waiting...", ps.length);
+  await Promise.all(ps).catch(e => console.log(e));
+  console.log({ removeWhitelist }, removeWhitelist.length);
+  fs.writeFileSync("removeWhitelist.json", JSON.stringify(removeWhitelist));
+};
+
 const deleteIdentifiers = async password => {
   const data = JSON.parse(fs.readFileSync("fvtriplets2.txt").toString());
   const triplets = uniqBy(chunk(data, 3), _ => _.join("_"));
@@ -89,7 +206,9 @@ const deleteIdentifiers = async password => {
   const res = await Promise.all(ps);
   console.log(res);
 };
-
+// checkIndexedOrDelete();
+fixInvalidIndexed();
+// console.log(process.env.ADMIN_PASSWORD);
+// deleteIdentifiers(process.env.ADMIN_PASSWORD);
 // main();
-console.log(process.env.ADMIN_PASSWORD);
-deleteIdentifiers(process.env.ADMIN_PASSWORD);
+saveImages(["", "", ""]);