@sabaaa1/common 0.0.1

package/src/data-structures/MultiThreadedUtxoUtils/MultiThreadedUtxoUtils.ts

@@ -0,0 +1,152 @@
+import { EncryptedOutputWithSign, OutputsWithSignAndLast } from '../../types/commitments.types';
+import { IMultiThreadedUtxoUtils } from '../../types/IMultiThreadedUtxoUtils';
+
+import { stealthAddressCacheDevice, stealthPairCacheDevice, UserKeys } from '../crypto-keys/keys';
+import { IUtxoConstructor, Utxo } from '../utxo/Utxo';
+import {
+  DecipherProcessPayload,
+  UtxoWorkerActionType,
+  UtxoWorkerPayload,
+} from '../../webworker/utxoWorker/utxoWorker.types';
+import { performTaskWithWorker } from '../../webworker/performTaskWithWorker';
+import { WorkerVariant } from '../../webworker/worker.registry';
+import { isNode } from '../../constants/vite.constants';
+
+declare global {
+  interface Navigator {
+    deviceMemory: number | undefined;
+  }
+}
+
+export class MultiThreadedUtxoUtils implements IMultiThreadedUtxoUtils {
+  public readonly NUM_WORKERS: number;
+
+  public readonly NUM_WORKERS_BATCH_PROCESS: number;
+
+  constructor() {
+    if (isNode) {
+      this.NUM_WORKERS = 1;
+      this.NUM_WORKERS_BATCH_PROCESS = 1;
+    } else {
+      this.NUM_WORKERS = 3; // 3 seems to offer the best performance, on my machine at least
+
+      // in general the more workers we use with batchProcess the better the performance even if there are
+      // less cores / threads on device, however the problem is that
+      // if the number is too large we'll run out of memory and fail to fetch balance
+      // to not encounter any problems we limit the amount of workers to amount of GiB RAM the user has
+      // this will never be more than 8, navigator.deviceMemory will only report the following values:
+      // 0.25, 0.5, 1, 2, 4, 8
+      // in website is running in http mode navigator.deviceMemory might not be available, we assume it to be 4 if it's not
+      this.NUM_WORKERS_BATCH_PROCESS = Math.ceil(navigator.deviceMemory ?? 4);
+    }
+  }
+
+  private async createUtxoPromise<T>(message: UtxoWorkerPayload) {
+    const data = await performTaskWithWorker<UtxoWorkerPayload, T>({
+      type: WorkerVariant.UTXO,
+      payload: message,
+    });
+    return data;
+  }
+
+  async batchFilterUtxosWithNullifier(allUtxos: Utxo[], nullifiers: Set<string>): Promise<Utxo[]> {
+    const slices = this.sliceArrayIntoParts(allUtxos, this.NUM_WORKERS);
+
+    const workerResponsePromises = slices.map((utxoListSlice) =>
+      this.createUtxoPromise<{
+        utxoConstructors: IUtxoConstructor[];
+        stealthPairCache: Record<string, string>;
+        stealthAddressCache: Record<string, string>;
+      }>({
+        type: UtxoWorkerActionType.BATCH_FILTER_UTXOS_WITH_NULLIFIER,
+        data: {
+          utxos: utxoListSlice.map((utxo) => utxo.getConstructableParams()),
+          nullifiers,
+        },
+      }),
+    );
+    const workerResults = await Promise.all(workerResponsePromises);
+    workerResults.forEach((item) => {
+      stealthPairCacheDevice.mergeWithSerialized(item.stealthPairCache);
+      stealthAddressCacheDevice.mergeWithSerialized(item.stealthAddressCache);
+    });
+    return workerResults.flatMap((item) => item.utxoConstructors).map((serializedUtxo) => new Utxo(serializedUtxo));
+  }
+
+  async buildBatchProcess(encryptedOutputs: EncryptedOutputWithSign[], userKeys: UserKeys) {
+    const slices = this.sliceArrayIntoParts(encryptedOutputs, this.NUM_WORKERS);
+
+    const messageResponsePromises = slices.map((encryptedOutputsSlice) =>
+      this.createUtxoPromise<IUtxoConstructor[]>({
+        type: UtxoWorkerActionType.BUILD_UTXOS,
+        data: { signature: userKeys.getSignature(), encryptedOutputs: encryptedOutputsSlice },
+      }),
+    );
+    const messageResponses = await Promise.all(messageResponsePromises);
+    const outputs = messageResponses.flat();
+    const result = outputs.map((item) => new Utxo(item));
+    return result;
+  }
+
+  sliceArrayIntoParts<T>(arr: T[], numberOfParts: number) {
+    const lengthOfSubarray = Math.ceil(arr.length / numberOfParts);
+    const subarrays = Array.from({ length: numberOfParts }, (_value, index) =>
+      arr.slice(lengthOfSubarray * index, lengthOfSubarray * (index + 1)),
+    );
+    return subarrays;
+  }
+
+  findLastOutput(lastOutputsArray: string[], lastOutput: string) {
+    let newLastOutput = lastOutput;
+
+    lastOutputsArray.forEach((value) => {
+      if (value !== lastOutput) newLastOutput = value;
+    });
+
+    return newLastOutput;
+  }
+
+  async batchProcess(encryptedOutputs: EncryptedOutputWithSign[], lastOutput: string, userKeys: UserKeys) {
+    const slices = this.sliceArrayIntoParts(encryptedOutputs, this.NUM_WORKERS_BATCH_PROCESS);
+
+    const signature = userKeys.getSignature();
+
+    const messageResponsePromises = slices.map((encryptedOutputsSlice) => {
+      const message: DecipherProcessPayload = {
+        type: UtxoWorkerActionType.DECIPHER_OUTPUTS,
+        data: {
+          encryptedOutputs: encryptedOutputsSlice,
+          lastOutput, // doesn't matter what we pass here, it will not be used regardless
+          signature,
+        },
+      };
+
+      return this.createUtxoPromise<OutputsWithSignAndLast>(message);
+    });
+
+    const workersResponses = await Promise.all(messageResponsePromises);
+
+    const resolvedEncryptedOutputs = workersResponses
+      .map(({ additionalEncryptedOutputs }) => additionalEncryptedOutputs)
+      .flat();
+
+    const uniqueValues = new Set<string>();
+    const uniqueEncryptedOutputs = resolvedEncryptedOutputs.reduce<EncryptedOutputWithSign[]>((prev, cur) => {
+      if (uniqueValues.has(cur.value)) return prev;
+      uniqueValues.add(cur.value);
+      return [...prev, cur];
+    }, []);
+
+    const newLastOutput = this.findLastOutput(
+      workersResponses.map((response) => response.lastOutput),
+      lastOutput,
+    );
+
+    const outputsWithSignAndLast: OutputsWithSignAndLast = {
+      additionalEncryptedOutputs: uniqueEncryptedOutputs,
+      lastOutput: newLastOutput ?? lastOutput, // keep the last output the same if no new output is present
+    };
+
+    return outputsWithSignAndLast;
+  }
+}

package/src/data-structures/TokenDBs/AlchemyPublicTokensDB.ts

@@ -0,0 +1,224 @@
+import { get, update } from 'idb-keyval';
+import { API } from '../../API/API';
+import { ALCHEMY_API_KEY, chainIds } from '../../constants/chains.constants';
+import { TokenType, zeroAddress } from '../../constants/protocol.constants';
+import { caseInsensitiveEqual, lowerCaseIncludes } from '../../functions/utils/caseInsensitive.utils';
+import { getPublicBalanceByTokenAddress } from '../../functions/utils/publicBalance.utils';
+import { getStateKey } from '../../functions/utils/string.utils';
+import { StorageKeys } from '../../types/cache.types';
+import { ERC20Token } from '../../types/token.types';
+import { IPublicTokensDB } from './IPublicTokensDB';
+import { tokenVisibilityStorage } from './token-visibility-db';
+import { httpClient } from '../tor';
+
+interface AlchemyResponse {
+  result: {
+    tokenBalances: { contractAddress: string; tokenBalance: string }[];
+    address: string;
+    pageKey?: string;
+  };
+}
+
+interface AlchemyNFTResponse {
+  ownedNfts: { contractAddress: string; tokenId: string }[];
+  pageKey: string;
+}
+
+export class AlchemyPublicTokensDB implements IPublicTokensDB {
+  private fetchedTokens: Record<string, ERC20Token[]> = {};
+
+  private readonly ALCHEMY_URLS = {
+    [chainIds.ethMainnet]: `eth-mainnet`,
+    [chainIds.polygon]: `polygon-mainnet`,
+    [chainIds.arbMainnet]: `arb-mainnet`,
+    [chainIds.optimism]: `opt-mainnet`,
+    [chainIds.base]: `base-mainnet`,
+    [chainIds.avalanche]: `avax-mainnet`,
+  };
+
+  // Known legitimate NFT collections that should be included in fallback
+  private readonly KNOWN_GOOD_NFTS: Record<number, string[]> = {
+    [chainIds.base]: [
+      '0x20bc8d248fe9c18a947406cea50d716828ab821f', // Hinkal Mason NFT
+    ],
+    [chainIds.optimism]: [
+      '0x3c3e0e73b53a19bcaf4178b8f9c9428fd2a2775e', // Uniswap v4 Positions
+    ],
+  };
+
+  private isWhitelistedNFT(address: string, chainId: number): boolean {
+    // Only allow known good NFT collections in the fallback to avoid spam
+    if (this.KNOWN_GOOD_NFTS[chainId]) {
+      return this.KNOWN_GOOD_NFTS[chainId].some((goodAddress) => caseInsensitiveEqual(address, goodAddress));
+    }
+    return false;
+  }
+
+  public async getPublicTokens(chainId: number, walletAddress: string): Promise<ERC20Token[]> {
+    const key = getStateKey(walletAddress, chainId);
+    const publicTokens = await get<Record<string, ERC20Token[]>>(StorageKeys.ALCHEMY_PUBLIC_TOKENS);
+    return publicTokens?.[key] || [];
+  }
+
+  public getFetchedTokens(chainId: number, walletAddress: string): ERC20Token[] {
+    const key = getStateKey(walletAddress, chainId);
+    return this.fetchedTokens[key] || [];
+  }
+
+  public async fetchAndUpdatePublicTokens(walletAddress: string, chainId: number) {
+    const stateKey = getStateKey(walletAddress, chainId);
+
+    const erc20TokenAddresses = await this.getERC20TokenAddresses(chainId, walletAddress);
+
+    const nftTokensInfo = await this.getNFTTokens(chainId, walletAddress);
+    const nftTokenAddresses = nftTokensInfo.map((nft) => nft.contractAddress);
+
+    const tokenAddresses = [...erc20TokenAddresses, ...nftTokenAddresses];
+
+    if (!tokenAddresses.length) return;
+
+    const tokensFromBackend = await API.tokensInfoCall(chainId, tokenAddresses);
+
+    // Fallback: Get minimal NFT metadata when backend returns null but Alchemy confirms it's indeed an NFT
+    // ONLY for NFTs with valid tokenIds (user actually owns them)
+    const erc20Tokens = tokensFromBackend
+      .map((token, index) => {
+        if (token) return token;
+
+        // Check if this address is an NFT that Alchemy found
+        const tokenAddress = tokenAddresses[index];
+        const nftInfo = nftTokensInfo.find((nft) => caseInsensitiveEqual(nft.contractAddress, tokenAddress));
+
+        if (nftInfo && nftInfo.tokenIds && nftInfo.tokenIds.length > 0) {
+          if (!this.isWhitelistedNFT(tokenAddress, chainId)) {
+            return null;
+          }
+          return {
+            chainId,
+            name: 'Unknown NFT',
+            symbol: 'NFT',
+            erc20TokenAddress: tokenAddress,
+            decimals: 0,
+            nftTokenType: TokenType.ERC721,
+            tokenIds: nftInfo.tokenIds,
+          } as ERC20Token;
+        }
+
+        return null;
+      })
+      .filter((token): token is ERC20Token => !!token);
+
+    const hiddenTokenAddresses = await tokenVisibilityStorage.hideSpamsAndFetchHiddenTokensAddresses(
+      chainId,
+      erc20Tokens,
+    );
+    const nonHiddenErc20Tokens = erc20Tokens.filter(
+      (token) => !lowerCaseIncludes(hiddenTokenAddresses, token.erc20TokenAddress), // filter out hidden tokens
+    );
+
+    const erc20TokensWithIds = nonHiddenErc20Tokens.map((token) => ({
+      ...token,
+      tokenIds: nftTokensInfo.find((nft) => caseInsensitiveEqual(nft.contractAddress, token.erc20TokenAddress))
+        ?.tokenIds,
+    }));
+
+    await update(StorageKeys.ALCHEMY_PUBLIC_TOKENS, (pubTokens: Record<string, ERC20Token[]> = {}) => {
+      const newPublicTokens = {
+        ...pubTokens,
+        [stateKey]: erc20TokensWithIds,
+      };
+      this.fetchedTokens = newPublicTokens;
+      return newPublicTokens;
+    });
+  }
+
+  private async getERC20TokenAddresses(chainId: number, walletAddress: string): Promise<string[]> {
+    const nativeTokenBalance = await getPublicBalanceByTokenAddress(chainId, walletAddress, zeroAddress);
+
+    const alchemyUrl = `https://${this.ALCHEMY_URLS[chainId]}.g.alchemy.com/v2/${ALCHEMY_API_KEY}`;
+    const allBalances: { contractAddress: string; tokenBalance: string }[] = [];
+    let pageKey: string | undefined;
+
+    do {
+      const params: (string | { pageKey: string })[] = [walletAddress, 'erc20'];
+      if (pageKey) params.push({ pageKey });
+
+      // eslint-disable-next-line no-await-in-loop
+      const response = await httpClient.post<AlchemyResponse>(alchemyUrl, {
+        jsonrpc: '2.0',
+        id: 1,
+        method: 'alchemy_getTokenBalances',
+        params,
+      });
+
+      const { result } = response;
+      allBalances.push(...(result?.tokenBalances ?? []));
+      pageKey = result?.pageKey;
+    } while (pageKey);
+
+    if (nativeTokenBalance && nativeTokenBalance > 0n) {
+      allBalances.push({
+        contractAddress: zeroAddress,
+        tokenBalance: nativeTokenBalance.toString(),
+      });
+    }
+
+    const uniqueBalances: Record<string, { contractAddress: string; tokenBalance: string }> = {};
+    allBalances.forEach((token) => {
+      if (!uniqueBalances[token.contractAddress] || BigInt(token.tokenBalance) > 0n) {
+        uniqueBalances[token.contractAddress] = token;
+      }
+    });
+
+    return Object.keys(uniqueBalances);
+  }
+
+  private async getNFTTokens(
+    chainId: number,
+    walletAddress: string,
+  ): Promise<{ contractAddress: string; tokenIds: string[] }[]> {
+    const baseUrl = `https://${this.ALCHEMY_URLS[chainId]}.g.alchemy.com/nft/v3/${ALCHEMY_API_KEY}/getNFTsForOwner`;
+
+    const allNfts: AlchemyNFTResponse['ownedNfts'] = [];
+    let pageKey: string | undefined;
+
+    // Handle pagination to get all NFTs
+    do {
+      const params = new URLSearchParams({
+        owner: walletAddress,
+        withMetadata: 'false',
+        pageSize: '100', // max is 100
+      });
+
+      if (pageKey) {
+        params.append('pageKey', pageKey);
+      }
+
+      // eslint-disable-next-line no-await-in-loop
+      const response = await httpClient.get<AlchemyNFTResponse>(`${baseUrl}?${params.toString()}`);
+
+      const { ownedNfts = [], pageKey: newPageKey } = response;
+
+      allNfts.push(...ownedNfts);
+      pageKey = newPageKey;
+    } while (pageKey);
+
+    const validNfts = allNfts.filter(({ contractAddress, tokenId }) => contractAddress && tokenId);
+
+    const groupedByContract = validNfts.reduce((acc: Map<string, string[]>, { contractAddress, tokenId }) => {
+      const contract = contractAddress.toLowerCase();
+      if (!acc.has(contract)) {
+        acc.set(contract, []);
+      }
+      acc.get(contract)?.push(tokenId);
+      return acc;
+    }, new Map<string, string[]>());
+
+    return Array.from(groupedByContract.entries()).map(([contractAddress, tokenIds]) => ({
+      contractAddress,
+      tokenIds,
+    }));
+  }
+}
+
+export const alchemyPublicTokensDB = new AlchemyPublicTokensDB();

package/src/data-structures/TokenDBs/ArcPublicTokensDB.ts

@@ -0,0 +1,198 @@
+import axios from 'axios';
+import { getContractWithFetcherByChainId } from '../../functions/web3/getContractMetadata';
+import { ContractType } from '../../types/ethereum-network.types';
+import { Logger } from '../../error-handling/logger';
+import { StorageKeys } from '../../types/cache.types';
+import { ERC20Token } from '../../types/token.types';
+import { getStateKey } from '../../functions/utils/string.utils';
+import { get, update } from 'idb-keyval';
+import { API } from '../../API/API';
+import { caseInsensitiveEqual, lowerCaseIncludes } from '../../functions/utils/caseInsensitive.utils';
+import { TokenType } from '../../constants/protocol.constants';
+import { tokenVisibilityStorage } from './token-visibility-db';
+import { chainIds } from '../../constants/chains.constants';
+
+interface ApiResponse {
+  message: string;
+  result: { contractAddress: string }[];
+  status: string;
+}
+
+interface NFTResponse {
+  items: NFTItem[];
+  next_page_params: NextPageParams | null;
+}
+
+interface NFTItem {
+  id: string;
+  token: { address_hash: string };
+}
+
+interface NextPageParams {
+  token_contract_address_hash: string;
+  token_id: string;
+  token_type: string;
+  [key: string]: string;
+}
+
+export class ArcPublicTokensDB {
+  private readonly API_BASE = 'https://testnet.arcscan.app/api';
+
+  private fetchedTokens: Record<string, ERC20Token[]> = {};
+
+  public async getPublicTokens(walletAddress: string): Promise<ERC20Token[]> {
+    const key = getStateKey(walletAddress, chainIds.arcTestnet);
+    const publicTokens = await get<Record<string, ERC20Token[]>>(StorageKeys.ARC_PUBLIC_TOKENS);
+    return publicTokens?.[key] || [];
+  }
+
+  public getFetchedTokens(walletAddress: string): ERC20Token[] {
+    const key = getStateKey(walletAddress, chainIds.arcTestnet);
+    return this.fetchedTokens[key] || [];
+  }
+
+  public async fetchAndUpdatePublicTokens(walletAddress: string) {
+    const stateKey = getStateKey(walletAddress, chainIds.arcTestnet);
+
+    const erc20TokenAddresses = await this.getERC20TokenAddresses(walletAddress);
+
+    const nftTokensInfo = await this.getNFTTokens(walletAddress);
+    const nftTokenAddresses = nftTokensInfo.map((nft) => nft.contractAddress);
+
+    const tokenAddresses = [...erc20TokenAddresses, ...nftTokenAddresses];
+
+    if (!tokenAddresses.length) return;
+
+    const tokensFromBackend = await API.tokensInfoCall(chainIds.arcTestnet, tokenAddresses);
+
+    // Fallback: Get minimal NFT metadata when backend returns null but arcscan confirms it's indeed an NFT
+    // ONLY for NFTs with valid tokenIds (user actually owns them)
+    const erc20Tokens = tokensFromBackend
+      .map((token, index) => {
+        if (token) return token;
+
+        // Check if this address is an NFT that arcscan found
+        const tokenAddress = tokenAddresses[index];
+        const nftInfo = nftTokensInfo.find((nft) => caseInsensitiveEqual(nft.contractAddress, tokenAddress));
+
+        if (nftInfo && nftInfo.tokenIds && nftInfo.tokenIds.length > 0) {
+          return {
+            chainId: chainIds.arcTestnet,
+            name: 'Unknown NFT',
+            symbol: 'NFT',
+            erc20TokenAddress: tokenAddress,
+            decimals: 0,
+            nftTokenType: TokenType.ERC721,
+            tokenIds: nftInfo.tokenIds,
+          } as ERC20Token;
+        }
+
+        return null;
+      })
+      .filter((token): token is ERC20Token => !!token);
+
+    const hiddenTokenAddresses = await tokenVisibilityStorage.hideSpamsAndFetchHiddenTokensAddresses(
+      chainIds.arcTestnet,
+      erc20Tokens,
+    );
+    const nonHiddenErc20Tokens = erc20Tokens.filter(
+      (token) => !lowerCaseIncludes(hiddenTokenAddresses, token.erc20TokenAddress), // filter out hidden tokens
+    );
+
+    const erc20TokensWithIds = nonHiddenErc20Tokens.map((token) => ({
+      ...token,
+      tokenIds: nftTokensInfo.find((nft) => caseInsensitiveEqual(nft.contractAddress, token.erc20TokenAddress))
+        ?.tokenIds,
+    }));
+
+    await update(StorageKeys.ARC_PUBLIC_TOKENS, (pubTokens: Record<string, ERC20Token[]> = {}) => {
+      const newPublicTokens = {
+        ...pubTokens,
+        [stateKey]: erc20TokensWithIds,
+      };
+      this.fetchedTokens = newPublicTokens;
+      return newPublicTokens;
+    });
+  }
+
+  private async getERC20TokenAddresses(walletAddress: string): Promise<string[]> {
+    // arc-testnet has native 18 decimal and normal 6 decimal usdc that's why we don't get balances of native usdc on arc testnet
+    const balances: { contractAddress: string; tokenBalance: string }[] = [];
+
+    try {
+      const tokenTxResponse = await axios.post<ApiResponse>(
+        `${this.API_BASE}?module=account&action=tokentx&address=${walletAddress}`,
+      );
+
+      const tokenTxData = tokenTxResponse.data;
+
+      if (tokenTxData.status === '1' && tokenTxData.result) {
+        const tokenAddresses = [...new Set(tokenTxData.result.map(({ contractAddress }) => contractAddress))];
+
+        // eslint-disable-next-line no-restricted-syntax
+        for (const tokenAddress of tokenAddresses) {
+          try {
+            const contract = getContractWithFetcherByChainId(
+              chainIds.arcTestnet,
+              ContractType.ERC20Contract,
+              tokenAddress as unknown as undefined,
+            );
+            // eslint-disable-next-line no-await-in-loop
+            const balance = BigInt(await contract['balanceOf'](walletAddress));
+            if (balance > 0n) {
+              balances.push({
+                contractAddress: tokenAddress,
+                tokenBalance: balance.toString(),
+              });
+            }
+          } catch (error) {
+            Logger.error('Error fetching ARC testnet token balance:', error);
+          }
+        }
+      }
+    } catch (error) {
+      Logger.error('Error fetching ARC testnet token transactions:', error);
+    }
+
+    return balances
+      .filter(({ tokenBalance }) => Number(tokenBalance) !== 0)
+      .map(({ contractAddress }) => contractAddress);
+  }
+
+  private async getNFTTokens(walletAddress: string): Promise<{ contractAddress: string; tokenIds: string[] }[]> {
+    const baseUrl = `${this.API_BASE}/v2/addresses/${walletAddress}/nft?type=ERC-721,ERC-1155`; // https://testnet.arcscan.app/api-docs?tab=rest_api  //get list of NFT owned by address
+    const allNfts: NFTResponse['items'] = [];
+    let nextPageParams: NextPageParams | undefined;
+
+    do {
+      const query = nextPageParams ? `&${new URLSearchParams(nextPageParams)}` : '';
+      // eslint-disable-next-line no-await-in-loop
+      const { data } = await axios.get<NFTResponse>(`${baseUrl}${query}`);
+
+      const { items = [], next_page_params: nextPageParamsResponse } = data;
+      allNfts.push(...items);
+      nextPageParams = nextPageParamsResponse
+        ? {
+            token_contract_address_hash: nextPageParamsResponse.token_contract_address_hash,
+            token_id: nextPageParamsResponse.token_id,
+            token_type: nextPageParamsResponse.token_type,
+          }
+        : undefined;
+    } while (nextPageParams);
+
+    const groupedByContract = allNfts.reduce((acc: Map<string, string[]>, item) => {
+      const contractAddress = item.token.address_hash.toLowerCase();
+      const tokenId = item.id;
+      if (!acc.has(contractAddress)) acc.set(contractAddress, []);
+      acc.get(contractAddress)?.push(tokenId);
+      return acc;
+    }, new Map<string, string[]>());
+
+    return Array.from(groupedByContract.entries()).map(([contractAddress, tokenIds]) => ({
+      contractAddress,
+      tokenIds,
+    }));
+  }
+}
+
+export const arcPublicTokensDB = new ArcPublicTokensDB();