@talismn/chaindata-provider 0.8.0 → 0.8.2

package/dist/talismn-chaindata-provider.esm.js

@@ -1,41 +1,126 @@
-// @dev : temporarily change branch here when testing changes in chaindata
-const CHAINDATA_BRANCH = "main";
+import { f as fetchChain, a as fetchSubstrateToken, b as fetchEvmNetwork, c as fetchChains, d as fetchEvmNetworks, g as githubTokenLogoUrl, e as githubUnknownTokenLogoUrl, h as fetchSubstrateTokens } from './net-il4EYhJN.esm.js';
+export { G as availableTokenLogoFilenames, y as chaindataChainByGenesisHashUrl, x as chaindataChainByIdUrl, v as chaindataChainsAllUrl, w as chaindataChainsSummaryUrl, B as chaindataEvmNetworkByIdUrl, z as chaindataEvmNetworksAllUrl, A as chaindataEvmNetworksSummaryUrl, E as chaindataMiniMetadatasAllUrl, D as chaindataTokenByIdUrl, C as chaindataTokensAllUrl, F as fetchMiniMetadatas, i as githubApi, j as githubCdn, s as githubChainLogoUrl, o as githubChaindataBaseUrl, m as githubChaindataBranch, q as githubChaindataChainsAssetsDir, n as githubChaindataDistDir, p as githubChaindataDistUrl, k as githubChaindataOrg, l as githubChaindataRepo, r as githubChaindataTokensAssetsDir, t as githubEvmNetworkLogoUrl, u as githubUnknownChainLogoUrl } from './net-il4EYhJN.esm.js';
+import { chains } from '../init/chains/dist/talismn-chaindata-provider-init-chains.esm.js';
+import { evmNetworks } from '../init/evm-networks/dist/talismn-chaindata-provider-init-evm-networks.esm.js';
+import { miniMetadatas } from '../init/mini-metadatas/dist/talismn-chaindata-provider-init-mini-metadatas.esm.js';
+import { tokens } from '../init/tokens/dist/talismn-chaindata-provider-init-tokens.esm.js';
+import { firstValueFrom, ReplaySubject, map } from 'rxjs';
+import { Dexie, liveQuery } from 'dexie';
+import anylogger from 'anylogger';
 
+// The init files imported in this module are generated by a script.
 //
-// Chaindata published files (dist folder)
+// They are used in two places:
+// 1. As fallbacks for fresh installations who are unable to pull an initial set of data from the upstream chaindata repo.
+// 2. As mock data for tests.
 //
+// They should be periodically updated with the latest state of chaindata.
+// You can update them by running the following command:
+// `pnpm chore:generate-init-data`
 
-const chaindataUrl = `https://raw.githubusercontent.com/TalismanSociety/chaindata/${CHAINDATA_BRANCH}/dist`;
-const chaindataChainsAllUrl = `${chaindataUrl}/chains/all.json`;
-const chaindataChainsSummaryUrl = `${chaindataUrl}/chains/summary.json`;
-const chaindataChainByIdUrl = chainId => `${chaindataUrl}/chains/byId/${chainId}.json`;
-const chaindataChainByGenesisHashUrl = genesisHash => `${chaindataUrl}/chains/byGenesisHash/${genesisHash}.json`;
-const chaindataEvmNetworksAllUrl = `${chaindataUrl}/evmNetworks/all.json`;
-const chaindataEvmNetworksSummaryUrl = `${chaindataUrl}/evmNetworks/summary.json`;
-const chaindataEvmNetworkByIdUrl = evmNetworkId => `${chaindataUrl}/evmNetworks/byId/${evmNetworkId}.json`;
-const chaindataTokensAllUrl = `${chaindataUrl}/tokens/all.json`;
-const chaindataTokenByIdUrl = tokenId => `${chaindataUrl}/tokens/byId/${tokenId}.json`;
-const chaindataMiniMetadatasAllUrl = `${chaindataUrl}/miniMetadatas/all.json`;
+const fetchInitChains = async () => chains;
+const fetchInitEvmNetworks = async () => evmNetworks;
+const fetchInitSubstrateTokens = async () => tokens;
+
+// TODO: Move `fetchMiniMetadatas` into `@talismn/balances`,
+// so that we don't have a circular import between `@talismn/balances` and `@talismn/chaindata-provider`.
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+const fetchInitMiniMetadatas = async () => miniMetadatas;
+
+/**
+ * Util to add our onfinality api key to any public onfinality RPC urls in an array of chains.
+ */
+const addCustomChainRpcs = (chains, onfinalityApiKey) => chains.map(chain => {
+  // copy chain instead of mutating
+  const chainWithCustomRpcs = {
+    ...chain
+  };
+  if (typeof onfinalityApiKey !== "string" || !onfinalityApiKey) return chainWithCustomRpcs;
+
+  // add rpcs
+  chainWithCustomRpcs.rpcs = (chainWithCustomRpcs.rpcs || []
+  // convert public onfinality rpc endpoints to private onfinality rpc endpoints
+  ).map(rpc => {
+    rpc.url = rpc.url.replace(/^wss:\/\/([A-z-]+)\.api\.onfinality\.io\/public-ws\/?$/, `wss://$1.api.onfinality.io/ws?apikey=${onfinalityApiKey}`);
+    return rpc;
+  })
+  // prioritise onfinality rpcs
+  .sort((a, b) => {
+    if (a.url.includes("api.onfinality.io")) return -1;
+    if (b.url.includes("api.onfinality.io")) return 1;
+    return 0;
+  });
+
+  // return copy
+  return chainWithCustomRpcs;
+});
 
 //
-// GitHub Repo Constants
+// Utils for parsing chaindata tokens.json
 //
 
-const githubApi = "https://api.github.com";
-const githubChaindataOrg = "TalismanSociety";
-const githubChaindataRepo = "chaindata";
-const githubChaindataBranch = CHAINDATA_BRANCH;
-const githubChaindataBaseUrl = `https://raw.githubusercontent.com/${githubChaindataOrg}/${githubChaindataRepo}/${githubChaindataBranch}`;
-const githubChainsUrl = `${githubChaindataBaseUrl}/data/chaindata.json`;
-const githubTestnetChainsUrl = `${githubChaindataBaseUrl}/data/testnets-chaindata.json`;
-const githubEvmNetworksUrl = `${githubChaindataBaseUrl}/data/evm-networks.json`;
-const githubChaindataChainsAssetsDir = "assets/chains";
-const githubChaindataTokensAssetsDir = "assets/tokens";
-const githubChainLogoUrl = chainId => `${githubChaindataBaseUrl}/${githubChaindataChainsAssetsDir}/${chainId}.svg`;
-const githubEvmNetworkLogoUrl = networkId => `${githubChaindataBaseUrl}/${githubChaindataChainsAssetsDir}/${networkId}.svg`;
-const githubTokenLogoUrl = tokenId => `${githubChaindataBaseUrl}/${githubChaindataTokensAssetsDir}/${tokenId}.svg`;
-const githubUnknownChainLogoUrl = githubChainLogoUrl("unknown");
-const githubUnknownTokenLogoUrl = githubTokenLogoUrl("unknown");
+const parseTokensResponse = tokens => tokens.filter(isTokenPartial).filter(isToken);
+const isTokenPartial = token => typeof token === "object" && token !== null;
+const isToken = token => {
+  const id = token.id;
+  if (typeof id !== "string") return false;
+  const type = token.type;
+  if (typeof type !== "string") return false;
+  const isTestnet = token.isTestnet;
+  if (typeof isTestnet !== "boolean") return false;
+  const symbol = token.symbol;
+  if (typeof symbol !== "string") return false;
+  const decimals = token.decimals;
+  if (typeof decimals !== "number") return false;
+  const logo = token.logo;
+  if (typeof logo !== "string") return false;
+
+  // coingeckoId can be undefined
+  // const coingeckoId = token.coingeckoId
+  // if (typeof coingeckoId !== "string") return false
+
+  return true;
+};
+
+//
+// map from Item[] to another type
+//
+
+const itemsToIds = items => items.map(({
+  id
+}) => id);
+const itemsToMapById = items => Object.fromEntries(items.map(item => [item.id, item]));
+const itemsToMapByGenesisHash = items => Object.fromEntries(items.flatMap(item => item.genesisHash ? [[item.genesisHash, item]] : []));
+
+//
+// filters for Item[] where Item.isCustom == true
+//
+
+const customChainsFilter = chains => chains.filter(chain => "isCustom" in chain && chain.isCustom);
+const customEvmNetworksFilter = evmNetworks => evmNetworks.filter(evmNetwork => "isCustom" in evmNetwork && evmNetwork.isCustom);
+const customTokensFilter = tokens => tokens.filter(token => "isCustom" in token && token.isCustom);
+
+//
+// Utils to wrap Observable methods with one-shot Promise methods
+//
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+const wrapObservableWithGetter = async (errorReason, observable) => {
+  return await withErrorReason(errorReason, () => firstValueFrom(observable));
+};
+const withErrorReason = async (reason, task) => {
+  try {
+    return await task();
+  } catch (cause) {
+    throw new Error(reason, {
+      cause
+    });
+  }
+};
+
+//
+// Utils which aren't used by this package, but are helpful for other packages
+//
 
 const isCustomChain = chain => {
   return "isCustom" in chain && chain.isCustom === true;
@@ -44,4 +129,730 @@ const isCustomEvmNetwork = evmNetwork => {
   return "isCustom" in evmNetwork && evmNetwork.isCustom === true;
 };
 
-export { chaindataChainByGenesisHashUrl, chaindataChainByIdUrl, chaindataChainsAllUrl, chaindataChainsSummaryUrl, chaindataEvmNetworkByIdUrl, chaindataEvmNetworksAllUrl, chaindataEvmNetworksSummaryUrl, chaindataMiniMetadatasAllUrl, chaindataTokenByIdUrl, chaindataTokensAllUrl, chaindataUrl, githubApi, githubChainLogoUrl, githubChaindataBaseUrl, githubChaindataBranch, githubChaindataChainsAssetsDir, githubChaindataOrg, githubChaindataRepo, githubChaindataTokensAssetsDir, githubChainsUrl, githubEvmNetworkLogoUrl, githubEvmNetworksUrl, githubTestnetChainsUrl, githubTokenLogoUrl, githubUnknownChainLogoUrl, githubUnknownTokenLogoUrl, isCustomChain, isCustomEvmNetwork };
+var packageJson = {
+	name: "@talismn/chaindata-provider",
+	version: "0.8.2",
+	author: "Talisman",
+	homepage: "https://talisman.xyz",
+	license: "GPL-3.0-or-later",
+	publishConfig: {
+		access: "public"
+	},
+	repository: {
+		directory: "packages/chaindata-provider",
+		type: "git",
+		url: "https://github.com/talismansociety/talisman.git"
+	},
+	main: "dist/talismn-chaindata-provider.cjs.js",
+	module: "dist/talismn-chaindata-provider.esm.js",
+	files: [
+		"/dist",
+		"/init",
+		"/net"
+	],
+	engines: {
+		node: ">=18"
+	},
+	scripts: {
+		test: "jest",
+		lint: "eslint src --max-warnings 0",
+		"chore:generate-init-data": "ts-node scripts/generateInitData.ts",
+		clean: "rm -rf dist init/*/dist net/dist .turbo node_modules"
+	},
+	dependencies: {
+		anylogger: "^1.0.11",
+		dexie: "^4.0.9",
+		rxjs: "^7.8.1"
+	},
+	devDependencies: {
+		"@talismn/eslint-config": "workspace:*",
+		"@talismn/tsconfig": "workspace:*",
+		"@types/jest": "^29.5.14",
+		eslint: "^8.57.1",
+		jest: "^29.7.0",
+		prettier: "^3.3.3",
+		"ts-jest": "^29.2.5",
+		"ts-node": "^10.9.2",
+		typescript: "^5.6.3"
+	},
+	preconstruct: {
+		entrypoints: [
+			"index.ts",
+			"init/chains.ts",
+			"init/evm-networks.ts",
+			"init/mini-metadatas.ts",
+			"init/tokens.ts",
+			"net.ts"
+		]
+	},
+	eslintConfig: {
+		root: true,
+		"extends": [
+			"@talismn/eslint-config/base"
+		]
+	}
+};
+
+var log = anylogger(packageJson.name);
+
+const subNativeTokenId = chainId => `${chainId}-substrate-native`.toLowerCase().replace(/ /g, "-");
+const evmNativeTokenId = chainId => `${chainId}-evm-native`.toLowerCase().replace(/ /g, "-");
+
+// for DB version 2, Wallet version 1.21.0
+const upgradeRemoveSymbolFromNativeTokenId = async tx => {
+  const tokensTable = tx.table("tokens");
+  const chainsTable = tx.table("chains");
+  const evmNetworksTable = tx.table("evmNetworks");
+  const nativeTokens = (await tokensTable.toArray()).filter(t => ["substrate-native", "evm-native"].includes(t.type));
+  const chains = await chainsTable.toArray();
+  const evmNetworks = await evmNetworksTable.toArray();
+  const tokenIdsToDelete = [];
+  const tokensToUpsert = [];
+  const chainsToUpsert = [];
+  const evmNetworksToUpsert = [];
+  for (const nativeToken of nativeTokens) {
+    const networkId = nativeToken.chain?.id || nativeToken.evmNetwork?.id;
+    if (!networkId) continue;
+    const id = nativeToken.type === "substrate-native" ? subNativeTokenId(networkId) : nativeToken.type === "evm-native" ? evmNativeTokenId(networkId) : undefined;
+    if (!id) continue;
+    const chain = chains.find(({
+      id
+    }) => id === networkId);
+    const evmNetwork = evmNetworks.find(({
+      id
+    }) => id === networkId);
+    tokenIdsToDelete.push(nativeToken.id);
+    tokensToUpsert.push({
+      ...nativeToken,
+      id
+    });
+    if (chain) chainsToUpsert.push({
+      ...chain,
+      nativeToken: {
+        id
+      }
+    });
+    if (evmNetwork) evmNetworksToUpsert.push({
+      ...evmNetwork,
+      nativeToken: {
+        id
+      }
+    });
+  }
+  await tokensTable.bulkPut(tokensToUpsert);
+  await chainsTable.bulkPut(chainsToUpsert);
+  await evmNetworksTable.bulkPut(evmNetworksToUpsert);
+  await tokensTable.bulkDelete(tokenIdsToDelete);
+};
+
+// for DB version 2, Wallet version 1.21.0
+const upgradeAddIsDefaultToExistingChains = async tx => {
+  const chainsTable = tx.table("chains");
+  const evmNetworksTable = tx.table("evmNetworks");
+  const tokensTable = tx.table("tokens");
+  await chainsTable.toCollection().modify(chain => {
+    if ("isCustom" in chain && chain.isCustom) return;
+    chain.isDefault = true;
+  });
+  await evmNetworksTable.toCollection().modify(evmNetwork => {
+    if ("isCustom" in evmNetwork && evmNetwork.isCustom) return;
+    evmNetwork.isDefault = true;
+  });
+  await tokensTable.toCollection().modify(token => {
+    if ("isCustom" in token && token.isCustom) return;
+    token.isDefault = true;
+  });
+};
+
+class TalismanChaindataDatabase extends Dexie {
+  constructor() {
+    super("TalismanChaindata");
+
+    // https://dexie.org/docs/Tutorial/Design#database-versioning
+    this.version(2).stores({
+      // You only need to specify properties that you wish to index.
+      // The object store will allow any properties on your stored objects but you can only query them by indexed properties
+      // https://dexie.org/docs/API-Reference#declare-database
+      //
+      // Never index properties containing images, movies or large (huge) strings. Store them in IndexedDB, yes! but just don’t index them!
+      // https://dexie.org/docs/Version/Version.stores()#warning
+      chains: "id, genesisHash, name",
+      evmNetworks: "id, name",
+      tokens: "id, type, symbol, coingeckoId, dcentName, contractAddress"
+    }).upgrade(upgradeRemoveSymbolFromNativeTokenId).upgrade(upgradeAddIsDefaultToExistingChains);
+  }
+}
+new TalismanChaindataDatabase();
+
+// removes the need to reference @talismn/balances in this package. should we ?
+const getNativeTokenId = (chainId, moduleType) => `${chainId}-${moduleType}`.toLowerCase().replace(/ /g, "-");
+const minimumHydrationInterval = 300_000; // 300_000ms = 300s = 5 minutes
+
+class ChaindataProvider {
+  #db;
+  #onfinalityApiKey;
+  #liveQueries;
+  #lastHydratedChainsAt = 0;
+  #lastHydratedEvmNetworksAt = 0;
+  #lastHydratedTokensAt = 0;
+  constructor(options) {
+    this.#db = new TalismanChaindataDatabase();
+    this.#onfinalityApiKey = options?.onfinalityApiKey ?? undefined;
+    this.#liveQueries = [liveQuery(() => this.#db.chains.toArray()).subscribe(this.chainsObservable), liveQuery(() => this.#db.evmNetworks.toArray()).subscribe(this.evmNetworksObservable), liveQuery(() => this.#db.tokens.toArray()).subscribe(this.tokensObservable)];
+  }
+  destroy() {
+    this.#liveQueries.forEach(subscription => subscription.unsubscribe());
+  }
+  setOnfinalityApiKey(apiKey) {
+    this.#onfinalityApiKey = apiKey;
+  }
+
+  //
+  // base items
+  //
+
+  chainsObservable = new ReplaySubject(1);
+  async chains() {
+    return await wrapObservableWithGetter("Failed to get chains", this.chainsObservable);
+  }
+  evmNetworksObservable = new ReplaySubject(1);
+  async evmNetworks() {
+    return await wrapObservableWithGetter("Failed to get evmNetworks", this.evmNetworksObservable);
+  }
+  tokensObservable = new ReplaySubject(1);
+  async tokens() {
+    return await wrapObservableWithGetter("Failed to get tokens", this.tokensObservable);
+  }
+
+  //
+  // custom item observables
+  //
+
+  get customChainsObservable() {
+    return this.chainsObservable.pipe(map(customChainsFilter));
+  }
+  async customChains() {
+    return await wrapObservableWithGetter("Failed to get custom chains", this.customChainsObservable);
+  }
+  get customEvmNetworksObservable() {
+    return this.evmNetworksObservable.pipe(map(customEvmNetworksFilter));
+  }
+  async customEvmNetworks() {
+    return await wrapObservableWithGetter("Failed to get custom evmNetworks", this.customEvmNetworksObservable);
+  }
+  get customTokensObservable() {
+    return this.tokensObservable.pipe(map(customTokensFilter));
+  }
+  async customTokens() {
+    return await wrapObservableWithGetter("Failed to get custom tokens", this.customTokensObservable);
+  }
+
+  //
+  // item ids
+  //
+
+  get chainIdsObservable() {
+    return this.chainsObservable.pipe(map(itemsToIds));
+  }
+  async chainIds() {
+    return await wrapObservableWithGetter("Failed to get chainIds", this.chainIdsObservable);
+  }
+  get evmNetworkIdsObservable() {
+    return this.evmNetworksObservable.pipe(map(itemsToIds));
+  }
+  async evmNetworkIds() {
+    return await wrapObservableWithGetter("Failed to get evmNetworkIds", this.evmNetworkIdsObservable);
+  }
+  get tokenIdsObservable() {
+    return this.tokensObservable.pipe(map(itemsToIds));
+  }
+  async tokenIds() {
+    return await wrapObservableWithGetter("Failed to get tokenIds", this.tokenIdsObservable);
+  }
+
+  //
+  // items by id
+  //
+
+  get chainsByIdObservable() {
+    return this.chainsObservable.pipe(map(itemsToMapById));
+  }
+  async chainsById() {
+    return await wrapObservableWithGetter("Failed to get chains by id", this.chainsByIdObservable);
+  }
+  get evmNetworksByIdObservable() {
+    return this.evmNetworksObservable.pipe(map(itemsToMapById));
+  }
+  async evmNetworksById() {
+    return await wrapObservableWithGetter("Failed to get evmNetworks by id", this.evmNetworksByIdObservable);
+  }
+  get tokensByIdObservable() {
+    return this.tokensObservable.pipe(map(itemsToMapById));
+  }
+  async tokensById() {
+    return await wrapObservableWithGetter("Failed to get tokens by id", this.tokensByIdObservable);
+  }
+  async tokensByIdForType(type) {
+    const tokensByIdForTypeObservable = this.tokensObservable.pipe(map(tokens => tokens.filter(token => token.type === type))).pipe(map(itemsToMapById));
+    return await wrapObservableWithGetter("Failed to get tokenIds", tokensByIdForTypeObservable);
+  }
+
+  //
+  // items by genesisHash
+  //
+
+  get chainsByGenesisHashObservable() {
+    return this.chainsObservable.pipe(map(itemsToMapByGenesisHash));
+  }
+  async chainsByGenesisHash() {
+    return await wrapObservableWithGetter("Failed to get chains by genesisHash", this.chainsByGenesisHashObservable);
+  }
+
+  //
+  // filters for a single item
+  //
+
+  async chainById(chainId) {
+    return await withErrorReason("Failed to get chain by id", async () => (await this.chainsById())[chainId] ?? null);
+  }
+  async chainByGenesisHash(genesisHash) {
+    return await withErrorReason("Failed to get chain by genesisHash", async () => (await this.chainsByGenesisHash())[genesisHash] ?? null);
+  }
+  async evmNetworkById(evmNetworkId) {
+    return await withErrorReason("Failed to get evmNetwork by id", async () => (await this.evmNetworksById())[evmNetworkId] ?? null);
+  }
+  async tokenById(tokenId) {
+    return await withErrorReason("Failed to get token by id", async () => (await this.tokensById())[tokenId] ?? null);
+  }
+
+  //
+  // mutations / methods with side-effects
+  //
+
+  async addCustomChain(customChain) {
+    try {
+      if (!("isCustom" in customChain && customChain.isCustom)) return;
+      return await this.#db.chains.put(customChain);
+    } catch (cause) {
+      throw new Error("Failed to add custom chain", {
+        cause
+      });
+    }
+  }
+  async removeCustomChain(chainId) {
+    try {
+      return await this.#db.chains
+      // only affect custom chains
+      .filter(chain => "isCustom" in chain && chain.isCustom)
+      // only affect the provided chainId
+      .filter(chain => chain.id === chainId)
+      // delete the chain (if exists)
+      .delete();
+    } catch (cause) {
+      throw new Error("Failed to remove custom chain", {
+        cause
+      });
+    }
+  }
+  async setCustomChains(chains) {
+    return await this.#db.transaction("rw", this.#db.chains, async () => {
+      const keys = await this.#db.chains.filter(chain => "isCustom" in chain && chain.isCustom).primaryKeys();
+      await this.#db.chains.bulkDelete(keys);
+      await this.#db.chains.bulkPut(chains.filter(chain => chain.isCustom));
+    });
+  }
+  async resetChain(chainId) {
+    const builtInChain = await fetchChain(chainId);
+    if (!builtInChain) throw new Error("Cannot reset non-built-in chain");
+    if (!builtInChain.nativeToken?.id) throw new Error("Failed to lookup native token (no token exists for chain)");
+    const builtInNativeToken = await fetchSubstrateToken(builtInChain?.nativeToken?.id);
+    if (!isTokenPartial(builtInNativeToken)) throw new Error("Failed to lookup native token");
+    if (!isToken(builtInNativeToken)) throw new Error("Failed to lookup native token (isToken test failed)");
+    try {
+      return await this.#db.transaction("rw", this.#db.chains, this.#db.tokens, async () => {
+        // delete chain and its native tokens (ensures cleanup of tokens with legacy ids)
+        await this.#db.tokens.filter(token => token.type === "substrate-native" && token.chain?.id === chainId).delete();
+        await this.#db.chains.delete(chainId);
+
+        // reprovision them from subsquid data
+        await this.#db.chains.put(builtInChain);
+        await this.#db.tokens.put(builtInNativeToken);
+      });
+    } catch (cause) {
+      throw new Error("Failed to reset chain", {
+        cause
+      });
+    }
+  }
+  async addCustomEvmNetwork(customEvmNetwork) {
+    try {
+      if (!("isCustom" in customEvmNetwork && customEvmNetwork.isCustom)) return Promise.resolve();
+      return await this.#db.evmNetworks.put(customEvmNetwork);
+    } catch (cause) {
+      throw new Error("Failed to add custom evm network", {
+        cause
+      });
+    }
+  }
+  async removeCustomEvmNetwork(evmNetworkId) {
+    if (await this.getIsBuiltInEvmNetwork(evmNetworkId)) throw new Error("Cannot remove built-in EVM network");
+    try {
+      return this.#db.transaction("rw", [this.#db.evmNetworks, this.#db.tokens], async () => {
+        await this.#db.evmNetworks.delete(evmNetworkId);
+        await this.#db.tokens.filter(token => token.evmNetwork?.id === evmNetworkId).delete();
+      });
+    } catch (cause) {
+      throw new Error("Failed to remove custom evm network", {
+        cause
+      });
+    }
+  }
+  async setCustomEvmNetworks(networks) {
+    return await this.#db.transaction("rw", this.#db.evmNetworks, async () => {
+      const keys = await this.#db.evmNetworks.filter(network => "isCustom" in network && network.isCustom).primaryKeys();
+      await this.#db.evmNetworks.bulkDelete(keys);
+      await this.#db.evmNetworks.bulkPut(networks.filter(network => network.isCustom));
+    });
+  }
+  async resetEvmNetwork(evmNetworkId) {
+    const builtInEvmNetwork = await fetchEvmNetwork(evmNetworkId);
+    if (!builtInEvmNetwork) throw new Error("Cannot reset non-built-in EVM network");
+    const nativeModule = builtInEvmNetwork.balancesConfig.find(c => c.moduleType === "evm-native");
+    if (!nativeModule?.moduleConfig) throw new Error("Failed to lookup native token (no token exists for network)");
+    const {
+      symbol,
+      decimals,
+      coingeckoId,
+      logo,
+      mirrorOf,
+      dcentName,
+      noDiscovery
+    } = nativeModule.moduleConfig;
+    if (!symbol) throw new Error("Missing native token symbol");
+    if (!decimals) throw new Error("Missing native token decimals");
+    const builtInNativeToken = {
+      id: getNativeTokenId(evmNetworkId, "evm-native"),
+      type: "evm-native",
+      evmNetwork: {
+        id: evmNetworkId
+      },
+      isTestnet: builtInEvmNetwork.isTestnet ?? false,
+      isDefault: true,
+      symbol,
+      decimals,
+      coingeckoId,
+      logo
+    };
+    if (mirrorOf) builtInNativeToken.mirrorOf = mirrorOf;
+    if (dcentName) builtInNativeToken.dcentName = dcentName;
+    if (noDiscovery) builtInNativeToken.noDiscovery = noDiscovery;
+    builtInEvmNetwork.nativeToken = {
+      id: builtInNativeToken.id
+    };
+    try {
+      return await this.#db.transaction("rw", this.#db.evmNetworks, this.#db.tokens, async () => {
+        // delete chain and its native tokens (ensures cleanup of tokens with legacy ids)
+        await this.#db.tokens.filter(token => token.type === "evm-native" && token.evmNetwork?.id === evmNetworkId).delete();
+        const networkToDelete = await this.#db.evmNetworks.get(evmNetworkId);
+        if (networkToDelete?.nativeToken?.id) await this.#db.tokens.delete(networkToDelete.nativeToken.id);
+        await this.#db.evmNetworks.delete(evmNetworkId);
+
+        // reprovision them from chaindata
+        await this.#db.evmNetworks.put(builtInEvmNetwork);
+        await this.#db.tokens.put(builtInNativeToken);
+      });
+    } catch (cause) {
+      throw new Error("Failed to reset evm network", {
+        cause
+      });
+    }
+  }
+  async addCustomToken(customToken) {
+    try {
+      if (!("isCustom" in customToken && customToken.isCustom)) return Promise.resolve();
+      return await this.#db.tokens.put(customToken);
+    } catch (cause) {
+      throw new Error("Failed to add custom token", {
+        cause
+      });
+    }
+  }
+  async removeCustomToken(tokenId) {
+    try {
+      return await this.#db.tokens
+      // only affect custom tokens
+      .filter(token => "isCustom" in token && Boolean(token.isCustom))
+      // only affect the provided token
+      .filter(token => token.id === tokenId)
+      // delete the token (if exists)
+      .delete();
+    } catch (cause) {
+      throw new Error("Failed to remove custom token", {
+        cause
+      });
+    }
+  }
+  async setCustomTokens(tokens) {
+    // TODO custom tokens have to go into localstorage
+    return await this.#db.transaction("rw", this.#db.tokens, async () => {
+      const keys = await this.#db.tokens.filter(token => "isCustom" in token && Boolean(token.isCustom)).primaryKeys();
+      await this.#db.tokens.bulkDelete(keys);
+      await this.#db.tokens.bulkPut(tokens.filter(token => "isCustom" in token && token.isCustom));
+    });
+  }
+  async removeToken(tokenId) {
+    try {
+      return await this.#db.tokens.delete(tokenId);
+    } catch (cause) {
+      throw new Error("Failed to remove token", {
+        cause
+      });
+    }
+  }
+
+  /**
+   * Hydrate the db with the latest chaindata from subsquid.
+   *
+   * @returns A promise which resolves to true if any db table has been hydrated, or false if all hydration has been skipped.
+   */
+  async hydrate({
+    // chainsArgs, // hydrateChains has no args
+    // evmNetworksArgs, // hydrateEvmNetworks has no args
+    tokensArgs
+  } = {}) {
+    return (await Promise.all([
+    // call inner hydration methods
+    this.hydrateChains(), this.hydrateEvmNetworks(), this.hydrateSubstrateTokens(...(tokensArgs ? tokensArgs : []))])
+
+    // return true if any hydration occurred
+    ).some(Boolean);
+  }
+
+  /**
+   * Hydrate the db with the latest chains from subsquid.
+   * Hydration is skipped when the last successful hydration was less than minimumHydrationInterval ms ago.
+   *
+   * @returns A promise which resolves to true if the db has been hydrated, or false if the hydration was skipped.
+   */
+  async hydrateChains() {
+    const now = Date.now();
+    if (now - this.#lastHydratedChainsAt < minimumHydrationInterval) return false;
+    const dbHasChains = (await this.#db.chains.count()) > 0;
+    try {
+      try {
+        var chains = addCustomChainRpcs(await fetchChains(), this.#onfinalityApiKey); // eslint-disable-line no-var
+        if (chains.length <= 0) throw new Error("Ignoring empty chaindata chains response");
+      } catch (error) {
+        if (dbHasChains) throw error;
+
+        // On first start-up (db is empty), if we fail to fetch chains then we should
+        // initialize the DB with the list of chains inside our init/chains.json file.
+        // This data will represent a relatively recent copy of what's in the squid,
+        // which will be better for our users than to have nothing at all.
+        var chains = addCustomChainRpcs(await fetchInitChains(), this.#onfinalityApiKey); // eslint-disable-line no-var
+      }
+
+      // TODO check if alec is this the right way to set native token
+      // note : many chains don't have a native module provisionned from chaindata => breaks edit network screen and probably send funds and tx screens
+      for (const chain of chains) {
+        const nativeTokenModule = chain.balancesConfig.find(c => c.moduleType === "substrate-native");
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        const symbol = nativeTokenModule?.moduleConfig?.symbol;
+        if (!symbol) continue;
+        chain.nativeToken = {
+          id: getNativeTokenId(chain.id, "substrate-native")
+        };
+      }
+      await this.#db.transaction("rw", this.#db.chains, async () => {
+        await this.#db.chains.filter(chain => !("isCustom" in chain && chain.isCustom)).delete();
+
+        // add all except ones matching custom existing ones (user may customize built-in chains)
+        const customChainIds = (await this.#db.chains.toArray()).map(chain => chain.id);
+        const newChains = chains.filter(chain => !customChainIds.includes(chain.id));
+        await this.#db.chains.bulkPut(newChains);
+      });
+      this.#lastHydratedChainsAt = now;
+      return true;
+    } catch (error) {
+      log.warn(`Failed to hydrate chains from chaindata`, error);
+      return false;
+    }
+  }
+
+  /**
+   * Hydrate the db with the latest evmNetworks from subsquid.
+   * Hydration is skipped when the last successful hydration was less than minimumHydrationInterval ms ago.
+   *
+   * @returns A promise which resolves to true if the db has been hydrated, or false if the hydration was skipped.
+   */
+  async hydrateEvmNetworks() {
+    const now = Date.now();
+    if (now - this.#lastHydratedEvmNetworksAt < minimumHydrationInterval) return false;
+    const dbHasEvmNetworks = (await this.#db.evmNetworks.count()) > 0;
+    try {
+      try {
+        var evmNetworks = await fetchEvmNetworks(); // eslint-disable-line no-var
+        if (evmNetworks.length <= 0) throw new Error("Ignoring empty chaindata evmNetworks response");
+      } catch (error) {
+        if (dbHasEvmNetworks) throw error;
+
+        // On first start-up (db is empty), if we fail to fetch evmNetworks then we should
+        // initialize the DB with the list of evmNetworks inside our init/evm-networks.json file.
+        // This data will represent a relatively recent copy of what's in the squid,
+        // which will be better for our users than to have nothing at all.
+        var evmNetworks = await fetchInitEvmNetworks(); // eslint-disable-line no-var
+      }
+
+      // set native token
+      for (const evmNetwork of evmNetworks) {
+        evmNetwork.nativeToken = {
+          id: getNativeTokenId(evmNetwork.id, "evm-native")
+        };
+      }
+      await this.#db.transaction("rw", this.#db.evmNetworks, async () => {
+        await this.#db.evmNetworks.filter(network => !("isCustom" in network && network.isCustom)).delete();
+
+        // remaining entries are custom networks
+        const existingCustomNetworks = await this.#db.evmNetworks.toArray();
+        const existingCustomNetworksById = Object.fromEntries(existingCustomNetworks.map(network => [network.id, network]));
+
+        // dont override custom networks, except for the balancesConfig property
+        const newNetworks = evmNetworks.map(network => {
+          const existing = existingCustomNetworksById[network.id];
+          return existing ? Object.assign({}, existing, {
+            balancesConfig: network.balancesConfig
+          }) : network;
+        });
+        await this.#db.evmNetworks.bulkPut(newNetworks);
+      });
+      this.#lastHydratedEvmNetworksAt = now;
+      return true;
+    } catch (error) {
+      log.warn(`Failed to hydrate evmNetworks from chaindata`, error);
+      return false;
+    }
+  }
+  async updateChainTokens(chainId, source, newTokens, availableTokenLogoFilenames) {
+    // TODO: Test logos and fall back to unknown token logo url
+    // (Maybe put the test into each balance module itself)
+
+    const existingChainTokens = await this.#db.tokens.filter(token => token.chain?.id === chainId && token.type === source).toArray();
+    newTokens.forEach(token => {
+      if (token.logo) return;
+      const symbolLogo = token.symbol.toLowerCase().replace(/ /g, "_");
+      if (availableTokenLogoFilenames.includes(`${symbolLogo}.svg`)) {
+        return token.logo = githubTokenLogoUrl(symbolLogo);
+      }
+
+      // TODO: Use coingeckoId logo if exists
+
+      return token.logo = githubUnknownTokenLogoUrl;
+    });
+    const notCustomTokenIds = existingChainTokens.filter(token => !("isCustom" in token && token.isCustom)).map(token => token.id);
+    const customTokenIds = existingChainTokens.filter(token => "isCustom" in token && token.isCustom).map(token => token.id);
+    await this.#db.transaction("rw", this.#db.tokens, this.#db.chains, async () => {
+      await this.#db.tokens.bulkDelete(notCustomTokenIds);
+      await this.#db.tokens.bulkPut(newTokens.filter(token => !customTokenIds.includes(token.id)));
+      //if (chain && shouldUpdateChain) await this.#db.chains.put(chain)
+    });
+  }
+  async updateEvmNetworkTokens(newTokens) {
+    const existingEvmNetworkTokens = await this.#db.tokens.filter(t => t.type.startsWith("evm-")).toArray();
+    const isCustomToken = token => "isCustom" in token && token.isCustom;
+
+    // don't override custom tokens
+    const customTokenIds = new Set();
+
+    // delete non-custom tokens which aren't in `newTokens`
+    const deleteTokenIds = new Set();
+    for (const token of existingEvmNetworkTokens) {
+      if (isCustomToken(token)) customTokenIds.add(token.id);else deleteTokenIds.add(token.id);
+    }
+    const tokensToUpdate = newTokens.filter(token => {
+      deleteTokenIds.delete(token.id);
+      return !customTokenIds.has(token.id);
+    });
+    this.#db.transaction("rw", this.#db.tokens, async () => {
+      // delete all existing non custom tokens
+      await this.#db.tokens.bulkDelete([...deleteTokenIds]);
+
+      // force update on all non custom tokens
+      await this.#db.tokens.bulkPut(tokensToUpdate);
+    });
+  }
+
+  /**
+   * Hydrate the db with the latest tokens from subsquid.
+   * Hydration is skipped when the last successful hydration was less than minimumHydrationInterval ms ago.
+   *
+   * @returns A promise which resolves to true if the db has been hydrated, or false if the hydration was skipped.
+   */
+  async hydrateSubstrateTokens(chainIdFilter) {
+    const now = Date.now();
+    if (now - this.#lastHydratedTokensAt < minimumHydrationInterval) return false;
+    const dbHasTokens = (await this.#db.tokens.count()) > 0;
+    try {
+      try {
+        var tokens = parseTokensResponse(await fetchSubstrateTokens()); // eslint-disable-line no-var
+        if (tokens.length <= 0) throw new Error("Ignoring empty chaindata tokens response");
+      } catch (error) {
+        if (dbHasTokens) throw error;
+
+        // On first start-up (db is empty), if we fail to fetch tokens then we should
+        // initialize the DB with the list of tokens inside our init/tokens.json file.
+        // This data will represent a relatively recent copy of what's in the squid,
+        // which will be better for our users than to have nothing at all.
+        var tokens = parseTokensResponse(await fetchInitSubstrateTokens()); // eslint-disable-line no-var
+      }
+      await this.#db.transaction("rw", this.#db.tokens, async () => {
+        const deleteChains = chainIdFilter ? new Set(chainIdFilter) : undefined;
+        const tokensToDelete = (await this.#db.tokens.toArray()).filter(token => {
+          // don't delete custom tokens
+          if ("isCustom" in token && token.isCustom) return false;
+
+          // delete all other tokens if chainIdFilter is not specified
+          if (deleteChains === undefined) return true;
+
+          // delete tokens on chainIdFilter chains is it is specified
+          if (token.chain?.id && deleteChains.has(token.chain.id)) return true;
+          return false;
+        }).map(token => token.id);
+        if (tokensToDelete.length) await this.#db.tokens.bulkDelete(tokensToDelete);
+
+        // add all except ones matching custom existing ones (user may customize built-in tokens)
+        const customTokenIds = (await this.#db.tokens.toArray()).map(token => token.id);
+        const newTokens = tokens.filter(token => {
+          // don't replace custom tokens
+          if (customTokenIds.includes(token.id)) return false;
+          if (deleteChains === undefined) return true;
+          if (!token.chain?.id) return true;
+          if (deleteChains.has(token.chain.id)) return true;
+          return false;
+        });
+        await this.#db.tokens.bulkPut(newTokens);
+      });
+      this.#lastHydratedTokensAt = now;
+      return true;
+    } catch (error) {
+      log.warn(`Failed to hydrate tokens from chaindata`, error);
+      return false;
+    }
+  }
+  async getIsBuiltInChain(chainId) {
+    const chain = await fetchChain(chainId);
+    return !!chain;
+  }
+  async getIsBuiltInEvmNetwork(evmNetworkId) {
+    try {
+      const evmNetwork = await fetchEvmNetwork(evmNetworkId);
+      return !!evmNetwork;
+    } catch (e) {
+      return false;
+    }
+  }
+  async transaction(mode, tables, scope) {
+    return await this.#db.transaction(mode, tables, scope);
+  }
+}
+
+export { ChaindataProvider, addCustomChainRpcs, customChainsFilter, customEvmNetworksFilter, customTokensFilter, fetchChain, fetchChains, fetchEvmNetwork, fetchEvmNetworks, fetchInitChains, fetchInitEvmNetworks, fetchInitMiniMetadatas, fetchInitSubstrateTokens, fetchSubstrateToken, fetchSubstrateTokens, githubTokenLogoUrl, githubUnknownTokenLogoUrl, isCustomChain, isCustomEvmNetwork, isToken, isTokenPartial, itemsToIds, itemsToMapByGenesisHash, itemsToMapById, parseTokensResponse, withErrorReason, wrapObservableWithGetter };