@talismn/balances 0.0.0-pr2043-20250618091117 → 0.0.0-pr2043-20250619003406

package/dist/talismn-balances.cjs.prod.js

@@ -1,8 +1,6 @@
 'use strict';
 
-var chaindataProvider = require('@talismn/chaindata-provider');
 var dexie = require('dexie');
-var rxjs = require('rxjs');
 var anylogger = require('anylogger');
 var tokenRates = require('@talismn/token-rates');
 var util = require('@talismn/util');
@@ -10,12 +8,14 @@ var BigNumber = require('bignumber.js');
 var util$1 = require('@polkadot/util');
 var utilCrypto = require('@polkadot/util-crypto');
 var pako = require('pako');
+var chaindataProvider = require('@talismn/chaindata-provider');
 var viem = require('viem');
 var isEqual = require('lodash/isEqual');
 var txwrapperCore = require('@substrate/txwrapper-core');
 var scale = require('@talismn/scale');
 var lodash = require('lodash');
 var camelCase = require('lodash/camelCase');
+var PQueue = require('p-queue');
 var sapi = require('@talismn/sapi');
 var types = require('@polkadot/types');
 var groupBy = require('lodash/groupBy');
@@ -23,8 +23,10 @@ var utils = require('@polkadot-api/utils');
 var polkadotApi = require('polkadot-api');
 var PromisePool = require('@supercharge/promise-pool');
 var chainConnector = require('@talismn/chain-connector');
+var rxjs = require('rxjs');
 var scaleTs = require('scale-ts');
 var upperFirst = require('lodash/upperFirst');
+var z = require('zod/v4');
 var apiContract = require('@polkadot/api-contract');
 
 function _interopDefault (e) { return e && e.__esModule ? e : { default: e }; }
@@ -34,9 +36,11 @@ var BigNumber__default = /*#__PURE__*/_interopDefault(BigNumber);
 var pako__default = /*#__PURE__*/_interopDefault(pako);
 var isEqual__default = /*#__PURE__*/_interopDefault(isEqual);
 var camelCase__default = /*#__PURE__*/_interopDefault(camelCase);
+var PQueue__default = /*#__PURE__*/_interopDefault(PQueue);
 var groupBy__default = /*#__PURE__*/_interopDefault(groupBy);
 var PromisePool__default = /*#__PURE__*/_interopDefault(PromisePool);
 var upperFirst__default = /*#__PURE__*/_interopDefault(upperFirst);
+var z__default = /*#__PURE__*/_interopDefault(z);
 
 // TODO: Document default balances module purpose/usage
 const DefaultBalanceModule = type => ({
@@ -120,13 +124,11 @@ class EvmTokenFetcher {
   // }
 }
 
-var packageJson = {
+var pkg = {
 	name: "@talismn/balances",
-	version: "0.0.0-pr2043-20250618091117"};
-
-const libVersion = packageJson.version;
+	version: "0.0.0-pr2043-20250619003406"};
 
-var log = anylogger__default.default(packageJson.name);
+var log = anylogger__default.default(pkg.name);
 
 function excludeFromTransferableAmount(locks) {
   if (typeof locks === "string") return BigInt(locks);
@@ -336,20 +338,13 @@ class Balances {
     return new SumBalancesFormatter(this);
   }
 }
-
-// type BalanceJsonEvm = BalanceJson & { evmNetworkId: string }
-
-// const isBalanceEvm = (balance: BalanceJson): balance is BalanceJsonEvm => "evmNetworkId" in balance
-
 const getBalanceId = balance => {
   const {
     source,
     address,
-    tokenId,
-    networkId
+    tokenId
   } = balance;
-  //const locationId = isBalanceEvm(balance) ? balance.evmNetworkId : balance.chainId
-  return [source, address, networkId, tokenId].filter(util.isTruthy).join("::");
+  return [source, address, tokenId].join("::");
 };
 
 /**
@@ -1061,257 +1056,6 @@ class TalismanBalancesDatabase extends dexie.Dexie {
 }
 const db = new TalismanBalancesDatabase();
 
-const minimumHydrationInterval = 300_000; // 300_000ms = 300s = 5 minutes
-
-/**
- * A substrate dapp needs access to a set of types when it wants to communicate with a blockchain node.
- *
- * These types are used to encode requests & decode responses via the SCALE codec.
- * Each chain generally has its own set of types.
- *
- * Substrate provides a construct to retrieve these types from a blockchain node.
- * The chain metadata.
- *
- * The metadata includes the types required for any communication with the chain,
- * including lots of methods which are not relevant to balance fetching.
- *
- * As such, the metadata can clock in at around 1-2MB per chain, which is a lot of storage
- * for browser-based dapps which want to connect to lots of chains.
- *
- * By utilizing the wonderful [scale-ts](https://github.com/unstoppablejs/unstoppablejs/tree/main/packages/scale-ts#readme) library,
- * we can trim the chain metadata down so that it only includes the types we need for balance fetching.
- *
- * Each balance module has a function to do just that, `BalanceModule::fetchSubstrateChainMeta`.
- *
- * But, we only want to run this operation when necessary.
- *
- * The purpose of this class, `MiniMetadataUpdater`, is to maintain a local cache of
- * trimmed-down metadatas, which we'll refer to as `MiniMetadatas`.
- */
-class MiniMetadataUpdater {
-  #lastHydratedMiniMetadatasAt = 0;
-  #lastHydratedCustomChainsAt = 0;
-  #chainConnectors;
-  #chaindataProvider;
-  #balanceModules;
-  constructor(chainConnectors, chaindataProvider, balanceModules) {
-    this.#chainConnectors = chainConnectors;
-    this.#chaindataProvider = chaindataProvider;
-    this.#balanceModules = balanceModules;
-  }
-
-  /** Subscribe to the metadata for a chain */
-  subscribe(chainId) {
-    return rxjs.from(dexie.liveQuery(() => db.miniMetadatas.filter(m => m.chainId === chainId).toArray().then(array => array[0])));
-  }
-  async update(chainIds) {
-    await this.updateSubstrateChains(chainIds);
-  }
-  async statuses(chains) {
-    const ids = await db.miniMetadatas.orderBy("id").primaryKeys();
-    const wantedIdsByChain = new Map(chains.flatMap(({
-      id: chainId,
-      specName,
-      specVersion
-    }) => {
-      if (specName === null) return [];
-      if (specVersion === null) return [];
-      return [[chainId, this.#balanceModules.filter(m => m.type.startsWith("substrate-")).map(({
-        type: source
-      }) => deriveMiniMetadataId({
-        source,
-        chainId,
-        specVersion,
-        libVersion
-      }))]];
-    }));
-    const statusesByChain = new Map(Array.from(wantedIdsByChain.entries()).map(([chainId, wantedIds]) => [chainId, wantedIds.every(wantedId => ids.includes(wantedId)) ? "good" : "none"]));
-    return {
-      wantedIdsByChain,
-      statusesByChain
-    };
-  }
-  async hydrateFromChaindata() {
-    // TODO review this. feels unnecessary to fetch them all
-
-    const now = Date.now();
-    if (now - this.#lastHydratedMiniMetadatasAt < minimumHydrationInterval) return false;
-    const dbHasMiniMetadatas = (await db.miniMetadatas.count()) > 0;
-    try {
-      try {
-        var miniMetadatas = await this.#chaindataProvider.miniMetadatas(); // eslint-disable-line no-var
-        if (miniMetadatas.length <= 0) throw new Error("Ignoring empty chaindata miniMetadatas response");
-      } catch (error) {
-        if (dbHasMiniMetadatas) throw error;
-        log.warn("Failed to fetch miniMetadatas from chaindata", error);
-        // On first start-up (db is empty), if we fail to fetch miniMetadatas then we should
-        // initialize the DB with the list of miniMetadatas inside our init/mini-metadatas.json file.
-        // This data will represent a relatively recent copy of what's in chaindata,
-        // which will be better for our users than to have nothing at all.
-        var miniMetadatas = await chaindataProvider.fetchInitMiniMetadatas(); // eslint-disable-line no-var
-      }
-      await db.miniMetadatas.bulkPut(miniMetadatas);
-      this.#lastHydratedMiniMetadatasAt = now;
-      return true;
-    } catch (error) {
-      log.warn(`Failed to hydrate miniMetadatas from chaindata`, error);
-      return false;
-    }
-  }
-  async hydrateCustomChains() {
-    // TODO
-    // const now = Date.now()
-    // if (now - this.#lastHydratedCustomChainsAt < minimumHydrationInterval) return false
-    // const chains = await this.#chaindataProvider.chains()
-    // const customChains = chains.filter(
-    //   (chain): chain is CustomChain => "isCustom" in chain && chain.isCustom,
-    // )
-    // const updatedCustomChains: Array<CustomChain> = []
-    // const concurrency = 4
-    // ;(
-    //   await PromisePool.withConcurrency(concurrency)
-    //     .for(customChains)
-    //     .process(async (customChain) => {
-    //       const send = (method: string, params: unknown[]) =>
-    //         this.#chainConnectors.substrate?.send(customChain.id, method, params)
-    //       const [genesisHash, runtimeVersion, chainName, chainType] = await Promise.all([
-    //         send("chain_getBlockHash", [0]),
-    //         send("state_getRuntimeVersion", []),
-    //         send("system_chain", []),
-    //         send("system_chainType", []),
-    //       ])
-    //       // deconstruct rpc data
-    //       const { specName, implName } = runtimeVersion
-    //       const specVersion = String(runtimeVersion.specVersion)
-    //       const changed =
-    //         customChain.genesisHash !== genesisHash ||
-    //         customChain.chainName !== chainName ||
-    //         !isEqual(customChain.chainType, chainType) ||
-    //         customChain.implName !== implName ||
-    //         customChain.specName !== specName ||
-    //         customChain.specVersion !== specVersion
-    //       if (!changed) return
-    //       customChain.genesisHash = genesisHash
-    //       customChain.chainName = chainName
-    //       customChain.chainType = chainType
-    //       customChain.implName = implName
-    //       customChain.specName = specName
-    //       customChain.specVersion = specVersion
-    //       updatedCustomChains.push(customChain)
-    //     })
-    // ).errors.forEach((error) => log.error("Error hydrating custom chains", error))
-    // if (updatedCustomChains.length > 0) {
-    //   await this.#chaindataProvider.transaction("rw", ["chains"], async () => {
-    //     for (const updatedCustomChain of updatedCustomChains) {
-    //       await this.#chaindataProvider.removeCustomChain(updatedCustomChain.id)
-    //       await this.#chaindataProvider.addCustomChain(updatedCustomChain)
-    //     }
-    //   })
-    // }
-    // if (updatedCustomChains.length > 0) this.#lastHydratedCustomChainsAt = now
-    // return true
-  }
-  async updateSubstrateChains(_chainIds) {
-    // const chains = new Map(
-    //   (await this.#chaindataProvider.chains()).map((chain) => [chain.id, chain]),
-    // )
-    // const filteredChains = chainIds.flatMap((chainId) => chains.get(chainId) ?? [])
-    // const ids = await balancesDb.miniMetadatas.orderBy("id").primaryKeys()
-    // const { wantedIdsByChain, statusesByChain } = await this.statuses(filteredChains)
-    // // clean up store
-    // const wantedIds = Array.from(wantedIdsByChain.values()).flatMap((ids) => ids)
-    // const unwantedIds = ids.filter((id) => !wantedIds.includes(id))
-    // if (unwantedIds.length > 0) {
-    //   const chainIds = Array.from(
-    //     new Set((await balancesDb.miniMetadatas.bulkGet(unwantedIds)).map((m) => m?.chainId)),
-    //   )
-    //   log.info(`Pruning ${unwantedIds.length} miniMetadatas on chains ${chainIds.join(", ")}`)
-    //   await balancesDb.miniMetadatas.bulkDelete(unwantedIds)
-    // }
-    // const needUpdates = Array.from(statusesByChain.entries())
-    //   .filter(([, status]) => status !== "good")
-    //   .map(([chainId]) => chainId)
-    // if (needUpdates.length > 0)
-    //   log.info(`${needUpdates.length} miniMetadatas need updates (${needUpdates.join(", ")})`)
-    // const availableTokenLogos = await availableTokenLogoFilenames().catch((error) => {
-    //   log.error("Failed to fetch available token logos", error)
-    //   return []
-    // })
-    // const concurrency = 12
-    // ;(
-    //   await PromisePool.withConcurrency(concurrency)
-    //     .for(needUpdates)
-    //     .process(async (chainId) => {
-    //       log.info(`Updating metadata for chain ${chainId}`)
-    //       const chain = chains.get(chainId)
-    //       if (!chain) return
-    //       const { specName, specVersion } = chain
-    //       if (specName === null) return
-    //       if (specVersion === null) return
-    //       const fetchMetadata = async () => {
-    //         try {
-    //           return await fetchBestMetadata(
-    //             (method, params, isCacheable) => {
-    //               if (!this.#chainConnectors.substrate)
-    //                 throw new Error("Substrate connector is not available")
-    //               return this.#chainConnectors.substrate.send(chainId, method, params, isCacheable)
-    //             },
-    //             true, // allow v14 fallback
-    //           )
-    //         } catch (err) {
-    //           log.warn(`Failed to fetch metadata for chain ${chainId}`)
-    //           return undefined
-    //         }
-    //       }
-    //       const [metadataRpc, systemProperties] = await Promise.all([
-    //         fetchMetadata(),
-    //         this.#chainConnectors.substrate?.send(chainId, "system_properties", []),
-    //       ])
-    //       for (const mod of this.#balanceModules.filter((m) => m.type.startsWith("substrate-"))) {
-    //         const balancesConfig = (chain.balancesConfig ?? []).find(
-    //           ({ moduleType }) => moduleType === mod.type,
-    //         )
-    //         const moduleConfig = balancesConfig?.moduleConfig ?? {}
-    //         const chainMeta = await mod.fetchSubstrateChainMeta(
-    //           chainId,
-    //           moduleConfig,
-    //           metadataRpc,
-    //           systemProperties,
-    //         )
-    //         const tokens = await mod.fetchSubstrateChainTokens(chainId, chainMeta, moduleConfig)
-    //         // update tokens in chaindata
-    //         await this.#chaindataProvider.updateChainTokens(
-    //           chainId,
-    //           mod.type,
-    //           Object.values(tokens),
-    //           availableTokenLogos,
-    //         )
-    //         // update miniMetadatas
-    //         const { miniMetadata: data, metadataVersion: version, ...extra } = chainMeta ?? {}
-    //         await balancesDb.miniMetadatas.put({
-    //           id: deriveMiniMetadataId({
-    //             source: mod.type,
-    //             chainId,
-    //             specName,
-    //             specVersion,
-    //             balancesConfig: JSON.stringify(moduleConfig),
-    //           }),
-    //           source: mod.type,
-    //           chainId,
-    //           specName,
-    //           specVersion,
-    //           balancesConfig: JSON.stringify(moduleConfig),
-    //           // TODO: Standardise return value from `fetchSubstrateChainMeta`
-    //           version,
-    //           data,
-    //           extra: JSON.stringify(extra),
-    //         })
-    //       }
-    //     })
-    // ).errors.forEach((error) => log.error("Error updating chain metadata", error))
-  }
-}
-
 const erc20Abi = [{
   constant: true,
   inputs: [],
@@ -2953,12 +2697,14 @@ async function getPoolBalance(publicClient, contractAddress, accountAddress) {
   }
 }
 
+const libVersion = pkg.version;
+
 // cache the promise so it can be shared across multiple calls
 const CACHE_GET_SPEC_VERSION = new Map();
 const fetchSpecVersion = async (chainConnector, networkId) => {
   const {
     specVersion
-  } = await chainConnector.send(networkId, "state_getRuntimeVersion", [true]);
+  } = await chainConnector.send(networkId, "state_getRuntimeVersion", [], true);
   return specVersion;
 };
 
@@ -3000,9 +2746,16 @@ const getMetadataRpc = async (chainConnector, networkId) => {
 
 // share requests as all modules will call this at once
 const CACHE = new Map();
-const getMiniMetadatas = async (chainConnector, chaindataProvider, networkId, specVersion) => {
+
+// ensures we dont fetch miniMetadatas on more than 4 chains at once
+const POOL = new PQueue__default.default({
+  concurrency: 4
+});
+const getMiniMetadatas = async (chainConnector, chaindataProvider, networkId, specVersion, signal) => {
   if (CACHE.has(networkId)) return CACHE.get(networkId);
-  const pResult = fetchMiniMetadatas(chainConnector, chaindataProvider, networkId, specVersion);
+  const pResult = POOL.add(() => fetchMiniMetadatas(chainConnector, chaindataProvider, networkId, specVersion), {
+    signal
+  });
   CACHE.set(networkId, pResult);
   try {
     return await pResult;
@@ -3014,49 +2767,57 @@ const getMiniMetadatas = async (chainConnector, chaindataProvider, networkId, sp
     CACHE.delete(networkId);
   }
 };
-const fetchMiniMetadatas = async (chainConnector, chaindataProvider, chainId, specVersion) => {
-  const metadataRpc = await getMetadataRpc(chainConnector, chainId);
-  const chainConnectors = {
-    substrate: chainConnector
-  };
-  const modules = defaultBalanceModules.map(mod => mod({
-    chainConnectors,
-    chaindataProvider
-  })).filter(mod => mod.type.startsWith("substrate-"));
-  return Promise.all(modules.map(async mod => {
-    const source = mod.type;
-    const chainMeta = await mod.fetchSubstrateChainMeta(chainId, {}, metadataRpc, {});
-    return {
-      id: deriveMiniMetadataId({
+const fetchMiniMetadatas = async (chainConnector, chaindataProvider, chainId, specVersion, signal) => {
+  const start = performance.now();
+  log.debug("[miniMetadata] fetching minimetadatas for %s", chainId);
+  try {
+    const metadataRpc = await getMetadataRpc(chainConnector, chainId);
+    signal?.throwIfAborted();
+    const chainConnectors = {
+      substrate: chainConnector,
+      evm: {} // wont be used but workarounds error for module creation
+    };
+    const modules = defaultBalanceModules.map(mod => mod({
+      chainConnectors,
+      chaindataProvider
+    })).filter(mod => mod.type.startsWith("substrate-"));
+    return Promise.all(modules.map(async mod => {
+      const source = mod.type;
+      const chainMeta = await mod.fetchSubstrateChainMeta(chainId, {}, metadataRpc);
+      return {
+        id: deriveMiniMetadataId({
+          source,
+          chainId,
+          specVersion,
+          libVersion
+        }),
         source,
         chainId,
         specVersion,
-        libVersion
-      }),
-      source,
-      chainId,
-      specVersion,
-      libVersion,
-      data: chainMeta?.miniMetadata ?? null
-    };
-  }));
+        libVersion,
+        data: chainMeta?.miniMetadata ?? null
+      };
+    }));
+  } finally {
+    log.debug("[miniMetadata] updated miniMetadatas for %s in %sms", chainId, performance.now() - start);
+  }
 };
 
-const getUpdatedMiniMetadatas = async (chainConnector, chaindataProvider, networkId, specVersion) => {
-  const miniMetadatas = await getMiniMetadatas(chainConnector, chaindataProvider, networkId, specVersion);
+const getUpdatedMiniMetadatas = async (chainConnector, chaindataProvider, chainId, specVersion, signal) => {
+  const miniMetadatas = await getMiniMetadatas(chainConnector, chaindataProvider, chainId, specVersion, signal);
+  signal?.throwIfAborted();
   await db.transaction("readwrite", "miniMetadatas", async tx => {
     await tx.miniMetadatas.where({
-      networkId
+      chainId
     }).delete();
     await tx.miniMetadatas.bulkPut(miniMetadatas);
   });
   return miniMetadatas;
 };
 
-const getMiniMetadata = async (chaindataProvider, chainConnector, chainId, source) => {
+const getMiniMetadata = async (chaindataProvider, chainConnector, chainId, source, signal) => {
   const specVersion = await getSpecVersion(chainConnector, chainId);
-
-  // TODO when working a chaindata branch, need a way to pass the libVersion used to derive the miniMetadataId got github
+  signal?.throwIfAborted();
   const miniMetadataId = deriveMiniMetadataId({
     source,
     chainId,
@@ -3066,11 +2827,13 @@ const getMiniMetadata = async (chaindataProvider, chainConnector, chainId, sourc
 
   // lookup local ones
   const [dbMiniMetadata, ghMiniMetadata] = await Promise.all([db.miniMetadatas.get(miniMetadataId), chaindataProvider.miniMetadataById(miniMetadataId)]);
+  signal?.throwIfAborted();
   const miniMetadata = dbMiniMetadata ?? ghMiniMetadata;
   if (miniMetadata) return miniMetadata;
 
   // update from live chain metadata and persist locally
-  const miniMetadatas = await getUpdatedMiniMetadatas(chainConnector, chaindataProvider, chainId, specVersion);
+  const miniMetadatas = await getUpdatedMiniMetadatas(chainConnector, chaindataProvider, chainId, specVersion, signal);
+  signal?.throwIfAborted();
   const found = miniMetadatas.find(m => m.id === miniMetadataId);
   if (!found) {
     log.warn("MiniMetadata not found in updated miniMetadatas", {
@@ -3157,6 +2920,28 @@ const buildStorageCoders = ({
     return [];
   }
 }));
+const buildNetworkStorageCoders = (chainId, miniMetadata, coders) => {
+  if (!miniMetadata.data) return null;
+  const metadata = scale.unifyMetadata(scale.decAnyMetadata(miniMetadata.data));
+  try {
+    const scaleBuilder = scale.getDynamicBuilder(scale.getLookupFn(metadata));
+    const builtCoders = Object.fromEntries(Object.entries(coders).flatMap(([key, moduleMethodOrFn]) => {
+      const [module, method] = typeof moduleMethodOrFn === "function" ? moduleMethodOrFn({
+        chainId
+      }) : moduleMethodOrFn;
+      try {
+        return [[key, scaleBuilder.buildStorage(module, method)]];
+      } catch (cause) {
+        log.trace(`Failed to build SCALE coder for chain ${chainId} (${module}::${method})`, cause);
+        return [];
+      }
+    }));
+    return builtCoders;
+  } catch (cause) {
+    log.error(`Failed to build SCALE coders for chain ${chainId} (${JSON.stringify(coders)})`, cause);
+  }
+  return null;
+};
 
 /**
  * Decodes & unwraps outputs and errors of a given result, contract, and method.
@@ -3357,15 +3142,9 @@ const SubAssetsModule = hydrate => {
   util$1.assert(chainConnector, "This module requires a substrate chain connector");
   return {
     ...DefaultBalanceModule(moduleType$4),
+    // TODO make synchronous at the module definition level ?
     async fetchSubstrateChainMeta(chainId, moduleConfig, metadataRpc) {
-      const isTestnet = (await chaindataProvider$1.chainById(chainId))?.isTestnet || false;
-      if (metadataRpc === undefined) return {
-        isTestnet
-      };
-      if ((moduleConfig?.tokens ?? []).length < 1) return {
-        isTestnet
-      };
-      const metadataVersion = scale.getMetadataVersion(metadataRpc);
+      if (!metadataRpc) return {};
       const metadata = scale.decAnyMetadata(metadataRpc);
       scale.compactMetadata(metadata, [{
         pallet: "Assets",
@@ -3373,9 +3152,7 @@ const SubAssetsModule = hydrate => {
       }]);
       const miniMetadata = scale.encodeMetadata(metadata);
       return {
-        isTestnet,
-        miniMetadata,
-        metadataVersion
+        miniMetadata
       };
     },
     async fetchSubstrateChainTokens(chainId, chainMeta, moduleConfig) {
@@ -3440,44 +3217,32 @@ const SubAssetsModule = hydrate => {
         return acc;
       }, {});
       const controller = new AbortController();
-      await Promise.all(lodash.toPairs(byNetwork).map(async ([networkId, addressesByToken]) => {
-        const queries = await buildNetworkQueries(networkId, chainConnector, chaindataProvider$1, addressesByToken);
-        if (controller.signal.aborted) return;
-        const stateHelper = new RpcStateQueryHelper(chainConnector, queries);
-        const unsubscribe = await stateHelper.subscribe((error, result) => {
-          //  console.log("SubstrateAssetsModule.callback", { error, result })
-          if (error) return callback(error);
-          const balances = result?.filter(b => b !== null) ?? [];
-          if (balances.length > 0) callback(null, new Balances(balances));
-        });
-        controller.signal.addEventListener("abort", () => {
-          log.debug("TMP subscribeBalances aborted, unsubscribing from network", networkId);
-          unsubscribe();
-        });
+      const pUnsubs = Promise.all(lodash.toPairs(byNetwork).map(async ([networkId, addressesByToken]) => {
+        try {
+          const queries = await buildNetworkQueries$1(networkId, chainConnector, chaindataProvider$1, addressesByToken, controller.signal);
+          if (controller.signal.aborted) return () => {};
+          const stateHelper = new RpcStateQueryHelper(chainConnector, queries);
+          return await stateHelper.subscribe((error, result) => {
+            //  console.log("SubstrateAssetsModule.callback", { error, result })
+            if (error) return callback(error);
+            const balances = result?.filter(b => b !== null) ?? [];
+            if (balances.length > 0) callback(null, new Balances(balances));
+          });
+        } catch (err) {
+          if (!controller.signal.aborted) log.error(`Failed to subscribe balances for network ${networkId}`, err);
+          return () => {};
+        }
       }));
-
-      // const networkIds = uniq(uniq(keys(addressesByToken)).map((tokenId) => parseSubAssetTokenId(tokenId).networkId))
-      // const
-
-      //console.log("SubstrateAssetsModule.subscribeBalances 1", { addressesByToken })
-      // const queries = await buildQueries(chaindataProvider, addressesByToken)
-      // //console.log("SubstrateAssetsModule.subscribeBalances 2", { queries, addressesByToken })
-      // const unsubscribe = await new RpcStateQueryHelper(chainConnector, queries).subscribe(
-      //   (error, result) => {
-      //     //  console.log("SubstrateAssetsModule.callback", { error, result })
-      //     if (error) return callback(error)
-      //     const balances = result?.filter((b): b is SubAssetsBalance => b !== null) ?? []
-      //     if (balances.length > 0) callback(null, new Balances(balances))
-      //   },
-      // )
-
       return () => {
         controller.abort();
+        pUnsubs.then(unsubs => {
+          unsubs.forEach(unsubscribe => unsubscribe());
+        });
       };
     },
     async fetchBalances(addressesByToken) {
       util$1.assert(chainConnectors.substrate, "This module requires a substrate chain connector");
-      const queries = await buildQueries$3(chaindataProvider$1, addressesByToken);
+      const queries = await buildQueries$3(chainConnector, chaindataProvider$1, addressesByToken);
       const result = await new RpcStateQueryHelper(chainConnectors.substrate, queries).fetch();
       const balances = result?.filter(b => b !== null) ?? [];
       return new Balances(balances);
@@ -3547,23 +3312,14 @@ const SubAssetsModule = hydrate => {
     }
   };
 };
-async function buildNetworkQueries(networkId, chainConnector, chaindataProvider, addressesByToken) {
-  const miniMetadata = await getMiniMetadata(chaindataProvider, chainConnector, networkId, moduleType$4);
-  const network = await chaindataProvider.chainById(networkId);
+async function buildNetworkQueries$1(networkId, chainConnector, chaindataProvider, addressesByToken, signal) {
+  const miniMetadata = await getMiniMetadata(chaindataProvider, chainConnector, networkId, moduleType$4, signal);
+  // console.log("Fetched miniMetadata for network", networkId, { miniMetadata })
+  const chain = await chaindataProvider.chainById(networkId);
   const tokensById = await chaindataProvider.tokensById();
-  const chainIds = [networkId];
-  const chains = network ? {
-    [networkId]: network
-  } : {};
-  const miniMetadatas = new Map([[miniMetadata.id, miniMetadata]]);
-  const chainStorageCoders = buildStorageCoders({
-    chainIds,
-    chains,
-    miniMetadatas,
-    moduleType: moduleType$4,
-    coders: {
-      storage: ["Assets", "Account"]
-    }
+  signal?.throwIfAborted();
+  const networkStorageCoders = buildNetworkStorageCoders(networkId, miniMetadata, {
+    storage: ["Assets", "Account"]
   });
   return Object.entries(addressesByToken).flatMap(([tokenId, addresses]) => {
     const token = tokensById[tokenId];
@@ -3575,19 +3331,14 @@ async function buildNetworkQueries(networkId, chainConnector, chaindataProvider,
       log.debug(`This module doesn't handle tokens of type ${token.type}`);
       return [];
     }
-    const networkId = token.networkId;
-    if (!networkId) {
-      log.warn(`Token ${tokenId} has no chain`);
-      return [];
-    }
-    const chain = chains[networkId];
+    //
     if (!chain) {
       log.warn(`Chain ${networkId} for token ${tokenId} not found`);
       return [];
     }
     return addresses.flatMap(address => {
-      const scaleCoder = chainStorageCoders.get(networkId)?.storage;
-      const stateKey = tryEncode(scaleCoder, BigInt(token.assetId), address) ?? tryEncode(scaleCoder, token.assetId, address);
+      const scaleCoder = networkStorageCoders?.storage;
+      const stateKey = tryEncode(scaleCoder, BigInt(token.assetId), address) ?? tryEncode(scaleCoder, Number(token.assetId), address);
       if (!stateKey) {
         log.warn(`Invalid assetId / address in ${networkId} storage query ${token.assetId} / ${address}`);
         return [];
@@ -3642,102 +3393,16 @@ async function buildNetworkQueries(networkId, chainConnector, chaindataProvider,
     });
   });
 }
-async function buildQueries$3(chaindataProvider, addressesByToken) {
-  const allChains = await chaindataProvider.chainsById();
-  const tokens = await chaindataProvider.tokensById();
-
-  // const networkIds = Object.keys(addressesByToken)
-
-  // const
-  // const miniMetadatas = await getMiniMetadatas(chainConnector, chaindataProvider, network)
-  const miniMetadatas = new Map((await db.miniMetadatas.toArray()).map(miniMetadata => [miniMetadata.id, miniMetadata]));
-  const uniqueChainIds = getUniqueChainIds(addressesByToken, tokens);
-  const chains = Object.fromEntries(uniqueChainIds.map(chainId => [chainId, allChains[chainId]]));
-  const chainStorageCoders = buildStorageCoders({
-    chainIds: uniqueChainIds,
-    chains,
-    miniMetadatas,
-    moduleType: "substrate-assets",
-    coders: {
-      storage: ["Assets", "Account"]
-    }
-  });
-  return Object.entries(addressesByToken).flatMap(([tokenId, addresses]) => {
-    const token = tokens[tokenId];
-    if (!token) {
-      log.warn(`Token ${tokenId} not found`);
-      return [];
-    }
-    if (token.type !== "substrate-assets") {
-      log.debug(`This module doesn't handle tokens of type ${token.type}`);
-      return [];
-    }
-    const networkId = token.networkId;
-    if (!networkId) {
-      log.warn(`Token ${tokenId} has no chain`);
-      return [];
-    }
-    const chain = chains[networkId];
-    if (!chain) {
-      log.warn(`Chain ${networkId} for token ${tokenId} not found`);
-      return [];
-    }
-    return addresses.flatMap(address => {
-      const scaleCoder = chainStorageCoders.get(networkId)?.storage;
-      const stateKey = tryEncode(scaleCoder, BigInt(token.assetId), address) ?? tryEncode(scaleCoder, token.assetId, address);
-      if (!stateKey) {
-        log.warn(`Invalid assetId / address in ${networkId} storage query ${token.assetId} / ${address}`);
-        return [];
-      }
-      const decodeResult = change => {
-        /** NOTE: This type is only a hint for typescript, the chain can actually return whatever it wants to */
-
-        const decoded = scale.decodeScale(scaleCoder, change, `Failed to decode substrate-assets balance on chain ${networkId}`) ?? {
-          balance: 0n,
-          status: {
-            type: "Liquid"
-          }};
-        const isFrozen = decoded?.status?.type === "Frozen";
-        const amount = (decoded?.balance ?? 0n).toString();
-
-        // due to the following balance calculations, which are made in the `Balance` type:
-        //
-        // total balance        = (free balance) + (reserved balance)
-        // transferable balance = (free balance) - (frozen balance)
-        //
-        // when `isFrozen` is true we need to set **both** the `free` and `frozen` amounts
-        // of this balance to the value we received from the RPC.
-        //
-        // if we only set the `frozen` amount, then the `total` calculation will be incorrect!
-        const free = amount;
-        const frozen = token.isFrozen || isFrozen ? amount : "0";
-
-        // include balance values even if zero, so that newly-zero values overwrite old values
-        const balanceValues = [{
-          type: "free",
-          label: "free",
-          amount: free.toString()
-        }, {
-          type: "locked",
-          label: "frozen",
-          amount: frozen.toString()
-        }];
-        return {
-          source: "substrate-assets",
-          status: "live",
-          address,
-          networkId,
-          tokenId: token.id,
-          values: balanceValues
-        };
-      };
-      return {
-        chainId: networkId,
-        stateKey,
-        decodeResult
-      };
-    });
-  });
+async function buildQueries$3(chainConnector, chaindataProvider$1, addressesByToken, signal) {
+  const byNetwork = lodash.keys(addressesByToken).reduce((acc, tokenId) => {
+    const networkId = chaindataProvider.parseSubAssetTokenId(tokenId).networkId;
+    if (!acc[networkId]) acc[networkId] = {};
+    acc[networkId][tokenId] = addressesByToken[tokenId];
+    return acc;
+  }, {});
+  return (await Promise.all(lodash.toPairs(byNetwork).map(([networkId, addressesByToken]) => {
+    return buildNetworkQueries$1(networkId, chainConnector, chaindataProvider$1, addressesByToken, signal);
+  }))).flat();
 }
 // NOTE: Different chains need different formats for assetId when encoding the stateKey
 // E.g. Polkadot Asset Hub needs it to be a string, Astar needs it to be a bigint
@@ -5413,8 +5078,23 @@ class SubNativeBalanceError extends Error {
   }
 }
 
-const DEFAULT_SYMBOL = "Unit";
-const DEFAULT_DECIMALS = 0;
+const DotNetworkPropertiesSimple = z__default.default.object({
+  tokenDecimals: z__default.default.number().optional().default(0),
+  tokenSymbol: z__default.default.string().optional().default("Unit")
+});
+const DotNetworkPropertiesArray = z__default.default.object({
+  tokenDecimals: z__default.default.array(z__default.default.number()).nonempty(),
+  tokenSymbol: z__default.default.array(z__default.default.string()).nonempty()
+});
+const DotNetworkProperties = z__default.default.union([DotNetworkPropertiesSimple, DotNetworkPropertiesArray]).transform(val => ({
+  tokenDecimals: Array.isArray(val.tokenDecimals) ? val.tokenDecimals[0] : val.tokenDecimals,
+  tokenSymbol: Array.isArray(val.tokenSymbol) ? val.tokenSymbol[0] : val.tokenSymbol
+}));
+const getChainProperties = async (chainConnector, networkId) => {
+  const properties = await chainConnector.send(networkId, "system_properties", [], true);
+  return DotNetworkProperties.parse(properties);
+};
+
 const POLLING_WINDOW_SIZE = 20;
 const MAX_SUBSCRIPTION_SIZE = 40;
 const SubNativeModule = hydrate => {
@@ -5428,29 +5108,187 @@ const SubNativeModule = hydrate => {
   const getModuleTokens = async () => {
     return await chaindataProvider$1.tokensByIdForType(moduleType$2);
   };
-  return {
-    ...DefaultBalanceModule(moduleType$2),
-    async fetchSubstrateChainMeta(chainId, moduleConfig, metadataRpc, systemProperties) {
-      const isTestnet = (await chaindataProvider$1.chainById(chainId))?.isTestnet || false;
-      if (moduleConfig?.disable === true || metadataRpc === undefined) return {
-        isTestnet
-      };
 
-      //
-      // extract system_properties
-      //
+  // subscribeBalances was split by network to prevent all subs to wait for all minimetadatas to be ready.
+  // however the multichain logic in there is so deep in the function below that i had to keep it as-is, and call it by per-network chunks
+  // TODO refactor this be actually network specific
+  const subscribeChainBalances = async (chainId, opts, callback) => {
+    const {
+      addressesByToken,
+      initialBalances
+    } = opts;
+    // full record of balances for this module
+    const subNativeBalances = new rxjs.BehaviorSubject(Object.fromEntries(initialBalances?.map(b => [getBalanceId(b), b]) ?? []));
+    // tokens which have a known positive balance
+    const positiveBalanceTokens = subNativeBalances.pipe(rxjs.map(balances => Array.from(new Set(Object.values(balances).map(b => b.tokenId)))), rxjs.share());
+
+    // tokens that will be subscribed to, simply a slice of the positive balance tokens of size MAX_SUBSCRIPTION_SIZE
+    const subscriptionTokens = positiveBalanceTokens.pipe(rxjs.map(tokens => tokens.sort(sortChains).slice(0, MAX_SUBSCRIPTION_SIZE)));
+
+    // an initialised balance is one where we have received a response for any type of 'subsource',
+    // until then they are initialising. We only need to maintain one map of tokens to addresses for this
+    const initialisingBalances = Object.entries(addressesByToken).reduce((acc, [tokenId, addresses]) => {
+      acc.set(tokenId, new Set(addresses));
+      return acc;
+    }, new Map());
+
+    // after thirty seconds, we need to kill the initialising balances
+    const initBalancesTimeout = setTimeout(() => {
+      initialisingBalances.clear();
+      // manually call the callback to ensure the caller gets the correct status
+      callback(null, {
+        status: "live",
+        data: Object.values(subNativeBalances.getValue())
+      });
+    }, 30_000);
+    const _callbackSub = subNativeBalances.pipe(rxjs.debounceTime(100)).subscribe({
+      next: balances => {
+        callback(null, {
+          status: initialisingBalances.size > 0 ? "initialising" : "live",
+          data: Object.values(balances)
+        });
+      },
+      error: error => callback(error),
+      complete: () => {
+        initialisingBalances.clear();
+        clearTimeout(initBalancesTimeout);
+      }
+    });
+    const unsubDeferred = util.Deferred();
+    // we return this to the caller so that they can let us know when they're no longer interested in this subscription
+    const callerUnsubscribe = () => {
+      subNativeBalances.complete();
+      _callbackSub.unsubscribe();
+      return unsubDeferred.reject(new Error(`Caller unsubscribed`));
+    };
+    // we queue up our work to clean up our subscription when this promise rejects
+    const callerUnsubscribed = unsubDeferred.promise;
+
+    // The update handler is to allow us to merge balances with the same id, and manage initialising and positive balances state for each
+    // balance type and network
+    const handleUpdateForSource = source => (error, result) => {
+      if (result) {
+        const currentBalances = subNativeBalances.getValue();
+
+        // first merge any balances with the same id within the result
+        const accumulatedUpdates = result.filter(b => b.values.length > 0).reduce((acc, b) => {
+          const bId = getBalanceId(b);
+          acc[bId] = mergeBalances(acc[bId], b, source, false);
+          return acc;
+        }, {});
+
+        // then merge these with the current balances
+        const mergedBalances = {};
+        Object.entries(accumulatedUpdates).forEach(([bId, b]) => {
+          // merge the values from the new balance into the existing balance, if there is one
+          mergedBalances[bId] = mergeBalances(currentBalances[bId], b, source, true);
+
+          // update initialisingBalances to remove balances which have been updated
+          const intialisingForToken = initialisingBalances.get(b.tokenId);
+          if (intialisingForToken) {
+            intialisingForToken.delete(b.address);
+            if (intialisingForToken.size === 0) initialisingBalances.delete(b.tokenId);else initialisingBalances.set(b.tokenId, intialisingForToken);
+          }
+        });
+        subNativeBalances.next({
+          ...currentBalances,
+          ...mergedBalances
+        });
+      }
+      if (error) {
+        if (error instanceof SubNativeBalanceError) {
+          // this type of error doesn't need to be handled by the caller
+          initialisingBalances.delete(error.tokenId);
+        } else return callback(error);
+      }
+    };
 
-      const {
-        tokenSymbol,
-        tokenDecimals
-      } = systemProperties ?? {};
-      const symbol = (Array.isArray(tokenSymbol) ? tokenSymbol[0] : tokenSymbol) ?? DEFAULT_SYMBOL;
-      const decimals = (Array.isArray(tokenDecimals) ? tokenDecimals[0] : tokenDecimals) ?? DEFAULT_DECIMALS;
+    // subscribe to addresses and tokens for which we have a known positive balance
+    const positiveSub = subscriptionTokens.pipe(rxjs.debounceTime(1000), rxjs.takeUntil(callerUnsubscribed), rxjs.map(tokenIds => tokenIds.reduce((acc, tokenId) => {
+      acc[tokenId] = addressesByToken[tokenId];
+      return acc;
+    }, {})), rxjs.distinctUntilChanged(isEqual__default.default), rxjs.switchMap(newAddressesByToken => {
+      return rxjs.from(queryCache.getQueries(newAddressesByToken)).pipe(rxjs.switchMap(baseQueries => {
+        return new rxjs.Observable(subscriber => {
+          if (!chainConnectors.substrate) return;
+          const unsubSubtensorStaking = subscribeSubtensorStaking(chaindataProvider$1, chainConnectors.substrate, newAddressesByToken, handleUpdateForSource("subtensor-staking"));
+          const unsubNompoolStaking = subscribeNompoolStaking(chaindataProvider$1, chainConnectors.substrate, newAddressesByToken, handleUpdateForSource("nompools-staking"));
+          const unsubCrowdloans = subscribeCrowdloans(chaindataProvider$1, chainConnectors.substrate, newAddressesByToken, handleUpdateForSource("crowdloan"));
+          const unsubBase = subscribeBase(baseQueries, chainConnectors.substrate, handleUpdateForSource("base"));
+          subscriber.add(async () => (await unsubSubtensorStaking)());
+          subscriber.add(async () => (await unsubNompoolStaking)());
+          subscriber.add(async () => (await unsubCrowdloans)());
+          subscriber.add(async () => (await unsubBase)());
+        });
+      }));
+    })).subscribe();
+
+    // for chains where we don't have a known positive balance, poll rather than subscribe
+    const poll = async (addressesByToken = {}) => {
+      const handleUpdate = handleUpdateForSource("base");
+      try {
+        const balances = await fetchBalances(addressesByToken);
+        handleUpdate(null, Object.values(balances.toJSON()));
+      } catch (error) {
+        if (error instanceof chainConnector.ChainConnectionError) {
+          // coerce ChainConnection errors into SubNativeBalance errors
+          const errorChainId = error.chainId;
+          Object.entries(await getModuleTokens()).filter(([, token]) => token.networkId === errorChainId).forEach(([tokenId]) => {
+            const wrappedError = new SubNativeBalanceError(tokenId, error.message);
+            handleUpdate(wrappedError);
+          });
+        } else {
+          log.error("unknown substrate native balance error", error);
+          handleUpdate(error);
+        }
+      }
+    };
+    // do one poll to get things started
+    const currentBalances = subNativeBalances.getValue();
+    const currentTokens = new Set(Object.values(currentBalances).map(b => b.tokenId));
+    const nonCurrentTokens = Object.keys(addressesByToken).filter(tokenId => !currentTokens.has(tokenId)).sort(sortChains);
+
+    // break nonCurrentTokens into chunks of POLLING_WINDOW_SIZE
+    await PromisePool__default.default.withConcurrency(POLLING_WINDOW_SIZE).for(nonCurrentTokens).process(async nonCurrentTokenId => await poll({
+      [nonCurrentTokenId]: addressesByToken[nonCurrentTokenId]
+    }));
+
+    // now poll every 30s on chains which are not subscriptionTokens
+    // we chunk this observable into batches of positive token ids, to prevent eating all the websocket connections
+    const pollingSub = rxjs.interval(30_000) // emit values every 30 seconds
+    .pipe(rxjs.takeUntil(callerUnsubscribed), rxjs.withLatestFrom(subscriptionTokens),
+    // Combine latest value from subscriptionTokens with each interval tick
+    rxjs.map(([, subscribedTokenIds]) =>
+    // Filter out tokens that are not subscribed
+    Object.keys(addressesByToken).filter(tokenId => !subscribedTokenIds.includes(tokenId))), rxjs.exhaustMap(tokenIds => rxjs.from(util$1.arrayChunk(tokenIds, POLLING_WINDOW_SIZE)).pipe(rxjs.concatMap(async tokenChunk => {
+      // tokenChunk is a chunk of tokenIds with size POLLING_WINDOW_SIZE
+      const pollingTokenAddresses = Object.fromEntries(tokenChunk.map(tokenId => [tokenId, addressesByToken[tokenId]]));
+      await poll(pollingTokenAddresses);
+      return true;
+    })))).subscribe();
+    return () => {
+      callerUnsubscribe();
+      positiveSub.unsubscribe();
+      pollingSub.unsubscribe();
+    };
+  };
+  const fetchBalances = async addressesByToken => {
+    util$1.assert(chainConnectors.substrate, "This module requires a substrate chain connector");
+    const queries = await queryCache.getQueries(addressesByToken);
+    util$1.assert(chainConnectors.substrate, "This module requires a substrate chain connector");
+    const result = await new RpcStateQueryHelper(chainConnectors.substrate, queries).fetch();
+    return new Balances(result ?? []);
+  };
+  return {
+    ...DefaultBalanceModule(moduleType$2),
+    async fetchSubstrateChainMeta(chainId, moduleConfig, metadataRpc) {
+      if (!metadataRpc) return {};
 
       //
       // process metadata into SCALE encoders/decoders
       //
       const metadataVersion = scale.getMetadataVersion(metadataRpc);
+      if (metadataVersion < 14) return {};
       const metadata = scale.decAnyMetadata(metadataRpc);
       const unifiedMetadata = scale.unifyMetadata(metadata);
 
@@ -5517,16 +5355,15 @@ const SubNativeModule = hydrate => {
       }) => name === "Freezes"));
       const useLegacyTransferableCalculation = !hasFreezesItem;
       const chainMeta = {
-        isTestnet,
+        // isTestnet,
         useLegacyTransferableCalculation,
-        symbol,
-        decimals,
+        // symbol,
+        // decimals,
         existentialDeposit,
         nominationPoolsPalletId,
         crowdloanPalletId,
         hasSubtensorPallet,
-        miniMetadata,
-        metadataVersion
+        miniMetadata
       };
       if (!useLegacyTransferableCalculation) delete chainMeta.useLegacyTransferableCalculation;
       if (!hasSubtensorPallet) delete chainMeta.hasSubtensorPallet;
@@ -5535,9 +5372,10 @@ const SubNativeModule = hydrate => {
     async fetchSubstrateChainTokens(chainId, chainMeta, moduleConfig) {
       if (moduleConfig?.disable === true) return {};
       const {
-        isTestnet,
-        symbol,
-        decimals,
+        tokenSymbol: symbol,
+        tokenDecimals: decimals
+      } = await getChainProperties(chainConnector$1, chainId);
+      const {
         existentialDeposit
       } = chainMeta;
       const id = chaindataProvider.subNativeTokenId(chainId);
@@ -5545,11 +5383,10 @@ const SubNativeModule = hydrate => {
         id,
         type: "substrate-native",
         platform: "polkadot",
-        isTestnet,
         isDefault: moduleConfig?.isDefault ?? true,
-        symbol: symbol ?? DEFAULT_SYMBOL,
-        name: moduleConfig?.name ?? symbol ?? DEFAULT_SYMBOL,
-        decimals: decimals ?? DEFAULT_DECIMALS,
+        symbol: symbol,
+        name: moduleConfig?.name ?? symbol,
+        decimals: decimals,
         logo: moduleConfig?.logo,
         existentialDeposit: existentialDeposit ?? "0",
         networkId: chainId
@@ -5566,169 +5403,43 @@ const SubNativeModule = hydrate => {
       initialBalances
     }, callback) {
       util$1.assert(chainConnectors.substrate, "This module requires a substrate chain connector");
-
-      // full record of balances for this module
-      const subNativeBalances = new rxjs.BehaviorSubject(Object.fromEntries(initialBalances?.map(b => [getBalanceId(b), b]) ?? []));
-      // tokens which have a known positive balance
-      const positiveBalanceTokens = subNativeBalances.pipe(rxjs.map(balances => Array.from(new Set(Object.values(balances).map(b => b.tokenId)))), rxjs.share());
-
-      // tokens that will be subscribed to, simply a slice of the positive balance tokens of size MAX_SUBSCRIPTION_SIZE
-      const subscriptionTokens = positiveBalanceTokens.pipe(rxjs.map(tokens => tokens.sort(sortChains).slice(0, MAX_SUBSCRIPTION_SIZE)));
-
-      // an initialised balance is one where we have received a response for any type of 'subsource',
-      // until then they are initialising. We only need to maintain one map of tokens to addresses for this
-      const initialisingBalances = Object.entries(addressesByToken).reduce((acc, [tokenId, addresses]) => {
-        acc.set(tokenId, new Set(addresses));
+      const addressesByTokenByNetwork = lodash.keys(addressesByToken).reduce((acc, tokenId) => {
+        const networkId = chaindataProvider.parseSubNativeTokenId(tokenId).networkId;
+        if (!acc[networkId]) acc[networkId] = {};
+        acc[networkId][tokenId] = addressesByToken[tokenId];
         return acc;
-      }, new Map());
-
-      // after thirty seconds, we need to kill the initialising balances
-      const initBalancesTimeout = setTimeout(() => {
-        initialisingBalances.clear();
-        // manually call the callback to ensure the caller gets the correct status
-        callback(null, {
-          status: "live",
-          data: Object.values(subNativeBalances.getValue())
-        });
-      }, 30_000);
-      const _callbackSub = subNativeBalances.pipe(rxjs.debounceTime(100)).subscribe({
-        next: balances => {
-          callback(null, {
-            status: initialisingBalances.size > 0 ? "initialising" : "live",
-            data: Object.values(balances)
-          });
-        },
-        error: error => callback(error),
-        complete: () => {
-          initialisingBalances.clear();
-          clearTimeout(initBalancesTimeout);
-        }
-      });
-      const unsubDeferred = util.Deferred();
-      // we return this to the caller so that they can let us know when they're no longer interested in this subscription
-      const callerUnsubscribe = () => {
-        subNativeBalances.complete();
-        _callbackSub.unsubscribe();
-        return unsubDeferred.reject(new Error(`Caller unsubscribed`));
-      };
-      // we queue up our work to clean up our subscription when this promise rejects
-      const callerUnsubscribed = unsubDeferred.promise;
-
-      // The update handler is to allow us to merge balances with the same id, and manage initialising and positive balances state for each
-      // balance type and network
-      const handleUpdateForSource = source => (error, result) => {
-        if (result) {
-          const currentBalances = subNativeBalances.getValue();
-
-          // first merge any balances with the same id within the result
-          const accumulatedUpdates = result.filter(b => b.values.length > 0).reduce((acc, b) => {
-            const bId = getBalanceId(b);
-            acc[bId] = mergeBalances(acc[bId], b, source, false);
-            return acc;
-          }, {});
-
-          // then merge these with the current balances
-          const mergedBalances = {};
-          Object.entries(accumulatedUpdates).forEach(([bId, b]) => {
-            // merge the values from the new balance into the existing balance, if there is one
-            mergedBalances[bId] = mergeBalances(currentBalances[bId], b, source, true);
-
-            // update initialisingBalances to remove balances which have been updated
-            const intialisingForToken = initialisingBalances.get(b.tokenId);
-            if (intialisingForToken) {
-              intialisingForToken.delete(b.address);
-              if (intialisingForToken.size === 0) initialisingBalances.delete(b.tokenId);else initialisingBalances.set(b.tokenId, intialisingForToken);
-            }
-          });
-          subNativeBalances.next({
-            ...currentBalances,
-            ...mergedBalances
-          });
-        }
-        if (error) {
-          if (error instanceof SubNativeBalanceError) {
-            // this type of error doesn't need to be handled by the caller
-            initialisingBalances.delete(error.tokenId);
-          } else return callback(error);
-        }
+      }, {});
+      const initialBalancesByNetwork = lodash.groupBy(initialBalances ?? [], "networkId");
+      const {
+        abort,
+        signal
+      } = new AbortController();
+      const safeCallback = (error, result) => {
+        if (signal.aborted) return;
+        // typescript isnt happy with fowarding parameters as is
+        return error ? callback(error, undefined) : callback(error, result);
       };
-
-      // subscribe to addresses and tokens for which we have a known positive balance
-      const positiveSub = subscriptionTokens.pipe(rxjs.debounceTime(1000), rxjs.takeUntil(callerUnsubscribed), rxjs.map(tokenIds => tokenIds.reduce((acc, tokenId) => {
-        acc[tokenId] = addressesByToken[tokenId];
-        return acc;
-      }, {})), rxjs.distinctUntilChanged(isEqual__default.default), rxjs.switchMap(newAddressesByToken => {
-        return rxjs.from(queryCache.getQueries(newAddressesByToken)).pipe(rxjs.switchMap(baseQueries => {
-          return new rxjs.Observable(subscriber => {
-            if (!chainConnectors.substrate) return;
-            const unsubSubtensorStaking = subscribeSubtensorStaking(chaindataProvider$1, chainConnectors.substrate, newAddressesByToken, handleUpdateForSource("subtensor-staking"));
-            const unsubNompoolStaking = subscribeNompoolStaking(chaindataProvider$1, chainConnectors.substrate, newAddressesByToken, handleUpdateForSource("nompools-staking"));
-            const unsubCrowdloans = subscribeCrowdloans(chaindataProvider$1, chainConnectors.substrate, newAddressesByToken, handleUpdateForSource("crowdloan"));
-            const unsubBase = subscribeBase(baseQueries, chainConnectors.substrate, handleUpdateForSource("base"));
-            subscriber.add(async () => (await unsubSubtensorStaking)());
-            subscriber.add(async () => (await unsubNompoolStaking)());
-            subscriber.add(async () => (await unsubCrowdloans)());
-            subscriber.add(async () => (await unsubBase)());
-          });
-        }));
-      })).subscribe();
-
-      // for chains where we don't have a known positive balance, poll rather than subscribe
-      const poll = async (addressesByToken = {}) => {
-        const handleUpdate = handleUpdateForSource("base");
+      const unsubsribeFns = Promise.all(lodash.keys(addressesByTokenByNetwork).map(async networkId => {
         try {
-          const balances = await this.fetchBalances(addressesByToken);
-          handleUpdate(null, Object.values(balances.toJSON()));
-        } catch (error) {
-          if (error instanceof chainConnector.ChainConnectionError) {
-            // coerce ChainConnection errors into SubNativeBalance errors
-            const errorChainId = error.chainId;
-            Object.entries(await getModuleTokens()).filter(([, token]) => token.networkId === errorChainId).forEach(([tokenId]) => {
-              const wrappedError = new SubNativeBalanceError(tokenId, error.message);
-              handleUpdate(wrappedError);
-            });
-          } else {
-            log.error("unknown substrate native balance error", error);
-            handleUpdate(error);
-          }
+          // this is what we want to be done separately for each network
+          // this will update the DB so minimetadata will be available when it's used, veeeeery far down the tree of subscribeChainBalances
+          await getMiniMetadata(chaindataProvider$1, chainConnector$1, networkId, moduleType$2, signal);
+        } catch (err) {
+          if (!signal.aborted) log.warn("Failed to get native token miniMetadata for network", networkId, err);
+          return () => {};
         }
-      };
-      // do one poll to get things started
-      const currentBalances = subNativeBalances.getValue();
-      const currentTokens = new Set(Object.values(currentBalances).map(b => b.tokenId));
-      const nonCurrentTokens = Object.keys(addressesByToken).filter(tokenId => !currentTokens.has(tokenId)).sort(sortChains);
-
-      // break nonCurrentTokens into chunks of POLLING_WINDOW_SIZE
-      await PromisePool__default.default.withConcurrency(POLLING_WINDOW_SIZE).for(nonCurrentTokens).process(async nonCurrentTokenId => await poll({
-        [nonCurrentTokenId]: addressesByToken[nonCurrentTokenId]
+        if (signal.aborted) return () => {};
+        return subscribeChainBalances(networkId, {
+          addressesByToken: addressesByTokenByNetwork[networkId] ?? {},
+          initialBalances: initialBalancesByNetwork[networkId] ?? []
+        }, safeCallback);
       }));
-
-      // now poll every 30s on chains which are not subscriptionTokens
-      // we chunk this observable into batches of positive token ids, to prevent eating all the websocket connections
-      const pollingSub = rxjs.interval(30_000) // emit values every 30 seconds
-      .pipe(rxjs.takeUntil(callerUnsubscribed), rxjs.withLatestFrom(subscriptionTokens),
-      // Combine latest value from subscriptionTokens with each interval tick
-      rxjs.map(([, subscribedTokenIds]) =>
-      // Filter out tokens that are not subscribed
-      Object.keys(addressesByToken).filter(tokenId => !subscribedTokenIds.includes(tokenId))), rxjs.exhaustMap(tokenIds => rxjs.from(util$1.arrayChunk(tokenIds, POLLING_WINDOW_SIZE)).pipe(rxjs.concatMap(async tokenChunk => {
-        // tokenChunk is a chunk of tokenIds with size POLLING_WINDOW_SIZE
-        const pollingTokenAddresses = Object.fromEntries(tokenChunk.map(tokenId => [tokenId, addressesByToken[tokenId]]));
-        await poll(pollingTokenAddresses);
-        return true;
-      })))).subscribe();
       return () => {
-        callerUnsubscribe();
-        positiveSub.unsubscribe();
-        pollingSub.unsubscribe();
+        abort();
+        unsubsribeFns.then(fns => fns.forEach(unsubscribe => unsubscribe()));
       };
     },
-    async fetchBalances(addressesByToken) {
-      util$1.assert(chainConnectors.substrate, "This module requires a substrate chain connector");
-      const queries = await queryCache.getQueries(addressesByToken);
-      util$1.assert(chainConnectors.substrate, "This module requires a substrate chain connector");
-      const result = await new RpcStateQueryHelper(chainConnectors.substrate, queries).fetch();
-      return new Balances(result ?? []);
-    },
+    fetchBalances,
     async transferToken({
       tokenId,
       from,
@@ -7200,7 +6911,6 @@ const SubTokensModule = hydrate => {
     ...DefaultBalanceModule(moduleType),
     async fetchSubstrateChainMeta(chainId, moduleConfig, metadataRpc) {
       if (metadataRpc === undefined) return {};
-      if ((moduleConfig?.tokens ?? []).length < 1) return {};
       const metadata = scale.decAnyMetadata(metadataRpc);
       const palletId = moduleConfig?.palletId ?? defaultPalletId;
       scale.compactMetadata(metadata, [{
@@ -7252,17 +6962,39 @@ const SubTokensModule = hydrate => {
     async subscribeBalances({
       addressesByToken
     }, callback) {
-      const queries = await buildQueries(chaindataProvider$1, addressesByToken);
-      const unsubscribe = await new RpcStateQueryHelper(chainConnector, queries).subscribe((error, result) => {
-        if (error) return callback(error);
-        const balances = result?.filter(b => b !== null) ?? [];
-        if (balances.length > 0) callback(null, new Balances(balances));
-      });
-      return unsubscribe;
+      const byNetwork = lodash.keys(addressesByToken).reduce((acc, tokenId) => {
+        const networkId = chaindataProvider.parseSubTokensTokenId(tokenId).networkId;
+        if (!acc[networkId]) acc[networkId] = {};
+        acc[networkId][tokenId] = addressesByToken[tokenId];
+        return acc;
+      }, {});
+      const controller = new AbortController();
+      const pUnsubs = Promise.all(lodash.toPairs(byNetwork).map(async ([networkId, addressesByToken]) => {
+        try {
+          const queries = await buildNetworkQueries(networkId, chainConnector, chaindataProvider$1, addressesByToken, controller.signal);
+          if (controller.signal.aborted) return () => {};
+          const stateHelper = new RpcStateQueryHelper(chainConnector, queries);
+          return await stateHelper.subscribe((error, result) => {
+            //  console.log("SubstrateAssetsModule.callback", { error, result })
+            if (error) return callback(error);
+            const balances = result?.filter(b => b !== null) ?? [];
+            if (balances.length > 0) callback(null, new Balances(balances));
+          });
+        } catch (err) {
+          if (!controller.signal.aborted) log.error(`Failed to subscribe balances for network ${networkId}`, err);
+          return () => {};
+        }
+      }));
+      return () => {
+        controller.abort();
+        pUnsubs.then(unsubs => {
+          unsubs.forEach(unsubscribe => unsubscribe());
+        });
+      };
     },
     async fetchBalances(addressesByToken) {
       util$1.assert(chainConnectors.substrate, "This module requires a substrate chain connector");
-      const queries = await buildQueries(chaindataProvider$1, addressesByToken);
+      const queries = await buildQueries(chainConnector, chaindataProvider$1, addressesByToken);
       const result = await new RpcStateQueryHelper(chainConnectors.substrate, queries).fetch();
       const balances = result?.filter(b => b !== null) ?? [];
       return new Balances(balances);
@@ -7381,23 +7113,16 @@ const SubTokensModule = hydrate => {
     }
   };
 };
-async function buildQueries(chaindataProvider, addressesByToken) {
-  const allChains = await chaindataProvider.chainsById();
+async function buildNetworkQueries(networkId, chainConnector, chaindataProvider, addressesByToken, signal) {
+  const miniMetadata = await getMiniMetadata(chaindataProvider, chainConnector, networkId, moduleType, signal);
+  const chain = await chaindataProvider.chainById(networkId);
   const tokens = await chaindataProvider.tokensById();
-  const miniMetadatas = new Map((await db.miniMetadatas.toArray()).map(miniMetadata => [miniMetadata.id, miniMetadata]));
-  const tokensPalletByChain = new Map(Object.values(allChains).map(chain => [chain.id, findChainMeta(miniMetadatas, moduleType, chain)[0]?.palletId]));
-  const uniqueChainIds = getUniqueChainIds(addressesByToken, tokens);
-  const chains = Object.fromEntries(uniqueChainIds.map(chainId => [chainId, allChains[chainId]]));
-  const chainStorageCoders = buildStorageCoders({
-    chainIds: uniqueChainIds,
-    chains,
-    miniMetadatas,
-    moduleType: "substrate-tokens",
-    coders: {
-      storage: ({
-        chainId
-      }) => [tokensPalletByChain.get(chainId) ?? defaultPalletId, "Accounts"]
-    }
+  if (!chain) return [];
+  signal?.throwIfAborted();
+  const tokensMetadata = miniMetadata;
+  const palletId = tokensMetadata.palletId ?? defaultPalletId;
+  const networkStorageCoders = buildNetworkStorageCoders(networkId, miniMetadata, {
+    storage: [palletId, "Accounts"]
   });
   return Object.entries(addressesByToken).flatMap(([tokenId, addresses]) => {
     const token = tokens[tokenId];
@@ -7409,18 +7134,8 @@ async function buildQueries(chaindataProvider, addressesByToken) {
       log.debug(`This module doesn't handle tokens of type ${token.type}`);
       return [];
     }
-    const networkId = token.networkId;
-    if (!networkId) {
-      log.warn(`Token ${tokenId} has no chain`);
-      return [];
-    }
-    const chain = chains[networkId];
-    if (!chain) {
-      log.warn(`Chain ${networkId} for token ${tokenId} not found`);
-      return [];
-    }
     return addresses.flatMap(address => {
-      const scaleCoder = chainStorageCoders.get(networkId)?.storage;
+      const scaleCoder = networkStorageCoders?.storage;
       const onChainId = (() => {
         try {
           return scale.papiParse(token.onChainId);
@@ -7471,35 +7186,20 @@ async function buildQueries(chaindataProvider, addressesByToken) {
     });
   });
 }
+async function buildQueries(chainConnector, chaindataProvider$1, addressesByToken, signal) {
+  const byNetwork = lodash.keys(addressesByToken).reduce((acc, tokenId) => {
+    const networkId = chaindataProvider.parseSubTokensTokenId(tokenId).networkId;
+    if (!acc[networkId]) acc[networkId] = {};
+    acc[networkId][tokenId] = addressesByToken[tokenId];
+    return acc;
+  }, {});
+  return (await Promise.all(lodash.toPairs(byNetwork).map(([networkId, addressesByToken]) => {
+    return buildNetworkQueries(networkId, chainConnector, chaindataProvider$1, addressesByToken, signal);
+  }))).flat();
+}
 
 const defaultBalanceModules = [EvmErc20Module, EvmNativeModule, EvmUniswapV2Module, SubAssetsModule, SubForeignAssetsModule, SubNativeModule, SubPsp22Module, SubTokensModule];
 
-/** Pulls the latest chaindata from https://github.com/TalismanSociety/chaindata */
-const hydrateChaindataAndMiniMetadata = async (chaindataProvider, miniMetadataUpdater) => {
-  // need chains to be provisioned first, or substrate balances won't fetch on first subscription
-  await chaindataProvider.hydrate();
-  await Promise.all([miniMetadataUpdater.hydrateFromChaindata(), miniMetadataUpdater.hydrateCustomChains()]);
-  const chains = await chaindataProvider.chains();
-  const {
-    statusesByChain
-  } = await miniMetadataUpdater.statuses(chains);
-  const goodChains = [...statusesByChain.entries()].flatMap(([chainId, status]) => status === "good" ? chainId : []);
-  await chaindataProvider.hydrateSubstrateTokens(goodChains);
-};
-
-/** Builds any missing miniMetadatas (e.g. for the user's custom substrate chains) */
-const updateCustomMiniMetadata = async (chaindataProvider, miniMetadataUpdater) => {
-  const chainIds = await chaindataProvider.chainIds();
-  await miniMetadataUpdater.update(chainIds);
-};
-
-/** Fetches any missing Evm Tokens */
-const updateEvmTokens = async (chaindataProvider, evmTokenFetcher) => {
-  await chaindataProvider.hydrate();
-  const evmNetworkIds = await chaindataProvider.evmNetworkIds();
-  await evmTokenFetcher.update(evmNetworkIds);
-};
-
 exports.Balance = Balance;
 exports.BalanceFormatter = BalanceFormatter;
 exports.BalanceValueGetter = BalanceValueGetter;
@@ -7511,7 +7211,6 @@ exports.EvmNativeModule = EvmNativeModule;
 exports.EvmTokenFetcher = EvmTokenFetcher;
 exports.EvmUniswapV2Module = EvmUniswapV2Module;
 exports.FiatSumBalancesFormatter = FiatSumBalancesFormatter;
-exports.MiniMetadataUpdater = MiniMetadataUpdater;
 exports.ONE_ALPHA_TOKEN = ONE_ALPHA_TOKEN;
 exports.PlanckSumBalancesFormatter = PlanckSumBalancesFormatter;
 exports.RpcStateQueryHelper = RpcStateQueryHelper;
@@ -7527,6 +7226,7 @@ exports.SumBalancesFormatter = SumBalancesFormatter;
 exports.TalismanBalancesDatabase = TalismanBalancesDatabase;
 exports.abiMulticall = abiMulticall;
 exports.balances = balances;
+exports.buildNetworkStorageCoders = buildNetworkStorageCoders;
 exports.buildStorageCoders = buildStorageCoders;
 exports.calculateAlphaPrice = calculateAlphaPrice;
 exports.calculateTaoAmountFromAlpha = calculateTaoAmountFromAlpha;
@@ -7550,9 +7250,6 @@ exports.getBalanceId = getBalanceId;
 exports.getLockTitle = getLockTitle;
 exports.getUniqueChainIds = getUniqueChainIds;
 exports.getValueId = getValueId;
-exports.hydrateChaindataAndMiniMetadata = hydrateChaindataAndMiniMetadata;
 exports.includeInTotalExtraAmount = includeInTotalExtraAmount;
 exports.makeContractCaller = makeContractCaller;
 exports.uniswapV2PairAbi = uniswapV2PairAbi;
-exports.updateCustomMiniMetadata = updateCustomMiniMetadata;
-exports.updateEvmTokens = updateEvmTokens;