@exodus/market-history 10.2.3 → 10.3.0

package/CHANGELOG.md

@@ -3,6 +3,18 @@
 All notable changes to this project will be documented in this file.
 See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
 
+## [10.3.0](https://github.com/ExodusMovement/exodus-hydra/compare/@exodus/market-history@10.2.4...@exodus/market-history@10.3.0) (2025-07-01)
+
+### Features
+
+- feat: merge price-api with market-history (#12962)
+
+## [10.2.4](https://github.com/ExodusMovement/exodus-hydra/compare/@exodus/market-history@10.2.3...@exodus/market-history@10.2.4) (2025-06-17)
+
+### Performance
+
+- perf(market-history): Optimize price transformation logic (#12932)
+
 ## [10.2.3](https://github.com/ExodusMovement/exodus-hydra/compare/@exodus/market-history@10.2.2...@exodus/market-history@10.2.3) (2025-06-16)
 
 ### Performance

package/module/constants.js

@@ -0,0 +1,10 @@
+const HOURLY_LIMIT = 24 * 7
+const DAILY_LIMIT = 3000
+const MINUTE_LIMIT = 60 * 2 // we have only 2 hours of minute level data cached
+
+export const LIMIT = {
+  minute: MINUTE_LIMIT,
+  hour: HOURLY_LIMIT,
+  day: DAILY_LIMIT,
+}
+export const START_TIME = new Date('2015-12-09').getTime()

package/module/crop-history.js

@@ -0,0 +1,13 @@
+import ms from 'ms'
+import lastTimestampFromPricesMap from './last-timestamp-from-prices-map.js'
+
+const hourMs = ms('1h')
+
+const cropHistory = ({ history, hourlyLimit }) => {
+  const lastCachedTime = lastTimestampFromPricesMap(history)
+  if (!lastCachedTime) return history
+  const limitedTime = lastCachedTime - hourMs * hourlyLimit
+  return new Map([...history].filter(([key]) => key > limitedTime))
+}
+
+export default cropHistory

package/module/fetch-and-process-tickers.js

@@ -0,0 +1,61 @@
+import processApiResponse from './process-api-response.js'
+
+export default async function fetchAndProcessTickers({
+  assetTickersToFetch,
+  api,
+  fiatTicker,
+  granularity,
+  timestamp,
+  ignoreInvalidSymbols,
+  historicalPricesMap,
+  fetchedPricesMap,
+  specificTimestamp,
+  requestLimit,
+}) {
+  if (assetTickersToFetch.length === 0) {
+    return
+  }
+
+  const maxLimit = Math.max(...assetTickersToFetch.map(({ limit }) => limit))
+
+  const requests = [
+    assetTickersToFetch.filter(({ limit }) => limit !== maxLimit),
+    assetTickersToFetch.filter(({ limit }) => limit === maxLimit),
+  ]
+
+  await Promise.all(
+    requests
+      .filter((tickers) => tickers.length > 0)
+      .map(async (currentTickersToFetch) => {
+        const limit = Math.max(...currentTickersToFetch.map(({ limit }) => limit))
+        const requestParams = Object.create(null)
+        requestParams.assets = currentTickersToFetch.map((item) => item.tickerSymbol)
+        requestParams.fiatArray = [fiatTicker]
+        requestParams.granularity = granularity
+        requestParams.limit = limit
+        requestParams.timestamp = timestamp
+        requestParams.ignoreInvalidSymbols = ignoreInvalidSymbols
+
+        const fetchedPrices = await api(requestParams)
+
+        Object.keys(fetchedPrices).forEach((assetTicker) => {
+          const newHistory = processApiResponse({
+            assetTicker,
+            fiatTicker,
+            fetchedPrices,
+            assetTickersToFetch,
+            ignoreInvalidSymbols,
+            historicalPricesMap,
+            granularity,
+            specificTimestamp,
+            requestLimit,
+          })
+
+          if (newHistory) {
+            fetchedPricesMap.set(assetTicker, newHistory)
+            historicalPricesMap.set(assetTicker, newHistory)
+          }
+        })
+      })
+  )
+}

package/module/fetch-historical-prices.js

@@ -0,0 +1,88 @@
+import dayjs from '@exodus/dayjs'
+import prepareFetchInstructions from './prepare-fetch-instructions.js'
+import fetchAndProcessTickers from './fetch-and-process-tickers.js'
+
+const runtimeCacheDefault = new Map()
+const getRuntimeCacheKeyDefault = ({ fiatTicker, assetTicker, granularity }) =>
+  `${assetTicker}_${fiatTicker}_${granularity}`
+
+export default async function fetchHistoricalPrices({
+  api,
+  assetTickers,
+  fiatTicker,
+  granularity = 'day',
+  getCacheFromStorage,
+  requestLimit,
+  timestamp,
+  getCurrentTime = () => Date.now(),
+  ignoreInvalidSymbols = false,
+  ignoreCache = false,
+  runtimeCache = runtimeCacheDefault,
+  getRuntimeCacheKey = getRuntimeCacheKeyDefault,
+}) {
+  const specificTimestamp = timestamp ? dayjs(timestamp).utc().startOf(granularity).valueOf() : null
+
+  const requestTimestamp =
+    specificTimestamp ||
+    dayjs.utc(getCurrentTime()).subtract(1, granularity).startOf(granularity).valueOf()
+
+  const historicalPricesMap = new Map()
+  const fetchedPricesMap = new Map()
+
+  const tickerSymbols = Array.isArray(assetTickers) ? assetTickers : [assetTickers]
+
+  const instructions = await Promise.all(
+    tickerSymbols.map((tickerSymbol) =>
+      prepareFetchInstructions({
+        tickerSymbol,
+        fiatTicker,
+        granularity,
+        getCacheFromStorage,
+        requestLimit,
+        specificTimestamp,
+        ignoreCache,
+        runtimeCache,
+        getRuntimeCacheKey,
+        requestTimestamp,
+      })
+    )
+  )
+
+  const assetTickersToFetch = []
+  instructions.forEach(({ history, limit, lastCachedItem, tickerSymbol }) => {
+    historicalPricesMap.set(tickerSymbol, history)
+    if (limit) {
+      assetTickersToFetch.push({ tickerSymbol, limit, lastCachedItem })
+    }
+  })
+
+  await fetchAndProcessTickers({
+    assetTickersToFetch,
+    api,
+    fiatTicker,
+    granularity,
+    timestamp,
+    ignoreInvalidSymbols,
+    historicalPricesMap,
+    fetchedPricesMap,
+    specificTimestamp,
+    requestLimit,
+  })
+
+  for (const tickerSymbol of tickerSymbols) {
+    const runtimeCacheKey = getRuntimeCacheKey({
+      fiatTicker,
+      assetTicker: tickerSymbol,
+      granularity,
+    })
+    const history = historicalPricesMap.get(tickerSymbol)
+    if (history && history.size > 0) {
+      runtimeCache.set(runtimeCacheKey, [...history])
+    }
+  }
+
+  return {
+    historicalPricesMap,
+    fetchedPricesMap,
+  }
+}

package/module/find-invalid-price.js

@@ -0,0 +1,20 @@
+const isValid = (value) => value !== 0 && Number.isFinite(value)
+
+const findInvalidPrice = (data, lastCachedItem) =>
+  data.find((item, index) => {
+    if (!Number.isFinite(item.close) || !Number.isFinite(item.open)) return true
+
+    if (lastCachedItem && isValid(lastCachedItem.close) && index === 0) {
+      // check first item has value. It must, because we have cache for previous period.
+      return !isValid(item.close)
+    }
+
+    if (index > 0) {
+      const prevItem = data[index - 1]
+      // check if price is zero but previous wasn't
+      if (!isValid(item.close) && isValid(prevItem.close)) return true
+      if (!isValid(item.open) && isValid(prevItem.open)) return true
+    }
+  })
+
+export default findInvalidPrice

package/module/get-limit-to-fetch.js

@@ -0,0 +1,42 @@
+import { timeToDays, timeToHours, timeToMinutes } from './utils.js'
+import { LIMIT, START_TIME } from './constants.js'
+
+import lastTimestampFromPricesMap from './last-timestamp-from-prices-map.js'
+
+/**
+ * @typedef {import('./types.d.ts').GranularityType} GranularityType
+ * @typedef {import('./types.d.ts').HistoryType} HistoryType
+ */
+
+/**
+ * @param {object} options
+ * @param {GranularityType} options.granularity
+ * @param {HistoryType} options.history
+ * @param {number} options.requestTimestamp
+ * @param {number} [options.requestLimit]
+ * @param {number} [options.specificTimestamp]
+ * @returns {number}
+ */
+const getLimit = ({ granularity, history, requestLimit, requestTimestamp, specificTimestamp }) => {
+  if (specificTimestamp) return 1
+  // lastCachedTime is always UTC start of hour or day
+  const lastCachedTime = lastTimestampFromPricesMap(history)
+
+  const limitFromOptions = requestLimit || LIMIT[granularity]
+
+  if (granularity === 'minute') {
+    // don't fetch only if it's still same UTC minute
+    if (!lastCachedTime) return limitFromOptions
+    return Math.ceil(Math.min(limitFromOptions, timeToMinutes(requestTimestamp - lastCachedTime)))
+  }
+
+  if (granularity === 'hour') {
+    // don't fetch only if it's still same UTC hour
+    if (!lastCachedTime) return limitFromOptions
+    return Math.min(limitFromOptions, timeToHours(requestTimestamp - lastCachedTime))
+  }
+
+  return Math.min(limitFromOptions, timeToDays(requestTimestamp - (lastCachedTime || START_TIME)))
+}
+
+export default getLimit

package/module/index.js

@@ -1,25 +1,16 @@
-import { mapValues } from '@exodus/basic-utils'
 import dayjs from '@exodus/dayjs'
-import { fetchHistoricalPrices } from '@exodus/price-api'
 import delay from 'delay'
 import makeConcurrent, { byArguments as makeConcurrentByArguments } from 'make-concurrent'
 import ms from 'ms'
 import { createInMemoryAtom, difference } from '@exodus/atoms'
 import { getAssetFromTicker, parseGranularity } from '../utils.js'
+import fetchHistoricalPrices from './fetch-historical-prices.js'
 
 const CLEAR_MARKET_HISTORY_CACHE_KEY = 'clear-market-history-cache'
 const CLEAR_MARKET_HISTORY_CACHE_FROM_REMOTE_CONFIG_KEY =
   'clear-market-history-cache-from-remote-config'
 const MARKET_HISTORY_REFRESH_KEY = 'market-history-cache-refresh'
 
-const transformPriceEntries = (entries = []) => {
-  const closePrices = entries.map((entry) => [entry[0], entry[1].close])
-  return Object.fromEntries(closePrices)
-}
-
-const transformPricesByAssetName = (pricesByAssetName) =>
-  mapValues(pricesByAssetName, (entries) => transformPriceEntries(entries))
-
 // when provided cache version from local or remote config is different from stored on device we clear the cache and fetch new prices
 const _invalidateStorageCache = async ({
   storage,
@@ -58,9 +49,37 @@ const getOldestTimestampFromCache = (cache) =>
   cache.length === 0 ? null : Math.min(...cache.map((item) => item[0]))
 
 const filterMapAssetDataFromServer = (data) =>
-  data
-    .filter((piece) => piece.close !== 0)
-    .map((piece) => [piece.time * 1000, { close: piece.close }])
+  data.filter((item) => item.close !== 0).map((item) => [item.time * 1000, { close: item.close }])
+
+const mapPricesForCache = (pricesMap, assetNames, assets) => {
+  const result = Object.create(null)
+  for (const assetName of assetNames) {
+    const assetPricesMap = pricesMap.get(assets[assetName].ticker)
+    result[assetName] = assetPricesMap ? [...assetPricesMap] : []
+  }
+
+  return result
+}
+
+const transformHistoricalPrices = (pricesMap, assetNames, assets) => {
+  const result = Object.create(null)
+  for (const assetName of assetNames) {
+    const assetPrices = pricesMap.get(assets[assetName].ticker)
+    if (assetPrices) {
+      const closePrices = Object.create(null)
+      // assetPrices is a Map, iterate it directly to avoid creating an intermediate array
+      for (const [time, priceData] of assetPrices.entries()) {
+        closePrices[time] = priceData.close
+      }
+
+      result[assetName] = closePrices
+    } else {
+      result[assetName] = Object.create(null)
+    }
+  }
+
+  return result
+}
 
 const delayWithJitter = (ms, jitter = 0, signal) =>
   delay(Math.floor(ms + Math.random() * jitter), { signal })
@@ -229,29 +248,23 @@ class MarketHistoryMonitor {
       requestLimit: requestLimit || this.#granularityRequestLimits[granularity],
     })
 
-    const mapPrices = (pricesMap) => {
-      return Object.fromEntries(
-        assetNames.map((assetName) => {
-          const assetPricesMap = pricesMap.get(assets[assetName].ticker)
-          const value = assetPricesMap ? [...assetPricesMap] : []
-          return [assetName, value]
-        })
-      )
-    }
-
-    const pricesByAssetName = mapPrices(fetchedPricesMap)
-
-    const fullHistoryByAssetName =
-      fetchedPricesMap.size === historicalPricesMap.size
-        ? pricesByAssetName
-        : mapPrices(historicalPricesMap)
-
     if (fetchedPricesMap.size > 0) {
-      await this.#setCache({ currency, granularity, pricesByAssetName })
+      const pricesByAssetNameForCache = mapPricesForCache(fetchedPricesMap, assetNames, assets)
+      await this.#setCache({
+        currency,
+        granularity,
+        pricesByAssetName: pricesByAssetNameForCache,
+      })
     }
 
+    const fullHistoryByAssetName = transformHistoricalPrices(
+      historicalPricesMap,
+      assetNames,
+      assets
+    )
+
     return {
-      prices: transformPricesByAssetName(fullHistoryByAssetName),
+      prices: fullHistoryByAssetName,
       hasNewPrices: fetchedPricesMap.size > 0,
     }
   }
@@ -631,7 +644,9 @@ class MarketHistoryMonitor {
       })
 
       await this.#marketHistoryAtom.set(async (current) => {
-        const transformedPrices = transformPriceEntries([...updatedAssetHistoryMap])
+        const transformedPrices = Object.fromEntries(
+          [...updatedAssetHistoryMap].map(([time, priceData]) => [time, priceData.close])
+        )
         const parsedGranularity = parseGranularity(granularity)
 
         return {

package/module/last-timestamp-from-prices-map.js

@@ -0,0 +1,4 @@
+const lastTimestampFromPricesMap = (history) =>
+  history.size === 0 ? null : Math.max(...history.keys())
+
+export default lastTimestampFromPricesMap

package/module/prepare-fetch-instructions.js

@@ -0,0 +1,48 @@
+import getLimit from './get-limit-to-fetch.js'
+import tryToGetCache from './try-to-load-cache.js'
+import lastTimestampFromPricesMap from './last-timestamp-from-prices-map.js'
+
+export default async function prepareFetchInstructions({
+  tickerSymbol,
+  fiatTicker,
+  granularity,
+  getCacheFromStorage,
+  requestLimit,
+  specificTimestamp,
+  ignoreCache,
+  runtimeCache,
+  getRuntimeCacheKey,
+  requestTimestamp,
+}) {
+  const runtimeCacheKey = getRuntimeCacheKey({
+    fiatTicker,
+    assetTicker: tickerSymbol,
+    granularity,
+  })
+  const history = await tryToGetCache({
+    runtimeCacheKey,
+    runtimeCache,
+    getCacheFromStorage: () => getCacheFromStorage(tickerSymbol),
+  })
+
+  if (specificTimestamp && history.get(specificTimestamp)) {
+    return { history }
+  }
+
+  const limit = getLimit({
+    granularity,
+    history: ignoreCache ? new Map() : history,
+    requestLimit,
+    requestTimestamp,
+    specificTimestamp,
+  })
+
+  if (limit <= 0) {
+    return { history }
+  }
+
+  const lastCachedTime = lastTimestampFromPricesMap(history)
+  const lastCachedItem = history.get(lastCachedTime)
+
+  return { history, limit, lastCachedItem, tickerSymbol }
+}

package/module/process-api-response.js

@@ -0,0 +1,56 @@
+import findInvalidPrice from './find-invalid-price.js'
+import cropHistory from './crop-history.js'
+
+export default function processApiResponse({
+  assetTicker,
+  fiatTicker,
+  fetchedPrices,
+  assetTickersToFetch,
+  ignoreInvalidSymbols,
+  historicalPricesMap,
+  granularity,
+  specificTimestamp,
+  requestLimit,
+}) {
+  if (assetTicker === 'requestErrors') {
+    const errors = fetchedPrices[assetTicker]
+    Object.keys(errors).forEach((key) => {
+      if (key !== 'invalidCryptoSymbols' || !ignoreInvalidSymbols) {
+        console.warn(`pricing-server: ${key}: `, errors[key])
+      }
+    })
+    return null
+  }
+
+  const newData = fetchedPrices[assetTicker][fiatTicker]
+  const { lastCachedItem } =
+    assetTickersToFetch.find(({ tickerSymbol }) => tickerSymbol === assetTicker) ||
+    Object.create(null)
+  const wrongDataItem = findInvalidPrice(newData, lastCachedItem)
+
+  if (wrongDataItem) {
+    const date = new Date(wrongDataItem.time * 1000)
+    console.warn(
+      `pricing-server: invalid ${assetTicker} price for ${date.toISOString()}: ${JSON.stringify(
+        wrongDataItem
+      )}`
+    )
+    return null
+  }
+
+  const filteredData = newData
+    .filter((piece) => piece.close !== 0)
+    .map((piece) => ({ time: piece.time, close: piece.close }))
+
+  let history = historicalPricesMap.get(assetTicker) || new Map()
+
+  for (const price of filteredData) {
+    history.set(price.time * 1000, { close: price.close })
+  }
+
+  if (granularity === 'hour' && !specificTimestamp && filteredData.length > 0 && requestLimit) {
+    history = cropHistory({ history, hourlyLimit: requestLimit })
+  }
+
+  return history
+}

package/module/try-to-load-cache.js

@@ -0,0 +1,41 @@
+/**
+ * @typedef {import('./types.js').RuntimeCacheType} RuntimeCacheType
+ * @typedef {import('./types.js').HistoryType} HistoryType
+ * @typedef {import('./types.js').GetCacheFromStorageResponseType} GetCacheFromStorageResponseType
+ */
+
+/**
+ * @param {object} options
+ * @param {string} options.runtimeCacheKey
+ * @param {RuntimeCacheType} options.runtimeCache
+ * @param {() => Promise<GetCacheFromStorageResponseType>} options.getCacheFromStorage
+ * @returns {Promise<HistoryType>}
+ */
+
+const tryToGetCache = async ({ runtimeCacheKey, runtimeCache, getCacheFromStorage }) => {
+  const cache = runtimeCache.get(runtimeCacheKey)
+
+  if (cache) {
+    return new Map(cache)
+  }
+
+  if (!getCacheFromStorage) {
+    return new Map()
+  }
+
+  try {
+    let cacheFromStorage = await getCacheFromStorage()
+
+    if (typeof cacheFromStorage === 'string') {
+      // backwards compatibility
+      const fixedCache = JSON.parse(cacheFromStorage)
+      cacheFromStorage = fixedCache
+    }
+
+    return new Map(cacheFromStorage)
+  } catch (e) {
+    throw new Error('failed to load cache from storage. ' + e)
+  }
+}
+
+export default tryToGetCache

package/module/utils.js

@@ -0,0 +1,5 @@
+import ms from 'ms'
+
+export const timeToDays = (time) => time / ms('1d')
+export const timeToHours = (time) => time / ms('1h')
+export const timeToMinutes = (time) => time / ms('1m')

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@exodus/market-history",
-  "version": "10.2.3",
+  "version": "10.3.0",
   "description": "Fetches historical prices for assets",
   "author": "Exodus Movement, Inc.",
   "license": "MIT",
@@ -27,7 +27,6 @@
     "@exodus/atoms": "^9.0.0",
     "@exodus/basic-utils": "^3.0.1",
     "@exodus/dayjs": "^1.0.2",
-    "@exodus/price-api": "^4.1.1",
     "@exodus/remote-config-atoms": "^1.1.0",
     "delay": "^5.0.0",
     "make-concurrent": "^5.4.0",
@@ -59,5 +58,5 @@
   "publishConfig": {
     "access": "public"
   },
-  "gitHead": "a75b31501250fd4a5001ee29eaf929c4dead9c44"
+  "gitHead": "891258ec0890e3af85055d95b59be88d69b79de6"
 }