@tradejs/app 1.0.5 → 1.0.8

package/src/app/store/data.ts

@@ -1,11 +1,11 @@
-import { useEffect, useRef, useState } from 'react';
+import { useEffect, useMemo, useState } from 'react';
 import { create } from 'zustand';
-import _ from 'lodash';
 import { get, set } from 'idb-keyval';
 import { useSearchParams } from 'next/navigation';
 import { KlineChartData, Interval, Filters, Provider } from '@tradejs/types';
 import { kline } from '@actions/kline';
 import { isWrongData, mergeData } from '@tradejs/core/data';
+import { normalizeEndToIntervalBoundary } from '@app/lib/klineWindow';
 
 interface DataState {
   data: Map<string, KlineChartData | null>;
@@ -17,8 +17,61 @@ interface DataState {
   ) => void;
 }
 
-const getKey = (filters: Pick<Filters, 'provider' | 'symbol' | 'interval'>) =>
-  `${filters.provider || 'bybit'}_${filters.symbol}_${filters.interval}`;
+interface DataRequest {
+  key: string;
+  provider: Provider;
+  symbol: string;
+  interval: Interval;
+  start: number;
+  end: number;
+  cacheBucketEnd: number;
+  cacheOnly: boolean;
+}
+
+const MIN_CACHED_CANDLES = 2;
+
+const getKey = (provider: Provider, symbol: string, interval: Interval) =>
+  `${provider}_${symbol}_${interval}`;
+
+const getProvider = (provider?: Provider): Provider => provider || 'bybit';
+
+const toRequest = (filters: Filters, cacheOnly: boolean): DataRequest => {
+  const provider = getProvider(filters.provider);
+
+  return {
+    key: getKey(provider, filters.symbol, filters.interval),
+    provider,
+    symbol: filters.symbol,
+    interval: filters.interval,
+    start: filters.start,
+    end: filters.end,
+    cacheBucketEnd: normalizeEndToIntervalBoundary(
+      filters.end,
+      filters.interval,
+    ),
+    cacheOnly,
+  };
+};
+
+const getRequestKey = ({
+  key,
+  start,
+  cacheBucketEnd,
+  cacheOnly,
+}: DataRequest) =>
+  `${key}_${start}_${cacheBucketEnd}_${cacheOnly ? 'cache' : 'live'}`;
+
+const filterDataToWindow = (
+  data: KlineChartData,
+  start: number,
+  end: number,
+): KlineChartData =>
+  data.filter((candle) => candle.timestamp >= start && candle.timestamp <= end);
+
+const hasContinuityData = (data: KlineChartData) =>
+  data.length > MIN_CACHED_CANDLES;
+
+const inFlightRequests = new Map<string, Promise<KlineChartData>>();
 
 const useDataStore = create<DataState>((set) => ({
   data: new Map<string, KlineChartData | null>(),
@@ -26,7 +79,7 @@ const useDataStore = create<DataState>((set) => ({
     set(({ data }) => {
       const next = new Map(data);
 
-      next.set(getKey({ provider, symbol, interval }), newData);
+      next.set(getKey(provider, symbol, interval), newData);
 
       return {
         data: next,
@@ -34,111 +87,189 @@ const useDataStore = create<DataState>((set) => ({
     }),
 }));
 
-export const useData = (filters: Filters) => {
-  const key = getKey(filters);
-  const prevKey = useRef(key);
-  const retried = useRef(false);
-  const [fulfilled, setFulfilled] = useState(false);
-  const storedData = useDataStore((s) => s.data.get(key));
-  const setData = useDataStore((s) => s.setData);
+const loadCachedData = async (key: string) =>
+  ((await get(key)) as KlineChartData | null) ?? [];
 
-  const searchParams = useSearchParams();
-  const cacheOnly = Boolean(searchParams.get('cacheOnly')) ?? false;
+const clearCachedData = async (key: string) => {
+  await set(key, []);
+};
 
-  useEffect(() => {
-    if (key !== prevKey.current) {
-      setFulfilled(false);
-      prevKey.current = key;
-      retried.current = false;
+const getFetchStart = (start: number, data: KlineChartData) =>
+  Math.max(
+    start,
+    hasContinuityData(data) ? data[data.length - 2]?.timestamp || 0 : 0,
+  );
+
+const requestKline = async (
+  request: Pick<
+    DataRequest,
+    'provider' | 'symbol' | 'interval' | 'end' | 'cacheOnly'
+  > & { start: number },
+) =>
+  kline({
+    provider: request.provider,
+    symbol: request.symbol,
+    interval: request.interval,
+    start: request.start,
+    end: request.end,
+    cacheOnly: request.cacheOnly,
+  });
+
+const loadCurrentData = async ({
+  key,
+  symbol,
+  interval,
+}: Pick<DataRequest, 'key' | 'symbol' | 'interval'>) => {
+  let currentData = [
+    ...(useDataStore.getState().data.get(key) ?? []),
+  ] as KlineChartData;
+
+  if (currentData.length < MIN_CACHED_CANDLES) {
+    const cachedData = await loadCachedData(key);
+
+    if (hasContinuityData(cachedData)) {
+      currentData = [...cachedData];
     }
+  }
 
-    const updateData = async () => {
-      const { provider = 'bybit', symbol, interval, start, end } = filters;
-      if (!symbol) {
-        if (!fulfilled) {
-          setFulfilled(true);
-        }
-        return;
-      }
-      let currentData = [...(storedData ?? [])];
+  if (hasContinuityData(currentData) && isWrongData(interval, currentData)) {
+    console.warn('Wrong kline continuity, drop cache', symbol, interval);
+    await clearCachedData(key);
+    return [];
+  }
 
-      if (!currentData || currentData.length < 2) {
-        const cachedResult = (await get(key)) as KlineChartData | null;
+  return currentData;
+};
 
-        if (cachedResult && cachedResult.length > 2) {
-          currentData = [...cachedResult];
-        }
-      }
+const mergeFreshData = async (
+  dataRequest: DataRequest,
+  currentData: KlineChartData,
+) => {
+  const incrementalData = await requestKline({
+    ...dataRequest,
+    start: getFetchStart(dataRequest.start, currentData),
+  });
 
-      if (currentData?.length > 2 && isWrongData(interval, currentData)) {
-        console.warn('Wrong kline continuity, drop cache', symbol, interval);
-        currentData = [];
-        set(key, []);
-      }
+  const mergedData = mergeData(currentData, incrementalData);
 
-      const normStart = Math.max(
-        start,
-        currentData?.length > 2
-          ? currentData[currentData.length - 2]?.timestamp || 0
-          : 0,
-      );
-
-      const newData = await kline({
-        provider,
-        symbol,
-        interval,
-        start: normStart,
-        end,
-        cacheOnly,
-      });
-
-      const finalData = mergeData(currentData, newData);
-
-      if (
-        !cacheOnly &&
-        !retried.current &&
-        finalData.length > 2 &&
-        isWrongData(interval, finalData)
-      ) {
-        console.warn(
-          'Wrong kline continuity after merge, refetch full',
-          symbol,
+  if (
+    dataRequest.cacheOnly ||
+    !hasContinuityData(mergedData) ||
+    !isWrongData(dataRequest.interval, mergedData)
+  ) {
+    return mergedData;
+  }
+
+  console.warn(
+    'Wrong kline continuity after merge, refetch full',
+    dataRequest.symbol,
+    dataRequest.interval,
+  );
+  await clearCachedData(dataRequest.key);
+
+  const fullData = await requestKline({
+    ...dataRequest,
+    start: dataRequest.start,
+  });
+
+  return mergeData([], fullData);
+};
+
+const persistData = async (
+  {
+    provider,
+    symbol,
+    interval,
+    key,
+  }: Pick<DataRequest, 'provider' | 'symbol' | 'interval' | 'key'>,
+  data: KlineChartData,
+) => {
+  useDataStore.getState().setData(provider, symbol, interval, data);
+  await set(key, data);
+};
+
+const fetchAndStoreData = async (dataRequest: DataRequest) => {
+  const requestKey = getRequestKey(dataRequest);
+  const existingRequest = inFlightRequests.get(requestKey);
+
+  if (existingRequest) {
+    return existingRequest;
+  }
+
+  const pendingRequest = (async () => {
+    const currentData = await loadCurrentData(dataRequest);
+    const finalData = await mergeFreshData(dataRequest, currentData);
+    await persistData(dataRequest, finalData);
+
+    return finalData;
+  })().finally(() => {
+    inFlightRequests.delete(requestKey);
+  });
+
+  inFlightRequests.set(requestKey, pendingRequest);
+
+  return pendingRequest;
+};
+
+export const useData = (filters: Filters) => {
+  const searchParams = useSearchParams();
+  const cacheOnly = Boolean(searchParams.get('cacheOnly')) ?? false;
+  const { end, interval, provider, start, symbol } = filters;
+  const dataRequest = useMemo(
+    () =>
+      toRequest(
+        {
+          end,
           interval,
-        );
-        retried.current = true;
-        set(key, []);
-        const refetchData = await kline({
           provider,
-          symbol,
-          interval,
           start,
-          end,
-          cacheOnly,
-        });
-        const cleaned = mergeData([], refetchData);
-        setData(provider as Provider, symbol, interval, cleaned);
-        if (!fulfilled) {
-          setFulfilled(true);
+          symbol,
+        } as Filters,
+        cacheOnly,
+      ),
+    [cacheOnly, end, interval, provider, start, symbol],
+  );
+  const requestKey = useMemo(() => getRequestKey(dataRequest), [dataRequest]);
+  const [fulfilledRequestKey, setFulfilledRequestKey] = useState<string | null>(
+    null,
+  );
+  const storedData = useDataStore((s) => s.data.get(dataRequest.key));
+  const windowedData = useMemo(
+    () =>
+      filterDataToWindow(storedData ?? [], dataRequest.start, dataRequest.end),
+    [dataRequest.end, dataRequest.start, storedData],
+  );
+
+  const fulfilled = fulfilledRequestKey === requestKey;
+
+  useEffect(() => {
+    let cancelled = false;
+
+    const updateData = async () => {
+      if (!dataRequest.symbol) {
+        if (!cancelled) {
+          setFulfilledRequestKey(requestKey);
         }
-        set(key, cleaned);
         return;
       }
 
-      setData(provider as Provider, symbol, interval, finalData);
+      await fetchAndStoreData(dataRequest);
 
-      if (!fulfilled) {
-        setFulfilled(true);
+      if (!cancelled) {
+        setFulfilledRequestKey(requestKey);
       }
-
-      set(key, finalData);
     };
 
     void updateData();
-  }, [cacheOnly, filters, fulfilled, key, setData, storedData]);
+
+    return () => {
+      cancelled = true;
+    };
+  }, [dataRequest, requestKey]);
 
   return {
-    key,
-    data: storedData ?? [],
+    key: dataRequest.key,
+    data: windowedData,
     fulfilled,
   };
 };

package/src/app/store/index.ts

@@ -3,6 +3,7 @@ export { useTickers } from './tickers';
 export { useIndicators } from './indicators';
 export type { IndicatorRendererConfig } from './indicators';
 export { useData } from './data';
+export { useAiChatStore } from './ai';
 export {
   useTest,
   useTestsCompare,

package/src/app/store/tests.ts

@@ -16,6 +16,8 @@ import { parseTestName } from '@tradejs/core/backtest';
 
 const COMPARE_LOCAL_STORAGE_KEY = 'compare';
 const FAVORITE_LOCAL_STORAGE_KEY = 'favorite';
+const BACKTEST_FILES_CACHE_KEY = 'backtest-files';
+const BACKTEST_FILES_CACHE_TTL_MS = 5 * 60 * 1000;
 
 const COLORS = [
   'purple',
@@ -28,6 +30,14 @@ const COLORS = [
   'green',
 ];
 
+type BacktestFilesCacheRecord = {
+  savedAt: number;
+  items: Items;
+};
+
+const isFresh = (savedAt: number) =>
+  Date.now() - savedAt < BACKTEST_FILES_CACHE_TTL_MS;
+
 interface BacktestState {
   backtests: Map<string, OrderLogData | null>;
   setBacktest: (id: string, backtest: OrderLogData) => void;
@@ -100,15 +110,26 @@ const useFavoriteTetstsStore = create<FavotiteTestsState>()(
 
 interface TestListState {
   tests: Items;
-  setTest: (tests: Items) => void;
+  loadedAt: number;
+  inFlight?: Promise<Items>;
+  setTest: (tests: Items, loadedAt?: number) => void;
+  setInFlight: (request?: Promise<Items>) => void;
   removeTest: (testName: string) => void;
 }
 
 const useTestListStore = create<TestListState>((set) => ({
   tests: [],
-  setTest: (tests) =>
+  loadedAt: 0,
+  inFlight: undefined,
+  setTest: (tests, loadedAt = Date.now()) =>
     set(() => ({
       tests,
+      loadedAt,
+      inFlight: undefined,
+    })),
+  setInFlight: (request) =>
+    set(() => ({
+      inFlight: request,
     })),
   removeTest: (testName) =>
     set(({ tests }) => ({
@@ -184,6 +205,44 @@ const useTestsStore = create<TestsState>((set) => ({
     }),
 }));
 
+const loadBacktestFilesList = async () => {
+  const { tests, loadedAt, inFlight, setInFlight, setTest } =
+    useTestListStore.getState();
+
+  if (loadedAt > 0 && isFresh(loadedAt)) {
+    return tests;
+  }
+
+  if (inFlight) {
+    return inFlight;
+  }
+
+  const pending = (async () => {
+    const cached = (await get(
+      BACKTEST_FILES_CACHE_KEY,
+    )) as BacktestFilesCacheRecord | null;
+
+    if (cached?.savedAt && isFresh(cached.savedAt)) {
+      setTest(cached.items, cached.savedAt);
+      return cached.items;
+    }
+
+    const newTests = await getBacktestFiles();
+    const savedAt = Date.now();
+    setTest(newTests, savedAt);
+    await set(BACKTEST_FILES_CACHE_KEY, {
+      savedAt,
+      items: newTests,
+    } satisfies BacktestFilesCacheRecord);
+    return newTests;
+  })().finally(() => {
+    useTestListStore.getState().setInFlight(undefined);
+  });
+
+  setInFlight(pending);
+  return pending;
+};
+
 export const useFavoriteTests = () => {
   const favotites = useFavoriteTetstsStore((s) => s.tests);
   const toggleFavorite = useFavoriteTetstsStore((s) => s.toggleFavorite);
@@ -215,14 +274,15 @@ export const useFavoriteTests = () => {
 
 interface TestListProps {
   symbol?: string;
+  enabled?: boolean;
 }
 
 export const useTestList = (filters: TestListProps = {}) => {
+  const { enabled = true } = filters;
   const [loadding, setLoading] = useState(false);
   const [fulFilled, setFulfilled] = useState(false);
   const [error, setError] = useState<unknown>(null);
   const tests = useTestListStore((s) => s.tests);
-  const setTest = useTestListStore((s) => s.setTest);
   const { favoriteItems } = useFavoriteTests();
 
   const testStrategyMap = new Map(
@@ -266,22 +326,24 @@ export const useTestList = (filters: TestListProps = {}) => {
   const noData = _.isEmpty(testItems);
 
   useEffect(() => {
+    if (!enabled) {
+      return;
+    }
+
     const loadData = async () => {
       try {
         setLoading(true);
         await delay();
-        const newTests = await getBacktestFiles();
+        await loadBacktestFilesList();
         setLoading(false);
         setFulfilled(true);
-
-        setTest(newTests);
       } catch (err) {
         setError(err);
       }
     };
 
     void loadData();
-  }, [setTest]);
+  }, [enabled]);
 
   return {
     loadding,
@@ -289,6 +351,7 @@ export const useTestList = (filters: TestListProps = {}) => {
     error,
     noData,
     tests: testItems,
+    ensureLoaded: loadBacktestFilesList,
   };
 };
 
@@ -318,7 +381,7 @@ export const useTest = (testName: string) => {
 
       let resolvedStrategy = strategyName;
       if (!resolvedStrategy) {
-        const newTests = await getBacktestFiles();
+        const newTests = await loadBacktestFilesList();
         setTestList(newTests);
         resolvedStrategy = newTests.find((item) => item.value === testName)
           ?.data?.strategyName as string | undefined;
@@ -400,6 +463,16 @@ export const useBacktestMutations = () => {
     removeFavorite(testName);
     removeFromCompare(testName);
 
+    const cache = (await get(
+      BACKTEST_FILES_CACHE_KEY,
+    )) as BacktestFilesCacheRecord | null;
+    if (cache?.items) {
+      await set(BACKTEST_FILES_CACHE_KEY, {
+        ...cache,
+        items: cache.items.filter((item) => item.value !== testName),
+      } satisfies BacktestFilesCacheRecord);
+    }
+
     await Promise.all([del(`test-${testName}`), del(`backtest-${testName}`)]);
   };
 
@@ -438,7 +511,7 @@ export const useBacktest = (id: string | undefined) => {
 
       let resolvedStrategy = strategyName;
       if (!resolvedStrategy) {
-        const newTests = await getBacktestFiles();
+        const newTests = await loadBacktestFilesList();
         setTestList(newTests);
         resolvedStrategy = newTests.find((item) => item.value === id)?.data
           ?.strategyName as string | undefined;
@@ -462,3 +535,17 @@ export const useBacktest = (id: string | undefined) => {
     loading,
   };
 };
+
+export const resetTestsStoreForTests = () => {
+  useDataStore.setState({
+    backtests: new Map<string, OrderLogData | null>(),
+  });
+  useTestListStore.setState({
+    tests: [],
+    loadedAt: 0,
+    inFlight: undefined,
+  });
+  useTestsStore.setState({
+    tests: new Map<string, TestResult | null>(),
+  });
+};