@tolgee/core 5.33.2 → 5.33.3-prerelease.2efc0e6b.0

package/src/Controller/Cache/Cache.ts

@@ -2,14 +2,16 @@ import {
   CacheDescriptor,
   CacheDescriptorInternal,
   NsFallback,
-  TranslationsFlat,
   TranslationValue,
   TreeTranslationsData,
   BackendGetRecordInternal,
   RecordFetchError,
+  LoadOptions,
+  CacheInternalRecord,
+  TranslationsFlat,
 } from '../../types';
 import { getFallbackArray, isPromise, unique } from '../../helpers';
-import { TolgeeStaticData } from '../State/initState';
+import { TolgeeStaticData, TolgeeStaticDataProp } from '../State/initState';
 import { ValueObserverInstance } from '../ValueObserver';
 
 import { decodeCacheKey, encodeCacheKey, flattenTranslations } from './helpers';
@@ -33,6 +35,7 @@ export function Cache(
   backendGetDevRecord: BackendGetRecordInternal,
   withDefaultNs: (descriptor: CacheDescriptor) => CacheDescriptorInternal,
   isInitialLoading: () => boolean,
+  isCacheDisabled: () => boolean,
   fetchingObserver: ValueObserverInstance<boolean>,
   loadingObserver: ValueObserverInstance<boolean>
 ) {
@@ -43,7 +46,7 @@ export function Cache(
 
   function addRecordInternal(
     descriptor: CacheDescriptorInternal,
-    data: TreeTranslationsData,
+    data: TranslationsFlat,
     recordVersion: number
   ) {
     const cacheKey = encodeCacheKey(descriptor);
@@ -51,7 +54,7 @@ export function Cache(
       data: flattenTranslations(data),
       version: recordVersion,
     });
-    events.onCacheChange.emit(descriptor);
+    events.onCacheChange.emit(decodeCacheKey(cacheKey));
   }
 
   /**
@@ -108,15 +111,23 @@ export function Cache(
   }
 
   const self = Object.freeze({
-    addStaticData(data: TolgeeStaticData | undefined) {
-      if (data) {
+    addStaticData(data: TolgeeStaticDataProp | undefined) {
+      if (Array.isArray(data)) {
+        for (const record of data) {
+          const key = encodeCacheKey(record);
+          const existing = cache.get(key);
+          if (!existing || existing.version === 0) {
+            addRecordInternal(record, flattenTranslations(record.data), 0);
+          }
+        }
+      } else if (data) {
         staticData = { ...staticData, ...data };
         Object.entries(data).forEach(([key, value]) => {
           if (typeof value !== 'function') {
             const descriptor = decodeCacheKey(key);
             const existing = cache.get(key);
             if (!existing || existing.version === 0) {
-              addRecordInternal(descriptor, value, 0);
+              addRecordInternal(descriptor, flattenTranslations(value), 0);
             }
           }
         });
@@ -129,7 +140,7 @@ export function Cache(
     },
 
     addRecord(descriptor: CacheDescriptorInternal, data: TreeTranslationsData) {
-      addRecordInternal(descriptor, data, version);
+      addRecordInternal(descriptor, flattenTranslations(data), version);
     },
 
     exists(descriptor: CacheDescriptorInternal, strict = false) {
@@ -140,20 +151,34 @@ export function Cache(
       return Boolean(record);
     },
 
-    getRecord(descriptor: CacheDescriptor) {
-      return cache.get(encodeCacheKey(withDefaultNs(descriptor)))?.data;
+    getRecord(descriptor: CacheDescriptor): CacheInternalRecord | undefined {
+      const descriptorWithNs = withDefaultNs(descriptor);
+      const cacheKey = encodeCacheKey(descriptorWithNs);
+      const cacheRecord = cache.get(cacheKey);
+      if (!cacheRecord) {
+        return undefined;
+      }
+      return {
+        ...descriptorWithNs,
+        cacheKey,
+        data: cacheRecord.data,
+      };
+    },
+
+    getAllRecords() {
+      const entries = Array.from(cache.entries());
+      return entries.map(([key]) => self.getRecord(decodeCacheKey(key)));
     },
 
     getTranslation(descriptor: CacheDescriptorInternal, key: string) {
-      return cache.get(encodeCacheKey(descriptor))?.data.get(key);
+      return cache.get(encodeCacheKey(descriptor))?.data[key];
     },
 
     getTranslationNs(namespaces: string[], languages: string[], key: string) {
       for (const namespace of namespaces) {
         for (const language of languages) {
-          const value = cache
-            .get(encodeCacheKey({ language, namespace }))
-            ?.data.get(key);
+          const value = cache.get(encodeCacheKey({ language, namespace }))
+            ?.data[key];
           if (value !== undefined && value !== null) {
             return [namespace];
           }
@@ -169,9 +194,8 @@ export function Cache(
     ) {
       for (const namespace of namespaces) {
         for (const language of languages) {
-          const value = cache
-            .get(encodeCacheKey({ language, namespace }))
-            ?.data.get(key);
+          const value = cache.get(encodeCacheKey({ language, namespace }))
+            ?.data[key];
           if (value !== undefined && value !== null) {
             return value;
           }
@@ -186,8 +210,10 @@ export function Cache(
       value: TranslationValue
     ) {
       const record = cache.get(encodeCacheKey(descriptor))?.data;
-      record?.set(key, value);
-      events.onCacheChange.emit({ ...descriptor, key });
+      if (record?.[key]) {
+        record[key] = value;
+        events.onCacheChange.emit({ ...descriptor, key });
+      }
     },
 
     isFetching(ns?: NsFallback) {
@@ -206,84 +232,119 @@ export function Cache(
       );
     },
 
-    isLoading(language: string | undefined, ns?: NsFallback) {
+    isLoading(language: string, ns?: NsFallback) {
       const namespaces = getFallbackArray(ns);
 
+      if (isInitialLoading()) {
+        return true;
+      }
+
+      const pendingCacheKeys = Array.from(asyncRequests.keys());
+
       return Boolean(
-        isInitialLoading() ||
-          Array.from(asyncRequests.keys()).find((key) => {
-            const descriptor = decodeCacheKey(key);
-            return (
-              (!namespaces.length ||
-                namespaces.includes(descriptor.namespace)) &&
-              !self.exists({
-                namespace: descriptor.namespace,
-                language: language!,
-              })
-            );
-          })
+        pendingCacheKeys.find((key) => {
+          const descriptor = decodeCacheKey(key);
+          return (
+            (!namespaces.length || namespaces.includes(descriptor.namespace)) &&
+            !self.exists({
+              namespace: descriptor.namespace,
+              language: language,
+            })
+          );
+        })
       );
     },
 
-    async loadRecords(descriptors: CacheDescriptor[], isDev: boolean) {
-      const withPromises = descriptors.map((descriptor) => {
+    async loadRecords(
+      descriptors: CacheDescriptor[],
+      options?: LoadOptions
+    ): Promise<CacheInternalRecord[]> {
+      type WithPromise = {
+        new: boolean;
+        language: string;
+        namespace: string;
+        cacheKey: string;
+        promise?: Promise<TreeTranslationsData | undefined>;
+        data?: TranslationsFlat;
+      };
+
+      const withPromises: WithPromise[] = descriptors.map((descriptor) => {
         const keyObject = withDefaultNs(descriptor);
         const cacheKey = encodeCacheKey(keyObject);
+        if (options?.useCache) {
+          const exists = self.exists(keyObject, true);
+
+          if (exists) {
+            return {
+              ...keyObject,
+              new: false,
+              cacheKey,
+              data: self.getRecord(keyObject)!.data,
+            } as WithPromise;
+          }
+        }
+
         const existingPromise = asyncRequests.get(cacheKey);
 
         if (existingPromise) {
           return {
+            ...keyObject,
             new: false,
             promise: existingPromise,
-            keyObject,
             cacheKey,
           };
         }
+
         const dataPromise =
-          fetchData(keyObject, isDev) || Promise.resolve(undefined);
+          fetchData(keyObject, !options?.noDev) || Promise.resolve(undefined);
+
         asyncRequests.set(cacheKey, dataPromise);
+
         return {
+          ...keyObject,
           new: true,
           promise: dataPromise,
-          keyObject,
           cacheKey,
         };
       });
       fetchingObserver.notify();
       loadingObserver.notify();
 
-      const results = await Promise.all(withPromises.map((val) => val.promise));
+      const promisesToWait = withPromises
+        .map((val) => val.promise)
+        .filter(Boolean);
 
-      withPromises.forEach((value, i) => {
-        const promiseChanged =
-          asyncRequests.get(value.cacheKey) !== value.promise;
+      const fetchedData = await Promise.all(promisesToWait);
+
+      withPromises.forEach((value) => {
+        if (value.promise) {
+          value.data = flattenTranslations(fetchedData[0] ?? {});
+          fetchedData.shift();
+        }
         // if promise has changed in between, it means cache been invalidated or
         // new data are being fetched
+        const promiseChanged =
+          asyncRequests.get(value.cacheKey) !== value.promise;
         if (value.new && !promiseChanged) {
           asyncRequests.delete(value.cacheKey);
-          const data = results[i];
-          if (data) {
-            self.addRecord(value.keyObject, data);
-          } else if (!self.getRecord(value.keyObject)) {
+          if (value.data) {
+            self.addRecord(value, value.data);
+          } else if (!self.getRecord(value)) {
             // if no data exist, put empty object
-            self.addRecord(value.keyObject, {});
+            // so we know we don't have to fetch again
+            self.addRecord(value, {});
           }
         }
       });
       fetchingObserver.notify();
       loadingObserver.notify();
 
-      return withPromises.map((val) => self.getRecord(val.keyObject)!);
-    },
-
-    getAllRecords() {
-      const entries = Array.from(cache.entries());
-      return entries.map(([key, entry]) => {
-        return {
-          ...decodeCacheKey(key),
-          data: entry.data,
-        };
-      });
+      return withPromises.map((val) => ({
+        language: val.language,
+        namespace: val.namespace,
+        data: val.data ?? {},
+        cacheKey: val.cacheKey,
+      }));
     },
   });
 

package/src/Controller/Cache/helpers.ts

@@ -1,6 +1,10 @@
-import { CacheDescriptorInternal, TreeTranslationsData } from '../../types';
+import {
+  CacheDescriptorInternal,
+  TranslationsFlat,
+  TreeTranslationsData,
+} from '../../types';
 
-export const flattenTranslations = (
+export const flattenTranslationsToMap = (
   data: TreeTranslationsData
 ): Map<string, string> => {
   const result: Map<string, string> = new Map();
@@ -10,7 +14,7 @@ export const flattenTranslations = (
       return;
     }
     if (typeof value === 'object') {
-      flattenTranslations(value).forEach((flatValue, flatKey) => {
+      flattenTranslationsToMap(value).forEach((flatValue, flatKey) => {
         result.set(key + '.' + flatKey, flatValue);
       });
       return;
@@ -20,6 +24,11 @@ export const flattenTranslations = (
   return result;
 };
 
+export const flattenTranslations = (
+  data: TreeTranslationsData
+): TranslationsFlat => {
+  return Object.fromEntries(flattenTranslationsToMap(data).entries());
+};
 export const decodeCacheKey = (key: string): CacheDescriptorInternal => {
   const [firstPart, ...rest] = key.split(':');
   // if namespaces contains ":" it won't get lost

package/src/Controller/Controller.ts

@@ -6,9 +6,14 @@ import {
   TFnType,
   NsType,
   KeyAndNamespacesInternal,
+  CacheDescriptorInternal,
+  LoadOptions,
+  LoadRequiredOptions,
+  LoadMatrixOptions,
+  MatrixOptions,
 } from '../types';
 import { Cache } from './Cache/Cache';
-import { getFallbackArray } from '../helpers';
+import { getFallbackArray, unique } from '../helpers';
 import { Plugins } from './Plugins/Plugins';
 import { ValueObserver } from './ValueObserver';
 import { State } from './State/State';
@@ -20,7 +25,7 @@ type StateServiceProps = {
 };
 
 export function Controller({ options }: StateServiceProps) {
-  const events = Events(getFallbackNs, getDefaultNs);
+  const events = Events();
   const fetchingObserver = ValueObserver<boolean>(
     false,
     () => cache.isFetching(),
@@ -55,6 +60,7 @@ export function Controller({ options }: StateServiceProps) {
     pluginService.getBackendDevRecord,
     state.withDefaultNs,
     state.isInitialLoading,
+    state.isCacheDisabled,
     fetchingObserver,
     loadingObserver
   );
@@ -82,16 +88,16 @@ export function Controller({ options }: StateServiceProps) {
   // gets all namespaces where translation could be located
   // takes (ns|default, fallback ns)
   function getDefaultAndFallbackNs(ns?: NsType) {
-    return [...getFallbackArray(getDefaultNs(ns)), ...getFallbackNs()];
+    return unique([...getFallbackArray(getDefaultNs(ns)), ...getFallbackNs()]);
   }
 
   // gets all namespaces which need to be loaded
   // takes (ns|default, initial ns, fallback ns, active ns)
-  function getRequiredNamespaces(ns: NsFallback) {
-    return [
+  function getRequiredNamespaces(ns?: NsFallback) {
+    return unique([
       ...getFallbackArray(ns ?? getDefaultNs()),
       ...state.getRequiredNamespaces(),
-    ];
+    ]);
   }
 
   function changeTranslation(
@@ -114,25 +120,55 @@ export function Controller({ options }: StateServiceProps) {
     cache.addStaticData(state.getInitialOptions().staticData);
   }
 
-  function getRequiredRecords(lang?: string, ns?: NsFallback) {
+  function getRequiredDescriptors(lang?: string, ns?: NsFallback) {
     const languages = state.getFallbackLangs(lang);
     const namespaces = getRequiredNamespaces(ns);
-    const result: CacheDescriptor[] = [];
+    const result: CacheDescriptorInternal[] = [];
     languages.forEach((language) => {
       namespaces.forEach((namespace) => {
-        if (!cache.exists({ language, namespace }, true)) {
-          result.push({ language, namespace });
-        }
+        result.push({ language, namespace });
       });
     });
     return result;
   }
 
-  function loadRequiredRecords(lang?: string, ns?: NsFallback) {
-    const descriptors = getRequiredRecords(lang, ns);
-    if (descriptors.length) {
-      return valueOrPromise(self.loadRecords(descriptors), () => {});
+  function getMissingDescriptors(lang?: string, ns?: NsFallback) {
+    return getRequiredDescriptors(lang, ns).filter(
+      (descriptor) => !cache.exists(descriptor, true)
+    );
+  }
+
+  function getMatrixRecords(options: MatrixOptions) {
+    let languages: string[] = [];
+    let namespaces: string[] = [];
+    if (Array.isArray(options.languages)) {
+      languages = options.languages;
+    } else if (options.languages === 'all') {
+      const availableLanguages = self.getAvailableLanguages();
+      if (!availableLanguages) {
+        throw new Error(missingOptionError('availableLanguages'));
+      }
+      languages = availableLanguages;
+    }
+
+    if (Array.isArray(options.namespaces)) {
+      namespaces = options.namespaces;
+    } else if (options.namespaces === 'all') {
+      const availableNs = self.getAvailableNs();
+      if (!availableNs) {
+        throw new Error(missingOptionError('availableNs'));
+      }
+      namespaces = availableNs;
     }
+
+    const records: CacheDescriptorInternal[] = [];
+
+    languages.forEach((language) => {
+      namespaces.forEach((namespace) => {
+        records.push({ language, namespace });
+      });
+    });
+    return records;
   }
 
   function getTranslationNs({ key, ns }: KeyAndNamespacesInternal) {
@@ -149,8 +185,13 @@ export function Controller({ options }: StateServiceProps) {
 
   function loadInitial() {
     const data = valueOrPromise(initializeLanguage(), () => {
-      // fail if there is no language
-      return loadRequiredRecords();
+      const missingDescriptors = getMissingDescriptors();
+      if (
+        missingDescriptors.length &&
+        state.getInitialOptions().autoLoadRequiredData
+      ) {
+        return cache.loadRecords(missingDescriptors, { useCache: true });
+      }
     });
 
     if (isPromise(data)) {
@@ -208,7 +249,7 @@ export function Controller({ options }: StateServiceProps) {
     getTranslationNs: getTranslationNs,
     getDefaultAndFallbackNs: getDefaultAndFallbackNs,
     findPositions: pluginService.findPositions,
-    getRequiredRecords: getRequiredRecords,
+    getRequiredDescriptors: getRequiredDescriptors,
     async changeLanguage(language: string) {
       if (
         state.getPendingLanguage() === language &&
@@ -218,8 +259,10 @@ export function Controller({ options }: StateServiceProps) {
       }
       state.setPendingLanguage(language);
 
-      if (state.isRunning()) {
-        await loadRequiredRecords(language);
+      if (state.isRunning() && state.getInitialOptions().autoLoadRequiredData) {
+        await cache.loadRecords(getRequiredDescriptors(language), {
+          useCache: true,
+        });
       }
 
       if (language === state.getPendingLanguage()) {
@@ -235,16 +278,14 @@ export function Controller({ options }: StateServiceProps) {
         state.addActiveNs(ns);
       }
       if (state.isRunning()) {
-        await loadRequiredRecords(undefined, ns);
+        await cache.loadRecords(getRequiredDescriptors(undefined, ns), {
+          useCache: true,
+        });
       }
     },
 
-    loadRecords(descriptors: CacheDescriptor[]) {
-      return cache.loadRecords(descriptors, self.isDev());
-    },
-
-    async loadRecord(descriptor: CacheDescriptor) {
-      return (await self.loadRecords([descriptor]))[0];
+    async loadRecord(descriptor: CacheDescriptor, options?: LoadOptions) {
+      return (await self.loadRecords([descriptor], options))[0]?.data;
     },
 
     isLoading(ns?: NsFallback) {
@@ -282,6 +323,19 @@ export function Controller({ options }: StateServiceProps) {
       );
     },
 
+    async loadRequired(options?: LoadRequiredOptions) {
+      if (!options?.language) {
+        await initializeLanguage();
+      }
+      const requiredRecords = getRequiredDescriptors(options?.language);
+      return self.loadRecords(requiredRecords, options);
+    },
+
+    async loadMatrix(options: LoadMatrixOptions) {
+      const records = getMatrixRecords(options);
+      return self.loadRecords(records, options);
+    },
+
     run() {
       checkCorrectConfiguration();
       if (!state.isRunning()) {

package/src/Controller/Events/EventEmitter.ts

@@ -1,13 +1,14 @@
-import { Subscription, Listener } from '../../types';
+import { Subscription, Handler, ListenerEvent } from '../../types';
 
-export function EventEmitter<T>(
+export const EventEmitter = <Event extends ListenerEvent<string, any>>(
+  type: Event['type'],
   isActive: () => boolean
-): EventEmitterInstance<T> {
-  let handlers: Listener<T>[] = [];
+): EventEmitterInstance<Event> => {
+  let handlers: Handler<Event>[] = [];
 
-  return Object.freeze({
-    listen(handler: Listener<T>): Subscription {
-      const handlerWrapper: Listener<T> = (e) => {
+  return {
+    listen(handler: (e: Event) => void): Subscription {
+      const handlerWrapper: Handler<Event> = (e) => {
         handler(e);
       };
 
@@ -19,15 +20,22 @@ export function EventEmitter<T>(
         },
       };
     },
-    emit(data: T) {
+    emit(data: Event['value']) {
       if (isActive()) {
-        handlers.forEach((handler) => handler({ value: data }));
+        handlers.forEach((handler) =>
+          handler({ type: type, value: data } as Event)
+        );
       }
     },
-  });
-}
-
-export type EventEmitterInstance<T> = {
-  readonly listen: (handler: Listener<T>) => Subscription;
-  readonly emit: (data: T) => void;
+  } as unknown as EventEmitterInstance<Event>;
 };
+
+export type EventEmitterInstance<Event> =
+  Event extends ListenerEvent<infer E, infer T>
+    ? {
+        readonly listen: (
+          handler: Handler<ListenerEvent<E, T>>
+        ) => Subscription;
+        readonly emit: (data: T) => void;
+      }
+    : never;

package/src/Controller/Events/EventEmitterCombined.ts

@@ -0,0 +1,55 @@
+import { Subscription, ListenerEvent, CombinedHandler } from '../../types';
+
+export function EventEmitterCombined<E extends ListenerEvent<string, any>>(
+  isActive: () => boolean
+): EventEmitterCombinedInstance<E> {
+  let handlers: CombinedHandler<E>[] = [];
+
+  let queue: E[] = [];
+
+  // merge events in queue into one event
+  function solveQueue() {
+    if (queue.length === 0) {
+      return;
+    }
+    const queueCopy = queue;
+    queue = [];
+    handlers.forEach((handler) => {
+      handler(queueCopy);
+    });
+  }
+
+  return Object.freeze({
+    listen(handler: (e: E[]) => void): Subscription {
+      const handlerWrapper: CombinedHandler<E> = (events) => {
+        handler(events);
+      };
+
+      handlers.push(handlerWrapper);
+
+      return {
+        unsubscribe() {
+          handlers = handlers.filter((i) => handlerWrapper !== i);
+        },
+      };
+    },
+    emit(e: E, delayed: boolean) {
+      if (isActive()) {
+        if (isActive()) {
+          queue.push(e);
+          if (!delayed) {
+            solveQueue();
+          } else {
+            setTimeout(solveQueue, 0);
+          }
+        }
+      }
+    },
+  });
+}
+
+export type EventEmitterCombinedInstance<E extends ListenerEvent<string, any>> =
+  {
+    readonly listen: (handler: CombinedHandler<E>) => Subscription;
+    readonly emit: (e: E, delayed: boolean) => void;
+  };