flexmonster-mongo-connector 2.9.13 → 2.9.16

package/src/cache/DataManager.ts

@@ -19,6 +19,7 @@ import { IRequestArgument } from "../requests/apiRequests/IRequestArgument";
 import { RequestFactory } from "../requests/requestsFactory.ts/RequestsFactory";
 import { ArrayDataObject } from "./dataObject/impl/ArrayDataObject";
 import { SimpleNumericIterator } from "./customIterators/SimpleNumericIterator";
+import { HashGenerator } from "../utils/HashGenerator";
 //import { CachedDataInterface } from "./dataObject/CachedDataInterface";
 //import { AbstractDataObject } from "./dataObject/impl/AbstractDataObject";
 
@@ -26,7 +27,7 @@ export class DataManager {
 
     private _queryBuilder: QueryBuilder;
     private _queryExecutor: MongoQueryExecutor;
-    private _cacheManager: IDataCache<string, any>;
+    private _cacheManager: IDataCache<CacheKeyInterface, any>;
     private isProbabilisticCacheFlushEnabled: boolean = true;
     private _requestsRegister: Register<string, DataIterationInterface>;
     //private readonly CHUNK_SIZE: number = 50000;
@@ -75,13 +76,17 @@ export class DataManager {
     }
 
     private async _getData(apiRequest: IApiRequest): Promise<AbstractDataObject> {
-        let query: string = JSON.stringify(apiRequest.requestArgument.clientQuery);
-        let data: AbstractDataObject = this.getDataFromCache(query);
+        const cacheKey: CacheKeyInterface = {
+            databaseName: apiRequest.requestArgument.db.databaseName,
+            index: apiRequest.requestArgument.index,
+            clientQuery: apiRequest.requestArgument.clientQuery
+        };
+        let data: AbstractDataObject = this.getDataFromCache(cacheKey);
         LoggingManager.log(`Client query: ${JSON.stringify(apiRequest.requestArgument.clientQuery)}`);
 
         if (data === undefined) {
             data = await apiRequest.getData(this._queryBuilder, this._queryExecutor);
-            this.setDataToCache(query, <AbstractDataObject>data);
+            this.setDataToCache(cacheKey, <AbstractDataObject>data);
 
             if (ConfigManager.getInstance().currentConfig.cacheEnabled) {
                 LoggingManager.log(`Putting ${apiRequest.loggingTemplate} data to cache`);
@@ -94,14 +99,14 @@ export class DataManager {
         return data;
     }
 
-    private getDataFromCache(queryString: string): AbstractDataObject {
+    private getDataFromCache(keyObject: CacheKeyInterface): AbstractDataObject {
         if (this._cacheManager === null) return undefined;
-        return this._cacheManager.getCache(queryString); 
+        return this._cacheManager.getCache(keyObject); 
     }
 
-    private setDataToCache(queryString: string, data: AbstractDataObject): void {
+    private setDataToCache(keyObject: CacheKeyInterface, data: AbstractDataObject): void {
         if (this._cacheManager === null) return undefined;
-        this._cacheManager.setCache(queryString, data);
+        this._cacheManager.setCache(keyObject, data);
     }
 
     private getCacheMemoryStatus(): string {
@@ -122,6 +127,17 @@ export class DataManager {
         }
         return dataInstance.getChunk(iterator);
     }
+
+    // private getCacheKey(objectKey: CacheKeyInterface): string {
+    //     if (objectKey == null) throw new Error("Null cache object key exception");
+    //     return HashGenerator.createHashFromObject(objectKey);
+    // }
+}
+
+export interface CacheKeyInterface {
+    clientQuery: object,
+    databaseName: string,
+    index: string    
 }
 
 export interface DataIterationInterface {

package/src/cache/IDataCache.ts

@@ -1,7 +1,7 @@
 import { ICacheStrategie } from "./cacheStrategies/ICacheStrategie";
 
 export interface IDataCache<KeyType, DataType> {
-    hasKey(key: KeyType): boolean;
+    //hasKey(key: KeyType): boolean;
     getCache(key: KeyType): DataType;
     setCache(key: KeyType, data: DataType): void;
     clearCache(): void;

package/src/cache/impl/LocalDataCache.ts

@@ -3,8 +3,10 @@ import { ICacheStrategie } from "../cacheStrategies/ICacheStrategie";
 import { ConfigInterface } from "../../config/ConfigInterface";
 import { AbstractDataObject } from "../dataObject/impl/AbstractDataObject";
 import { DataRetrievalInterface } from "../dataObject/DataRetrievalInterface";
+import { CacheKeyInterface } from "../DataManager";
+import { HashGenerator } from "../../utils/HashGenerator";
 
-export class LocalDataCache implements IDataCache<string, any> {
+export class LocalDataCache implements IDataCache<CacheKeyInterface, any> {
 
     private _cache: Map<string, CachedDataObject> = null;
     private timeToLive: number = 0; //TTL for data objects in minutes, default 0 means we are not going to update cache
@@ -26,14 +28,15 @@ export class LocalDataCache implements IDataCache<string, any> {
         this.timeToLive = minutes;
     }
 
-    public hasKey(key: string): boolean {
-        if (key == null) throw new Error("Null key exception");
-        return this._cache.has(key);
-    }
+    // public hasKey(key: CacheKeyInterface): boolean {
+    //     if (key == null) throw new Error("Null key exception");
+    //     return this._cache.has(key);
+    // }
 
-    public getCache(key: string): DataRetrievalInterface {
+    public getCache(key: CacheKeyInterface): DataRetrievalInterface {
         if (key == null) throw new Error("Null key exception");
-        let cachedDataObject: CachedDataObject = this._cache.get(key);
+        const keyHash: string = this.getCacheKey(key);
+        let cachedDataObject: CachedDataObject = this._cache.get(keyHash);
         
         if (typeof cachedDataObject !== "undefined" && this._cacheStrategie.isCacheStaled(cachedDataObject, this.timeToLive)) {
             // console.log(">>>>>stalled", ((new Date().getTime() - cachedDataObject.timeStamp) / (1000 * 60)));
@@ -56,19 +59,26 @@ export class LocalDataCache implements IDataCache<string, any> {
     //         > cachedDataObject.timeStamp + this.timeToLive * 1000;
     // }
 
-    private removeFromCache(key: string): void {
+    private removeFromCache(key: CacheKeyInterface): void {
         if (key == null) throw new Error("Null key exception");
-        const cachedItem: CachedDataObject = this._cache.get(key);
+        const keyHash: string = this.getCacheKey(key);
+        const cachedItem: CachedDataObject = this._cache.get(keyHash);
         if (typeof cachedItem === "undefined") return;
 
-        this._cache.delete(key);
+        this._removeFromCache(keyHash);
         this._currentCacheSize-= cachedItem.dataMemorySize;
         cachedItem.data = null;
         return;
     }
 
-    public setCache(key: string, data: AbstractDataObject): void {
-        this._cache.set(key, 
+    private _removeFromCache(keyHash: string): void {
+        this._cache.delete(keyHash);
+    }
+
+
+    public setCache(key: CacheKeyInterface, data: AbstractDataObject): void {
+        const keyHash: string = this.getCacheKey(key);
+        this._cache.set(keyHash, 
             {
                 data: data,
                 timeStamp: new Date().getTime(),
@@ -101,7 +111,7 @@ export class LocalDataCache implements IDataCache<string, any> {
         let garbageCollectingItem: GarbageColletingItemsInterface = null;
         while (this._cacheSizeLimit * this._garbageCollectingCoefficient <= this._currentCacheSize) {
             garbageCollectingItem = entriesList.shift();
-            this.removeFromCache(garbageCollectingItem.key);
+            this._removeFromCache(garbageCollectingItem.key);
         }
 
         entriesList = null;
@@ -132,6 +142,12 @@ export class LocalDataCache implements IDataCache<string, any> {
         this._currentCacheSize = 0;
     }
 
+    private getCacheKey(objectKey: CacheKeyInterface): string {
+        if (objectKey == null) throw new Error("Null cache object key exception");
+        const objectKeyString: string = JSON.stringify(objectKey.clientQuery) + objectKey.databaseName + objectKey.index;
+        return HashGenerator.createHashFromString(objectKeyString);
+    }
+
     public getCacheMemoryStatus(): string {
         return `Current cache size: ${(this._currentCacheSize / (1024 * 1024)).toFixed(2)} MB`;
     }

package/src/parsers/MongoResponseParser.ts

@@ -3,12 +3,12 @@ import {Delimeter} from '../utils/consts/Delimeters';
 import {APISchema} from '../schema/APISchema';
 import {IRequestField} from '../requests/apiRequests/IRequestArgument';
 import {MongoFieldType} from '../utils/consts/MongoFieldType';
-import { Cursor } from 'mongodb';
 import { IQuery } from '../query/IQuery';
 import { ArrayDataObject } from '../cache/dataObject/impl/ArrayDataObject';
 import { FlatResultDataObject } from '../cache/dataObject/impl/FlatRequestDataObject';
 import { MemoryCalculator, MemoryConstants } from '../utils/MemoryCalculator';
 import { AggregationApiRequest } from '../requests/apiRequests/impl/AggregationApiRequest';
+import { AbstractCursor } from 'mongodb';
 
 export class MongoResponseParser {
 
@@ -62,14 +62,14 @@ export class MongoResponseParser {
     public parseCalculationsFromCursor(cursor: Promise<any>, query: IQuery[], dataChunkSize: number, startDate: Date = null,
         aggregationApiRequest: AggregationApiRequest): Promise<ArrayDataObject> {
         return new Promise((resolve, reject) => {
-            cursor.then(async (documents: Cursor) => {
+            cursor.then(async (documents: AbstractCursor) => {
                 const parsingResult = await this.parseAggregations(documents, query, dataChunkSize, startDate, aggregationApiRequest);
                 resolve(new ArrayDataObject(parsingResult.result, parsingResult.startDate, parsingResult.memorySize));
             });
         });
     }
 
-    private async parseAggregations(documents: Cursor, queries: IQuery[], dataChunkSize: number, startDate: Date = null, 
+    private async parseAggregations(documents: AbstractCursor, queries: IQuery[], dataChunkSize: number, startDate: Date = null, 
         aggregationApiRequest: AggregationApiRequest): Promise<any> {
 
         let parsedData: any[] = [];
@@ -227,7 +227,7 @@ export class MongoResponseParser {
         });
     }
 
-    private async parseDrillThroughData(documents: Cursor, fields: IRequestField[], dataChunkSize: number): Promise<any> {
+    private async parseDrillThroughData(documents: AbstractCursor, fields: IRequestField[], dataChunkSize: number): Promise<any> {
         const result: any = {
             "fields": [],
             "hits": []

package/src/utils/HashGenerator.ts

@@ -3,6 +3,10 @@ import * as Crypto from 'crypto';
 export class HashGenerator {
 
     public static createHashFromString(input: string): string {
-        return Crypto.createHash('sha512').update(input).digest('hex');
+        return Crypto.createHash('sha256').update(input).digest('hex');
+    }
+
+    public static createHashFromObject(object: any): string {
+        return this.createHashFromString(JSON.stringify(object));
     }
 }