@bikdotai/bik-shared-backend 20.3.1 → 20.3.2-beta.0

package/lib/elastic/reports/reports.service.js

@@ -74,7 +74,6 @@ _a = ReportService;
 ReportService.generateReport = (args) => __awaiter(void 0, void 0, void 0, function* () {
     var _b;
     var _c, _d;
-    const startTime = new Date().getTime();
     let { storeId, moduleName, reportName, payload, getSignedUrl } = args;
     const module = yield (_b = './' + moduleName + '/' + reportName, Promise.resolve().then(() => __importStar(require(_b))));
     const { exportFileName } = payload;
@@ -83,6 +82,9 @@ ReportService.generateReport = (args) => __awaiter(void 0, void 0, void 0, funct
     try {
         const reportInstance = new module[fileName]();
         const response = yield reportInstance.executeReport(storeId, payload);
+        console.log(`@ReportService generateReport initial apiResponse for ${reportName}`, {
+            response: JSON.stringify(response),
+        });
         if (response.response &&
             (response.nextPage || Object.keys((_c = response.searchAfter) !== null && _c !== void 0 ? _c : {}).length)) {
             const uuid = (0, uuid_1.v4)();
@@ -91,7 +93,7 @@ ReportService.generateReport = (args) => __awaiter(void 0, void 0, void 0, funct
             yield file_helper_1.FileHelper.appendDataToLocalFileCSV(fileName, response.header, response.response, true);
             const timer = new timer_1.Timer();
             timer.startTimer();
-            const res = yield ReportService.fetchDataAndSaveInFile({
+            return yield ReportService.fetchDataAndSaveInFile({
                 args,
                 reportInstance,
                 nextPage: response.nextPage,
@@ -100,13 +102,8 @@ ReportService.generateReport = (args) => __awaiter(void 0, void 0, void 0, funct
                 actionType: payload.actionType,
                 timer,
             });
-            const endTime = new Date().getTime();
-            console.log('@generateReport', 'Time taken to generate recur report in sec', reportName, (endTime - startTime) / 1000);
-            return res;
         }
         response.csvLink = yield reports_helper_1.ReportsServiceHelper.convertTableDataToCSV(response.response || [], response.header, storeId, exportFileName || reportName, getSignedUrl);
-        const endTime = new Date().getTime();
-        console.log('@generateReport', 'Time taken to generate report in sec', reportName, (endTime - startTime) / 1000);
         return apis_1.FunctionResponseHelper.success(response);
     }
     catch (error) {
@@ -124,6 +121,9 @@ ReportService.fetchDataAndSaveInFile = (request) => __awaiter(void 0, void 0, vo
     let { nextPage, searchAfter } = request;
     const { storeId, reportName, payload } = args;
     const { exportFileName } = payload;
+    if (!reportInstance) {
+        throw new Error('reportInstance is required for V2 implementation');
+    }
     const firebaseStorageService = new firebaseStorageService_1.FirebaseStorageService();
     const TIME_LIMIT = 25; // 25 sec
     let iterationCount = 0;
@@ -134,10 +134,19 @@ ReportService.fetchDataAndSaveInFile = (request) => __awaiter(void 0, void 0, vo
         // Iterative loop - allows GC to clean up each iteration
         while (true) {
             iterationCount++;
+            // Log memory usage every 10 iterations
+            if (iterationCount % 10 === 0) {
+                const memUsage = process.memoryUsage();
+                console.log(`@fetchDataAndSaveInFile [Iteration ${iterationCount}] Memory:`, {
+                    heapUsed: `${(memUsage.heapUsed / 1024 / 1024).toFixed(2)} MB`,
+                    heapTotal: `${(memUsage.heapTotal / 1024 / 1024).toFixed(2)} MB`,
+                    rss: `${(memUsage.rss / 1024 / 1024).toFixed(2)} MB`,
+                    external: `${(memUsage.external / 1024 / 1024).toFixed(2)} MB`,
+                });
+            }
             // Check file size limit
             [isFileSizeExceeded, fileSize] = yield file_helper_1.FileHelper.isFileSizeExceeded(localTempFile, actionType || 'download');
             if (isFileSizeExceeded) {
-                console.log('@generateReport File size exceeded', fileSize, reportName, storeId);
                 break;
             }
             // Check time limit for view action

package/lib/events/events.d.ts

@@ -36,3 +36,11 @@ export declare function forwardEventsToPod(params: {
     requireEventValidation?: boolean;
     logEvent?: boolean;
 }): ResolvedFunctionResponse<PushEventsResponse>;
+export declare function recordEventsInBulk(rawEvents: Event[], isMigration?: boolean): Promise<import("../apis").FunctionResponse<{
+    bulkResponse: {
+        items: never[];
+        errors: boolean;
+        took: number;
+    };
+    failedIds: never[];
+}> | undefined>;

package/lib/events/events.js

@@ -32,7 +32,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
     });
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.forwardEventsToPod = exports.generateEventId = exports.validateEvent = exports.pushEventsWithoutCustomerId = exports.pushEvents = exports.pushEventWithoutCustomerId = exports.getUpdateEventSource = exports.pushEvent = exports.getUatStoreIds = exports.REQUIRE_EVENT_VALIDATION = void 0;
+exports.recordEventsInBulk = exports.forwardEventsToPod = exports.generateEventId = exports.validateEvent = exports.pushEventsWithoutCustomerId = exports.pushEvents = exports.pushEventWithoutCustomerId = exports.getUpdateEventSource = exports.pushEvent = exports.getUatStoreIds = exports.REQUIRE_EVENT_VALIDATION = void 0;
 const pubsub_listener_1 = require("../pubsub-listener");
 const Events = __importStar(require("@bikdotai/bik-models/events"));
 const string_1 = require("../utils/string");
@@ -42,6 +42,7 @@ const events_forwarding_1 = require("./forwarding/events-forwarding");
 const uuid_1 = require("uuid");
 const events_helper_1 = require("./events.helper");
 const auth_1 = require("../auth");
+const ingestion_1 = require("../elastic/counter/ingestion");
 exports.REQUIRE_EVENT_VALIDATION = true;
 function getUatStoreIds() {
     return (auth_1.EnvVariableHelper.access(auth_1.ServiceName.CONSTANTS, auth_1.ConstantsKeys.UAT_STORE_IDS) || '')
@@ -445,3 +446,26 @@ function sanitizeEventPropertiesAndFixTypes(event, eventValidator) {
     const uniqueFixedErrors = [...new Set(fixedErrors)];
     return { correctedEvent, errors: uniqueErrors, fixedErrors: uniqueFixedErrors };
 }
+function recordEventsInBulk(rawEvents, isMigration) {
+    return __awaiter(this, void 0, void 0, function* () {
+        let events;
+        try {
+            events = (yield new ingestion_1.CounterEventIngestionService().pushCounterEventsAndGiveRest(rawEvents));
+        }
+        catch (e) {
+            console.error('Error in pushCounterEventsAndGiveRest', e, { postOnSlack: true });
+            events = rawEvents;
+        }
+        if (!events || events.length === 0) {
+            return apis_1.FunctionResponseHelper.success({
+                bulkResponse: {
+                    items: [],
+                    errors: false,
+                    took: 0,
+                },
+                failedIds: [],
+            });
+        }
+    });
+}
+exports.recordEventsInBulk = recordEventsInBulk;

package/lib/events/schema/events.helper.d.ts

@@ -21,6 +21,7 @@ export declare class EventsHelper {
     static getHistogramAggregation(startDate: string | Date, endDate: string | Date, timeZone?: string, useExactBounds?: boolean): any;
     static identifyAndConvertDateString(str: string): string;
     static isStaticIndexName(indexName: string): boolean;
+    static isStoreSpecificIndexName(indexName: string): boolean;
     static getIndexOrAliasName(indexName?: string): string;
     static getDataStreamName(eventName: Events): string;
     static getIndexAndInstanceId(storeId: string, bikStore?: BikStore): Promise<GetIndexAndInstanceIdOutput>;

package/lib/events/schema/events.helper.js

@@ -66,9 +66,9 @@ class EventsHelper {
         //   nonAnalyticsDs,
         // });
         return {
-            ANALYTICS_CAMPAIGN_DATA_STREAM: campaignDs || 'prod-campaign-analytics-ds',
-            ANALYTICS_CHATBOT_DATA_STREAM: chatbotDs || 'prod-chatbot-analytics-ds',
-            NON_ANALYTICS_DATA_STREAM: nonAnalyticsDs || 'prod-non-analytics-ds',
+            ANALYTICS_CAMPAIGN_DATA_STREAM: campaignDs || 'staging-campaign-analytics-ds',
+            ANALYTICS_CHATBOT_DATA_STREAM: chatbotDs || 'staging-chatbot-analytics-ds',
+            NON_ANALYTICS_DATA_STREAM: nonAnalyticsDs || 'staging-non-analytics-ds',
         };
     }
     static getIndexNameForStoreId(storeId) {
@@ -144,14 +144,19 @@ class EventsHelper {
         }
         return str;
     }
+    // `store-events-*-ds` data streams for a store are static indices
+    // ex: bik-internal-events, bsp-prod, store-events-group-1-analytics-ds
     static isStaticIndexName(indexName) {
-        return exports.STATIC_INDEX_NAMES.includes(indexName);
+        return exports.STATIC_INDEX_NAMES.includes(indexName) || (indexName === null || indexName === void 0 ? void 0 : indexName.startsWith('store-events-'));
+    }
+    static isStoreSpecificIndexName(indexName) {
+        return indexName.startsWith('store-events-');
     }
     static getIndexOrAliasName(indexName) {
         if (!indexName) {
             return exports.INDEX_ALIAS_NAME; //events-prod-alias
         }
-        if (exports.STATIC_INDEX_NAMES.includes(indexName)) {
+        if (this.isStaticIndexName(indexName)) {
             return indexName; // static index names
         }
         if (indexName.includes('manifest')) {
@@ -178,8 +183,9 @@ class EventsHelper {
             const cacheKey = this.getIndexAndInstanceCacheKey(storeId);
             const centralCacheService = redis_1.CentralCacheService.getInstance();
             const { data: dataFromCache } = yield centralCacheService.retrieveData(cacheKey, {
-                cachesToUse: [types_1.CacheTypes.REDIS],
+                cachesToUse: [types_1.CacheTypes.LRU, types_1.CacheTypes.REDIS],
                 ttlMap: {
+                    [types_1.CacheTypes.LRU]: 60 * 5,
                     [types_1.CacheTypes.REDIS]: 7 * 60 * 60 * 24, // 7 days
                 },
             }, () => __awaiter(this, void 0, void 0, function* () {

package/lib/index.d.ts

@@ -8,6 +8,7 @@ export * from './variable-management';
 export * from './cloudTasks';
 export * from './releaseJet/index';
 export * from './fastify/main';
+export * from './database';
 export * from './alertsV2/alertsV2.service';
 export * from './index-filtering';
 export * from './messaging-transformer/messagingTransformer.service';

package/lib/index.js

@@ -24,6 +24,7 @@ __exportStar(require("./variable-management"), exports);
 __exportStar(require("./cloudTasks"), exports);
 __exportStar(require("./releaseJet/index"), exports);
 __exportStar(require("./fastify/main"), exports);
+__exportStar(require("./database"), exports);
 __exportStar(require("./alertsV2/alertsV2.service"), exports);
 __exportStar(require("./index-filtering"), exports);
 __exportStar(require("./messaging-transformer/messagingTransformer.service"), exports);

package/lib/merchant-events/elastic.search.d.ts

@@ -34,6 +34,10 @@ export declare class ElasticSearchService {
     deleteEventsFromElastic(index: string, documentIds: string[]): Promise<void>;
     bulk(bulkUpdateOperations: any): Promise<boolean>;
     recordCounterEvents(index: string, events: Map<string, CounterDbEvent>): ResolvedFunctionResponse<undefined>;
+    /**
+     * Helper method to check if an index name is a data stream
+     */
+    private isDataStream;
     bulkInsertToES(index: string | undefined, bulkBody: Event[]): ResolvedFunctionResponse<BulkInsertToESResponse>;
     bulkInsertToElastic(index: string, bulkBody: any[]): Promise<{
         success: boolean;
@@ -41,6 +45,7 @@ export declare class ElasticSearchService {
     }>;
     rawSearch(query: any): Promise<any>;
     count(query: any): Promise<CountResponse>;
+    private getDataStreamNameFromQuery;
     /**
      *
      * @param podName

package/lib/merchant-events/elastic.search.js

@@ -310,6 +310,12 @@ class ElasticSearchService {
             }
         });
     }
+    /**
+     * Helper method to check if an index name is a data stream
+     */
+    isDataStream(indexName) {
+        return indexName.includes('analytics-ds') || indexName.startsWith('.ds-');
+    }
     bulkInsertToES(index, bulkBody) {
         var _a, _b, _c;
         return __awaiter(this, void 0, void 0, function* () {
@@ -323,6 +329,7 @@ class ElasticSearchService {
                 const eventIdsForCreateOrUpdate = [];
                 const idToIndexMap = new Map();
                 const isDataStreamTarget = events_helper_1.EventsHelper.isDataStream(index);
+                const isStoreSpecificIndexName = events_helper_1.EventsHelper.isStoreSpecificIndexName(index);
                 for (let data of bulkBody) {
                     if ((data === null || data === void 0 ? void 0 : data.isMigrationEvent) && (data === null || data === void 0 ? void 0 : data.eventId)) {
                         // events to be updated (only for non-data-stream targets)
@@ -348,9 +355,13 @@ class ElasticSearchService {
                 }
                 if (eventIdsForCreateOrUpdate.length > 0) {
                     // for migration events we need to find the exact index where the event is stored from the alias and non-analytics data stream
+                    // for store specific index names, we only search in that index
                     const { NON_ANALYTICS_DATA_STREAM } = events_helper_1.EventsHelper.getDataStreamNamesFromEnv();
+                    const indexToSearch = isStoreSpecificIndexName
+                        ? index
+                        : [events_helper_1.INDEX_ALIAS_NAME, NON_ANALYTICS_DATA_STREAM];
                     const res = yield this.esClient.search({
-                        index: [events_helper_1.INDEX_ALIAS_NAME, NON_ANALYTICS_DATA_STREAM],
+                        index: indexToSearch,
                         _source: ['eventId'],
                         track_scores: false,
                         body: {
@@ -403,7 +414,7 @@ class ElasticSearchService {
                             });
                         }
                         else {
-                            // create a new document
+                            // new migration events will also go to the old alias
                             const _id = (data === null || data === void 0 ? void 0 : data.eventId) || (0, uuid_1.v4)();
                             documents.push({ index: { _index: events_helper_1.INDEX_ALIAS_NAME, _id } });
                             documents.push(data);
@@ -424,6 +435,7 @@ class ElasticSearchService {
                             });
                         }
                         else {
+                            // new migration events will also go to the old alias
                             const _id = (data === null || data === void 0 ? void 0 : data.eventId) || (0, uuid_1.v4)();
                             documents.push({ index: { _index: events_helper_1.INDEX_ALIAS_NAME, _id } });
                             documents.push(data);
@@ -484,6 +496,7 @@ class ElasticSearchService {
             return currOutput;
         });
     }
+    getDataStreamNameFromQuery() { }
     /**
      *
      * @param podName
@@ -507,6 +520,14 @@ class ElasticSearchService {
             var _d;
             logger_1.LogExecutionContextHelper.setCustomProperty('queryName', queryName);
             try {
+                //TODO: remove after testing
+                const info = yield this.esClient.info();
+                console.log('esClient info', {
+                    clusterName: info.cluster_name,
+                    clusterUuid: info.cluster_uuid,
+                    version: info.version.number,
+                    storeId: queryParams.storeId,
+                });
                 const executionStartTime = Date.now();
                 if (!queryParams.storeId) {
                     return apis_1.FunctionResponseHelper.failed(apis_1.ErrorCodes.VALIDATION_FAILED, 'storeId not present in queryParams');

package/lib/merchant-events/merchant.service.d.ts

@@ -9,7 +9,7 @@ export declare class BikInternalEventsService {
     static getInstance(): BikInternalEventsService;
     private static getContextFromSession;
     static logFunctionResponse(storeId: string, code: string, response: FunctionResponse<any>): Promise<FunctionResponse<undefined>>;
-    static preparePayloadForLogging(bikStore: BikStore, bikEvent: BikInternalEvent, skipCommonEvents?: boolean): BikInternalEvent | undefined;
+    static preparePayloadForLogging(bikStore: BikStore, bikEvent: BikInternalEvent, skipCommonEvents?: boolean): BikInternalEvent;
     static recordEventInMailModo(bikStore: BikStore, bikEvent: BikInternalEvent): ResolvedFunctionResponse<undefined>;
     static recordEventInSlack(bikStore: BikStore, bikEvent: BikInternalEvent, slackChannelId: string, messageFormattingConfig?: MessageFormattingConfig): ResolvedFunctionResponse<undefined>;
     static recordEventInElastic(bikStore: BikStore, bikEvent: BikInternalEvent, skipCommonEvents?: boolean): ResolvedFunctionResponse<undefined>;

package/lib/merchant-events/merchant.service.js

@@ -60,7 +60,7 @@ class BikInternalEventsService {
         });
     }
     static preparePayloadForLogging(bikStore, bikEvent, skipCommonEvents) {
-        var _a, _b, _c, _d, _e, _f, _g, _h, _j;
+        var _a, _b, _c, _d, _e, _f, _g, _h;
         const testEmail = [
             'dev@bikayi.com',
             'dev@bik.ai',
@@ -74,18 +74,19 @@ class BikInternalEventsService {
             'shrinkhla@bik.ai',
             'sumit.kumar1@bik.ai',
         ];
-        if (testEmail.includes((_a = bikStore === null || bikStore === void 0 ? void 0 : bikStore.bikLoginEmail) !== null && _a !== void 0 ? _a : '') || testEmail.includes(bikStore === null || bikStore === void 0 ? void 0 : bikStore.email)) {
-            // not logging events for internal test stores
-            return undefined;
-        }
+        // Removed Skip recording events for test users for testing
+        // if (testEmail.includes(bikStore?.bikLoginEmail ?? '') || testEmail.includes(bikStore?.email)) {
+        //   // not logging events for internal test stores
+        //   return undefined;
+        // }
         const commonEventsForBik = {
             'Store name': bikStore.storeName,
-            'Store type': ((_b = bikStore === null || bikStore === void 0 ? void 0 : bikStore.shopifyStats) === null || _b === void 0 ? void 0 : _b.isBigStore) ? 'Big' : 'Small',
-            Country: (_c = bikStore === null || bikStore === void 0 ? void 0 : bikStore.countryCode) !== null && _c !== void 0 ? _c : 'INR',
-            'Plan name': (_e = (_d = bikStore === null || bikStore === void 0 ? void 0 : bikStore.packageInformation) === null || _d === void 0 ? void 0 : _d.planType) !== null && _e !== void 0 ? _e : 'NA',
-            'Shop Id': (_g = (_f = bikStore === null || bikStore === void 0 ? void 0 : bikStore.partnerStore) === null || _f === void 0 ? void 0 : _f.id) !== null && _g !== void 0 ? _g : 'NA',
-            'Free trial': ((_h = bikStore === null || bikStore === void 0 ? void 0 : bikStore.packageInformation) === null || _h === void 0 ? void 0 : _h.isFreeTrial)
-                ? (_j = bikStore === null || bikStore === void 0 ? void 0 : bikStore.packageInformation) === null || _j === void 0 ? void 0 : _j.isFreeTrial
+            'Store type': ((_a = bikStore === null || bikStore === void 0 ? void 0 : bikStore.shopifyStats) === null || _a === void 0 ? void 0 : _a.isBigStore) ? 'Big' : 'Small',
+            Country: (_b = bikStore === null || bikStore === void 0 ? void 0 : bikStore.countryCode) !== null && _b !== void 0 ? _b : 'INR',
+            'Plan name': (_d = (_c = bikStore === null || bikStore === void 0 ? void 0 : bikStore.packageInformation) === null || _c === void 0 ? void 0 : _c.planType) !== null && _d !== void 0 ? _d : 'NA',
+            'Shop Id': (_f = (_e = bikStore === null || bikStore === void 0 ? void 0 : bikStore.partnerStore) === null || _e === void 0 ? void 0 : _e.id) !== null && _f !== void 0 ? _f : 'NA',
+            'Free trial': ((_g = bikStore === null || bikStore === void 0 ? void 0 : bikStore.packageInformation) === null || _g === void 0 ? void 0 : _g.isFreeTrial)
+                ? (_h = bikStore === null || bikStore === void 0 ? void 0 : bikStore.packageInformation) === null || _h === void 0 ? void 0 : _h.isFreeTrial
                 : false,
         };
         const bikMainEvent = bikStore.appType !== growth_1.AppType.GLOBAL && !skipCommonEvents

package/lib/recordAnalytics/recordAnalytics.service.js

@@ -25,10 +25,11 @@ class RecordAnalyticsService {
             if (!payload.appType) {
                 return apis_1.FunctionResponseHelper.failed(apis_1.ErrorCodes.MISSING_PARAMETERS, 'appType is missing');
             }
-            const isTestUser = this.checkForTestUsers(bikStore.bikLoginEmail || bikStore.email);
-            if (isTestUser) {
-                return apis_1.FunctionResponseHelper.success(true);
-            }
+            // Removed Skip recording events for test users for testing
+            // const isTestUser = this.checkForTestUsers(bikStore.bikLoginEmail || bikStore.email);
+            // if (isTestUser) {
+            //   return FunctionResponseHelper.success(true);
+            // }
             for (const destination of destinations) {
                 switch (destination) {
                     case recordAnalytics_model_1.ServiceType.AMPLITUDE:

package/lib/redis/redisPubSubService.d.ts

@@ -2,16 +2,14 @@ interface RedisConfig {
     channel: string;
 }
 export declare class RedisPubSubService {
-    private publisher;
-    private subscriber;
     private channel;
-    private readonly maxRetries;
-    private readonly baseDelayMs;
+    private subscribed;
     constructor(config: RedisConfig);
-    private isRetriableRedisError;
-    private retryWithBackoff;
+    private get publisher();
+    private get subscriber();
     publish<T>(data: T): Promise<void>;
     subscribe<T>(callback: (message: T) => void): void;
     close(): Promise<void>;
+    static closeAll(): Promise<void>;
 }
 export {};