enterprise-logging-system 1.0.55 → 1.1.1

package/dist/backend/repositories/AccessLogRepository.js

@@ -12,320 +12,182 @@ class AccessLogRepository extends BaseRepository_1.BaseRepository {
     /**
      * Ensure unique index exists for userId + tenantId + date
      * This prevents race conditions from creating duplicate entries
-     * NOTE: This index is only for aggregated daily summaries, not individual activities
      */
     async ensureUniqueIndex() {
         try {
-            // We DON'T want a unique index on individual activities
-            // Individual activities should allow multiple entries per day
-            // Only aggregated summaries should be unique
-            // Drop the unique index if it exists
-            try {
-                await this.collection.dropIndex('userId_1_tenantId_1_indexes.session_date_1');
-                console.log('✅ Dropped unique index - allowing multiple activities per day');
-            }
-            catch (error) {
-                // Index might not exist, that's fine
-            }
-            // Create a non-unique index for better query performance
             await this.collection.createIndex({
                 userId: 1,
                 tenantId: 1,
-                timestamp: -1
+                'indexes.session_date': 1
             }, {
-                background: true,
-                name: 'userId_tenantId_timestamp_idx'
+                unique: true,
+                partialFilterExpression: {
+                    'indexes.session_date': { $exists: true }
+                },
+                background: true
             });
-            console.log('✅ Created non-unique index for queries');
         }
         catch (error) {
-            console.log('Index creation info:', error.message);
+            // Index might already exist, ignore error
+            console.log('Unique index creation skipped (may already exist)');
         }
     }
     /**
-     * Get grouped access logs - one entry per user per day with aggregated data
+     * Log a page view. Implements upsert logic: if an access log entry already exists
+     * for this userId on the same day, it updates that record instead of creating
+     * a new one. This ensures one access log entry per user per day.
+     * Uses atomic operations to prevent race conditions.
      */
-    async listGroupedByUser(query) {
-        const { page = 1, limit = 20, search, tenantId, userId, startDate, endDate, sortBy = 'date', sortOrder = 'desc' } = query;
-        const skip = (page - 1) * limit;
-        const matchStage = {};
-        // Apply filters
-        if (tenantId)
-            matchStage.tenantId = tenantId;
-        if (userId)
-            matchStage.userId = userId;
-        if (startDate || endDate) {
-            matchStage.timestamp = {};
-            if (startDate)
-                matchStage.timestamp.$gte = new Date(startDate);
-            if (endDate)
-                matchStage.timestamp.$lte = new Date(endDate);
-        }
-        if (search) {
-            matchStage.$or = [
-                { username: { $regex: search, $options: 'i' } },
-                { employeeId: { $regex: search, $options: 'i' } },
-                { pageTitle: { $regex: search, $options: 'i' } },
-                { activityName: { $regex: search, $options: 'i' } }
-            ];
-        }
-        console.log('🔍 Match Stage:', JSON.stringify(matchStage, null, 2));
-        // First, let's check if we have any documents at all
-        const totalDocs = await this.collection.countDocuments({});
-        console.log('📊 Total documents in collection:', totalDocs);
-        // Check documents matching our filter
-        const matchingDocs = await this.collection.countDocuments(matchStage);
-        console.log('🎯 Documents matching filter:', matchingDocs);
-        const pipeline = [
-            { $match: matchStage },
-            {
-                $group: {
-                    _id: {
-                        userId: '$userId',
-                        date: {
-                            $dateToString: {
-                                format: '%Y-%m-%d',
-                                date: '$timestamp'
-                            }
-                        }
-                    },
-                    // User info (take first occurrence)
-                    username: { $first: '$username' },
-                    employeeId: { $first: '$employeeId' },
-                    userRole: { $first: '$userRole' },
-                    tenantId: { $first: '$tenantId' },
-                    // Session info
-                    sessionId: { $first: '$sessionId' },
-                    ipAddress: { $first: '$ipAddress' },
-                    browser: { $first: '$browser' },
-                    // Time tracking
-                    startTime: { $min: '$timestamp' },
-                    endTime: { $max: '$timestamp' },
-                    // Activity counts
-                    totalActivities: { $sum: 1 },
-                    totalPageViews: {
-                        $sum: { $cond: [{ $eq: ['$type', 'PAGE_OPEN'] }, 1, 0] }
-                    },
-                    totalActions: {
-                        $sum: { $cond: [{ $eq: ['$type', 'ACTION_PERFORMED'] }, 1, 0] }
-                    },
-                    // Collect all activities for detailed view
-                    activities: {
-                        $push: {
-                            _id: '$_id',
-                            timestamp: '$timestamp',
-                            type: '$type',
-                            activityType: '$activityType',
-                            activityName: '$activityName',
-                            pageTitle: '$pageTitle',
-                            pageUrl: '$pageUrl',
-                            actionType: '$actionType',
-                            actionTarget: '$actionTarget',
-                            actionData: '$actionData',
-                            activeDuration: '$activeDuration'
-                        }
-                    }
+    async logPageView(log) {
+        // Get start and end of the day for the current timestamp
+        const currentDate = new Date(log.timestamp);
+        // Create a unique daily key for this user (user_date format)
+        const dateKey = `${currentDate.getFullYear()}-${String(currentDate.getMonth() + 1).padStart(2, '0')}-${String(currentDate.getDate()).padStart(2, '0')}`;
+        const dailyKey = `${log.userId}_${dateKey}`;
+        // Use atomic findOneAndUpdate with upsert to prevent race conditions
+        // Filter by user_date index to ensure one entry per user per day
+        const filter = {
+            userId: log.userId,
+            tenantId: log.tenantId,
+            'indexes.user_date': dailyKey
+        };
+        const update = {
+            $set: {
+                // Always update these fields to latest values
+                timestamp: log.timestamp,
+                sessionId: log.sessionId,
+                pageId: log.pageId,
+                pageTitle: log.pageTitle,
+                pageUrl: log.pageUrl,
+                pageRoute: log.pageRoute,
+                previousPage: log.previousPage,
+                endTime: log.timestamp,
+                browser: log.browser,
+                geoLocation: log.geoLocation,
+                network: log.network,
+                ipAddress: log.ipAddress,
+                userAgent: log.userAgent,
+                activityType: log.activityType,
+                activityName: log.activityName,
+                ...(log.metadata && { metadata: log.metadata }),
+                indexes: {
+                    tenant_user_date: `${log.tenantId}_${log.userId}_${dateKey}`,
+                    session_time: `${log.sessionId}_${log.activityType}`,
+                    page_time: `${log.pageId}_${log.timestamp.getTime()}`,
+                    session_date: dailyKey
                 }
             },
-            {
-                $project: {
-                    _id: { $concat: ['$_id.userId', '_', '$_id.date'] },
-                    userId: '$_id.userId',
-                    username: 1,
-                    employeeId: 1,
-                    userRole: 1,
-                    tenantId: 1,
-                    sessionId: 1,
-                    ipAddress: 1,
-                    browser: 1,
-                    startTime: 1,
-                    endTime: 1,
-                    duration: {
-                        $subtract: ['$endTime', '$startTime']
-                    },
-                    totalActivities: 1,
-                    totalPageViews: 1,
-                    totalActions: 1,
-                    activities: 1,
-                    // For display compatibility
-                    timestamp: '$startTime',
-                    activityType: 'DAILY_SUMMARY',
-                    activityName: {
-                        $concat: [
-                            { $toString: '$totalActivities' },
-                            ' activities (',
-                            { $toString: '$totalPageViews' },
-                            ' page views, ',
-                            { $toString: '$totalActions' },
-                            ' actions)'
-                        ]
-                    }
-                }
-            }
-        ];
-        // Add sorting
-        const sortField = sortBy === 'date' ? 'startTime' : sortBy;
-        const sortDirection = sortOrder === 'asc' ? 1 : -1;
-        pipeline.push({ $sort: { [sortField]: sortDirection } });
-        console.log('🔧 Aggregation Pipeline:', JSON.stringify(pipeline, null, 2));
-        // Get total count
-        const countPipeline = [...pipeline, { $count: 'total' }];
-        const countResult = await this.collection.aggregate(countPipeline).toArray();
-        const total = countResult[0]?.total || 0;
-        console.log('📈 Total after aggregation:', total);
-        // Add pagination
-        pipeline.push({ $skip: skip }, { $limit: limit });
-        const data = await this.collection.aggregate(pipeline).toArray();
-        console.log('📋 Final result count:', data.length);
-        console.log('📋 Sample data:', data.slice(0, 2));
-        return {
-            data,
-            total,
-            page,
-            limit,
-            totalPages: Math.ceil(total / limit),
-            hasNext: page < Math.ceil(total / limit),
-            hasPrev: page > 1
-        };
-    }
-    /**
-     * Get detailed activities for a specific user and date
-     */
-    async getUserDayActivities(userId, date, tenantId) {
-        const startOfDay = new Date(date);
-        startOfDay.setHours(0, 0, 0, 0);
-        const endOfDay = new Date(date);
-        endOfDay.setHours(23, 59, 59, 999);
-        const filter = {
-            userId,
-            timestamp: {
-                $gte: startOfDay,
-                $lte: endOfDay
+            $setOnInsert: {
+                // Only set these on first insert
+                _id: this.generateId(),
+                type: 'PAGE_OPEN',
+                tenantId: log.tenantId,
+                userId: log.userId,
+                userRole: log.userRole,
+                username: log.username,
+                employeeId: log.employeeId,
+                eventTime: log.eventTime,
+                startTime: log.startTime || log.timestamp
+            },
+            $inc: {
+                // Increment activity count atomically (starts at 1 on first insert)
+                activityCount: 1
             }
         };
-        if (tenantId) {
-            filter.tenantId = tenantId;
-        }
-        return this.collection
-            .find(filter)
-            .sort({ timestamp: 1 })
-            .toArray();
-    }
-    /**
-     * Get ALL individual activities for a user (every click, every page view)
-     * This returns detailed activity log for comprehensive user tracking
-     */
-    async getUserAllActivities(userId, startDate, endDate, tenantId) {
-        const filter = { userId };
-        if (startDate || endDate) {
-            filter.timestamp = {};
-            if (startDate) {
-                const start = new Date(startDate);
-                start.setHours(0, 0, 0, 0);
-                filter.timestamp.$gte = start;
-            }
-            if (endDate) {
-                const end = new Date(endDate);
-                end.setHours(23, 59, 59, 999);
-                filter.timestamp.$lte = end;
-            }
+        const result = await this.collection.findOneAndUpdate(filter, update, {
+            upsert: true,
+            returnDocument: 'after'
+        });
+        if (result.value) {
+            // Calculate duration
+            const startTime = new Date(result.value.startTime || result.value.timestamp);
+            result.value.activeDuration = log.timestamp.getTime() - startTime.getTime();
+            return result.value;
         }
-        if (tenantId) {
-            filter.tenantId = tenantId;
+        // Fallback: if upsert somehow failed, try to find the existing record
+        const existingLog = await this.findOne(filter);
+        if (existingLog) {
+            return existingLog;
         }
-        // Get all individual activities (not aggregated ones)
-        return this.collection
-            .find(filter)
-            .sort({ timestamp: -1 }) // Most recent first
-            .toArray();
+        // Last resort: This should never happen with upsert, but handle it
+        throw new Error('Failed to create or update access log entry');
     }
     /**
-     * Log a page view as individual entry (not aggregated)
-     * This creates a separate log entry for each page view to track detailed user activity
+     * Log a user action. Implements upsert logic: if an access log entry already exists
+     * for this userId on the same day, it updates that record instead of creating
+     * a new one. This ensures one access log entry per user per day.
+     * Uses atomic operations to prevent race conditions.
      */
-    async logIndividualPageView(log) {
+    async logAction(log) {
+        // Get start and end of the day for the current timestamp
         const currentDate = new Date(log.timestamp);
+        // Create a unique daily key for this user
         const dateKey = `${currentDate.getFullYear()}-${String(currentDate.getMonth() + 1).padStart(2, '0')}-${String(currentDate.getDate()).padStart(2, '0')}`;
-        // Create individual entry with unique ID - NO UPSERT, just insert
-        const individualLog = {
-            _id: this.generateId(),
-            type: 'PAGE_OPEN',
-            tenantId: log.tenantId,
+        const dailyKey = `${log.userId}_${dateKey}`;
+        // Use atomic findOneAndUpdate with upsert - same filter as logPageView
+        const filter = {
             userId: log.userId,
-            userRole: log.userRole,
-            username: log.username,
-            employeeId: log.employeeId,
-            sessionId: log.sessionId,
-            ipAddress: log.ipAddress,
-            userAgent: log.userAgent,
-            timestamp: log.timestamp,
-            eventTime: log.eventTime,
-            startTime: log.startTime || log.timestamp,
-            activityType: log.activityType,
-            activityName: log.activityName,
-            pageId: log.pageId,
-            pageTitle: log.pageTitle,
-            pageUrl: log.pageUrl,
-            pageRoute: log.pageRoute,
-            previousPage: log.previousPage,
-            browser: log.browser,
-            geoLocation: log.geoLocation,
-            network: log.network,
-            metadata: log.metadata,
-            indexes: {
-                tenant_user_date: `${log.tenantId}_${log.userId}_${dateKey}`,
-                session_time: `${log.sessionId}_${log.activityType}`,
-                page_time: `${log.pageId}_${log.timestamp.getTime()}`,
-                session_date: `${log.userId}_${dateKey}`
-            }
-        };
-        await this.collection.insertOne(individualLog);
-        return individualLog;
-    }
-    /**
-     * Log a user action as individual entry (not aggregated)
-     * This creates a separate log entry for each action to track detailed user activity
-     */
-    async logIndividualAction(log) {
-        const currentDate = new Date(log.timestamp);
-        const dateKey = `${currentDate.getFullYear()}-${String(currentDate.getMonth() + 1).padStart(2, '0')}-${String(currentDate.getDate()).padStart(2, '0')}`;
-        // Create individual entry with unique ID - NO UPSERT, just insert
-        const individualLog = {
-            _id: this.generateId(),
-            type: 'ACTION_PERFORMED',
             tenantId: log.tenantId,
-            userId: log.userId,
-            userRole: log.userRole,
-            username: log.username,
-            employeeId: log.employeeId,
-            sessionId: log.sessionId,
-            ipAddress: log.ipAddress,
-            userAgent: log.userAgent,
-            timestamp: log.timestamp,
-            eventTime: log.eventTime,
-            startTime: log.startTime || log.timestamp,
-            activityType: log.activityType,
-            activityName: log.activityName,
-            actionType: log.actionType,
-            actionTarget: log.actionTarget,
-            actionData: log.actionData,
-            pageId: log.pageId,
-            browser: log.browser,
-            metadata: log.metadata,
-            indexes: {
-                tenant_user_date: `${log.tenantId}_${log.userId}_${dateKey}`,
-                session_time: `${log.sessionId}_${log.activityType}`,
-                page_time: `${log.pageId}_${log.timestamp.getTime()}`,
-                session_date: `${log.userId}_${dateKey}`
+            'indexes.session_date': dailyKey
+        };
+        const update = {
+            $set: {
+                // Always update these fields to latest values
+                timestamp: log.timestamp,
+                sessionId: log.sessionId,
+                activityType: log.activityType,
+                activityName: log.activityName,
+                actionType: log.actionType,
+                actionTarget: log.actionTarget,
+                actionData: log.actionData,
+                endTime: log.timestamp,
+                browser: log.browser,
+                ipAddress: log.ipAddress,
+                userAgent: log.userAgent,
+                ...(log.pageId && { pageId: log.pageId }),
+                ...(log.metadata && { metadata: log.metadata }),
+                indexes: {
+                    tenant_user_date: `${log.tenantId}_${log.userId}_${dateKey}`,
+                    session_time: `${log.sessionId}_${log.activityType}`,
+                    page_time: `${log.pageId}_${log.timestamp.getTime()}`,
+                    session_date: dailyKey
+                }
+            },
+            $setOnInsert: {
+                // Only set these on first insert
+                _id: this.generateId(),
+                type: 'ACTION_PERFORMED',
+                tenantId: log.tenantId,
+                userId: log.userId,
+                userRole: log.userRole,
+                username: log.username,
+                employeeId: log.employeeId,
+                eventTime: log.eventTime,
+                startTime: log.startTime || log.timestamp
+            },
+            $inc: {
+                // Track how many actions this user performed today (starts at 1 on first insert)
+                actionCount: 1
             }
         };
-        await this.collection.insertOne(individualLog);
-        return individualLog;
+        const result = await this.collection.findOneAndUpdate(filter, update, {
+            upsert: true,
+            returnDocument: 'after'
+        });
+        if (result.value) {
+            // Calculate duration for existing entries
+            const startTime = new Date(result.value.startTime || result.value.timestamp);
+            result.value.activeDuration = log.timestamp.getTime() - startTime.getTime();
+            return result.value;
+        }
+        // Fallback: if upsert failed, try to find the existing record
+        const existingLog = await this.findOne(filter);
+        if (existingLog) {
+            return existingLog;
+        }
+        // Last resort: This should never happen with upsert, but handle it
+        throw new Error('Failed to create or update access log entry');
     }
-    // OLD METHODS REMOVED - Use logIndividualPageView and logIndividualAction instead
-    // These old methods used upsert logic with unique indexes which caused E11000 errors
-    // when users revisited pages on the same day. The new approach stores each activity
-    // as a separate document and uses aggregation to show grouped views.
     async getUserTimeline(userId, dateRange) {
         return this.find({
             userId,
@@ -338,6 +200,7 @@ class AccessLogRepository extends BaseRepository_1.BaseRepository {
     /**
      * List access logs with filters, search, and pagination
      * Supports grouping by user per day to show one entry per user per day
+     * Excludes heavy fields like images, large metadata, etc. from listing
      */
     async list(query) {
         const filter = {};
@@ -369,9 +232,9 @@ class AccessLogRepository extends BaseRepository_1.BaseRepository {
                 { pageRoute: searchRegex }
             ];
         }
-        // If groupByUser is true, use the public listGroupedByUser method
+        // If groupByUser is true, use aggregation to get one entry per user
         if (query.groupByUser) {
-            return this.listGroupedByUser(query);
+            return this.listGroupedByUser(filter, query);
         }
         // Default behavior: list all entries
         const page = Math.max(1, query.page ?? 1);
@@ -381,7 +244,22 @@ class AccessLogRepository extends BaseRepository_1.BaseRepository {
             : 'timestamp';
         const sortOrder = query.sortOrder === 'asc' ? 1 : -1;
         const sort = { [sortKey]: sortOrder };
-        const result = await this.paginate(filter, page, limit, sort);
+        // Projection to include only essential fields for listing (more efficient than exclusion)
+        const projection = {
+            _id: 1,
+            username: 1,
+            employeeId: 1,
+            ipAddress: 1,
+            browser: 1,
+            startTime: 1,
+            endTime: 1,
+            timestamp: 1,
+            activeDuration: 1,
+            type: 1,
+            activityType: 1,
+            activityName: 1
+        };
+        const result = await this.paginate(filter, page, limit, sort, projection);
         return {
             data: result.data,
             total: result.total,
@@ -392,6 +270,107 @@ class AccessLogRepository extends BaseRepository_1.BaseRepository {
             hasPrev: result.page > 1
         };
     }
+    /**
+     * List access logs grouped by user per day - shows one entry per user per day with aggregated data
+     */
+    async listGroupedByUser(filter, query) {
+        const page = Math.max(1, query.page ?? 1);
+        const limit = Math.min(100, Math.max(1, query.limit ?? 20));
+        const skip = (page - 1) * limit;
+        // Build aggregation pipeline
+        const pipeline = [
+            { $match: filter },
+            {
+                $addFields: {
+                    // Extract date from timestamp for grouping
+                    dateOnly: {
+                        $dateToString: { format: '%Y-%m-%d', date: '$timestamp' }
+                    }
+                }
+            },
+            {
+                $sort: { timestamp: -1 } // Sort by latest timestamp first
+            },
+            {
+                $group: {
+                    _id: {
+                        userId: '$userId',
+                        date: '$dateOnly' // Group by both userId AND date
+                    },
+                    // Take the latest entry for each user per day
+                    latestLog: { $first: '$$ROOT' },
+                    // Aggregate statistics for this user on this day
+                    totalActivities: { $sum: '$activityCount' },
+                    totalActions: { $sum: '$actionCount' },
+                    firstActivity: { $min: '$startTime' },
+                    lastActivity: { $max: '$timestamp' },
+                    totalSessions: { $addToSet: '$sessionId' }
+                }
+            },
+            {
+                $replaceRoot: {
+                    newRoot: {
+                        $mergeObjects: [
+                            '$latestLog',
+                            {
+                                // Add aggregated fields for this day
+                                totalActivities: '$totalActivities',
+                                totalActions: '$totalActions',
+                                firstActivity: '$firstActivity',
+                                lastActivity: '$lastActivity',
+                                sessionCount: { $size: '$totalSessions' }
+                            }
+                        ]
+                    }
+                }
+            },
+            // Include only essential fields for listing
+            {
+                $project: {
+                    _id: 1,
+                    username: 1,
+                    employeeId: 1,
+                    ipAddress: 1,
+                    browser: 1,
+                    startTime: 1,
+                    endTime: 1,
+                    timestamp: 1,
+                    activeDuration: 1,
+                    type: 1,
+                    activityType: 1,
+                    activityName: 1,
+                    // Include aggregated fields
+                    totalActivities: 1,
+                    totalActions: 1,
+                    firstActivity: 1,
+                    lastActivity: 1,
+                    sessionCount: 1
+                }
+            },
+            {
+                $sort: { timestamp: -1 } // Sort final results by latest activity
+            }
+        ];
+        // Get total count of unique user-day combinations (before projection)
+        const countPipeline = [...pipeline.slice(0, 4), { $count: 'total' }];
+        const countResult = await this.collection.aggregate(countPipeline).toArray();
+        const total = countResult.length > 0 ? countResult[0].total : 0;
+        // Add pagination
+        pipeline.push({ $skip: skip });
+        pipeline.push({ $limit: limit });
+        // Execute aggregation
+        const data = await this.collection.aggregate(pipeline).toArray();
+        const totalPages = Math.ceil(total / limit);
+        return {
+            data,
+            total,
+            page,
+            limit,
+            totalPages,
+            hasNext: page < totalPages,
+            hasPrev: page > 1
+        };
+    }
     async updateActivityDuration(logId, endTime) {
         const log = await this.findById(logId);
         if (!log)
@@ -403,7 +382,7 @@ class AccessLogRepository extends BaseRepository_1.BaseRepository {
         });
     }
     generateId() {
-        return `acc_${Date.now()}_${Math.random().toString(36).substring(2, 11)}`;
+        return `acc_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
     }
 }
 exports.AccessLogRepository = AccessLogRepository;