@resolveio/server-lib 22.3.219 → 22.3.221

package/src/methods/logs.ts

@@ -0,0 +1,657 @@
+import axios from 'axios';
+import { URL } from 'url';
+import { LogMethodLatencies } from '../collections/log-method-latency.collection';
+import { LogSubscriptions } from '../collections/log-subscription.collection';
+import { Logs } from '../collections/log.collection';
+import { MethodManager } from '../managers/method.manager';
+import { ResolveIOServer } from '../resolveio-server-app';
+import { objectIdHexString } from '../util/common';
+import { ErrorReporter } from '../util/error-reporter';
+import { getCorrelationId } from '../util/error-tracking';
+
+type LogsSourceMode = 'auto' | 'local' | 'remote' | 'resolveio' | 'aicoder';
+
+const DEFAULT_RESOLVEIO_LOGS_ENDPOINT = 'https://backend.resolveio.com/api/logs/query';
+const MAX_LOG_QUERY_LIMIT = 5000;
+const DEFAULT_LOG_QUERY_LIMIT = 250;
+const REMOTE_LOG_QUERY_TIMEOUT_MS = 20000;
+
+function normalizeString(value: any): string {
+	if (value === undefined || value === null) {
+		return '';
+	}
+	return String(value).trim();
+}
+
+function normalizeLogsSourceMode(value: any): LogsSourceMode {
+	const mode = normalizeString(value).toLowerCase();
+	if (mode === 'local' || mode === 'remote' || mode === 'resolveio' || mode === 'aicoder' || mode === 'auto') {
+		return mode;
+	}
+	return 'auto';
+}
+
+function toSafeInteger(value: any, fallback: number): number {
+	const parsed = Number(value);
+	if (!Number.isFinite(parsed)) {
+		return fallback;
+	}
+	return Math.floor(parsed);
+}
+
+function looksLikeResolveIOHost(serverConfig: Record<string, any>): boolean {
+	const candidates = [
+		serverConfig?.ROOT_URL,
+		serverConfig?.SERVER_URL
+	];
+	for (const candidate of candidates) {
+		const url = normalizeString(candidate);
+		if (!url) {
+			continue;
+		}
+		try {
+			const parsed = new URL(url.includes('://') ? url : `https://${url}`);
+			const host = normalizeString(parsed.hostname).toLowerCase();
+			if (host === 'backend.resolveio.com' || host === 'resolveio.com' || host.endsWith('.resolveio.com')) {
+				return true;
+			}
+		}
+		catch {
+			const host = url.toLowerCase();
+			if (host.includes('backend.resolveio.com') || host.includes('resolveio.com')) {
+				return true;
+			}
+		}
+	}
+	return false;
+}
+
+function resolveLogsSourceMode(serverConfig: Record<string, any>, forcedMode?: any): LogsSourceMode {
+	const forced = normalizeLogsSourceMode(forcedMode);
+	if (forced !== 'auto') {
+		return forced;
+	}
+	const configured = normalizeLogsSourceMode(
+		process.env.LOGS_DATA_MODE
+		|| process.env.RESOLVEIO_LOGS_MODE
+		|| serverConfig?.LOGS_DATA_MODE
+		|| serverConfig?.RESOLVEIO_LOGS_MODE
+		|| serverConfig?.AI_ASSISTANT_MODE
+		|| process.env.AI_ASSISTANT_MODE
+	);
+	if (configured !== 'auto') {
+		return configured;
+	}
+	return 'resolveio';
+}
+
+function shouldUseRemoteLogsSource(serverConfig: Record<string, any>, forcedMode?: any): boolean {
+	const mode = resolveLogsSourceMode(serverConfig, forcedMode);
+	if (mode === 'remote') {
+		return true;
+	}
+	if (mode === 'local' || mode === 'aicoder') {
+		return false;
+	}
+	if (mode === 'resolveio') {
+		return !looksLikeResolveIOHost(serverConfig);
+	}
+	return false;
+}
+
+function normalizeLogQueryPayload(payload: any): {
+	query: Record<string, any>;
+	options: Record<string, any>;
+	includeCount: boolean;
+	sourceMode: LogsSourceMode;
+} {
+	const raw = (payload && typeof payload === 'object') ? payload : {};
+	const query = (raw.query && typeof raw.query === 'object') ? raw.query : {};
+	const optionsRaw = (raw.options && typeof raw.options === 'object') ? raw.options : {};
+	const limit = Math.max(1, Math.min(MAX_LOG_QUERY_LIMIT, toSafeInteger(optionsRaw.limit, DEFAULT_LOG_QUERY_LIMIT)));
+	const skip = Math.max(0, toSafeInteger(optionsRaw.skip, 0));
+	const sort = (optionsRaw.sort && typeof optionsRaw.sort === 'object') ? optionsRaw.sort : { createdAt: -1 };
+	const projection = (optionsRaw.projection && typeof optionsRaw.projection === 'object')
+		? optionsRaw.projection
+		: ((optionsRaw.fields && typeof optionsRaw.fields === 'object') ? optionsRaw.fields : undefined);
+	const includeCount = raw.include_count !== false && raw.includeCount !== false;
+	const sourceMode = normalizeLogsSourceMode(raw.force_source || raw.source_mode || raw.mode);
+
+	return {
+		query,
+		options: {
+			sort,
+			limit,
+			skip,
+			...(projection ? { projection } : {})
+		},
+		includeCount,
+		sourceMode
+	};
+}
+
+function resolveRemoteLogsEndpoint(serverConfig: Record<string, any>): string {
+	const endpoint = normalizeString(
+		process.env.RESOLVEIO_LOGS_ENDPOINT
+		|| process.env.REMOTE_LOGS_ENDPOINT
+		|| serverConfig?.RESOLVEIO_LOGS_ENDPOINT
+		|| serverConfig?.REMOTE_LOGS_ENDPOINT
+	);
+	return endpoint || DEFAULT_RESOLVEIO_LOGS_ENDPOINT;
+}
+
+function resolveRemoteLogsApiKey(serverConfig: Record<string, any>): string {
+	return normalizeString(
+		process.env.RESOLVEIO_LOGS_API_KEY
+		|| process.env.REMOTE_LOGS_API_KEY
+		|| serverConfig?.RESOLVEIO_LOGS_API_KEY
+		|| serverConfig?.REMOTE_LOGS_API_KEY
+	);
+}
+
+function reviveIsoDateStrings(value: any): any {
+	if (Array.isArray(value)) {
+		return value.map((entry) => reviveIsoDateStrings(entry));
+	}
+	if (!value || typeof value !== 'object') {
+		if (typeof value === 'string' && /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z$/.test(value)) {
+			const parsed = new Date(value);
+			if (!Number.isNaN(parsed.getTime())) {
+				return parsed;
+			}
+		}
+		return value;
+	}
+	const next: Record<string, any> = {};
+	Object.keys(value).forEach((key) => {
+		next[key] = reviveIsoDateStrings(value[key]);
+	});
+	return next;
+}
+
+async function runLocalLogsQuery(
+	query: Record<string, any>,
+	options: Record<string, any>,
+	includeCount: boolean
+): Promise<{ source: 'local'; logs: any[]; count: number }> {
+	const logs = await Logs.find(query, options);
+	const count = includeCount ? await Logs.countDocuments(query) : logs.length;
+	return {
+		source: 'local',
+		logs,
+		count
+	};
+}
+
+async function runRemoteLogsQuery(
+	serverConfig: Record<string, any>,
+	payload: {
+		query: Record<string, any>;
+		options: Record<string, any>;
+		includeCount: boolean;
+	}
+): Promise<{ source: 'remote'; logs: any[]; count: number; endpoint: string }> {
+	const endpoint = resolveRemoteLogsEndpoint(serverConfig);
+	const apiKey = resolveRemoteLogsApiKey(serverConfig);
+	if (!apiKey) {
+		throw new Error('Missing RESOLVEIO_LOGS_API_KEY for remote log query.');
+	}
+	const response = await axios.post(
+		endpoint,
+		{
+			query: payload.query,
+			options: payload.options,
+			include_count: payload.includeCount
+		},
+		{
+			headers: {
+				'Content-Type': 'application/json',
+				'x-resolveio-logs-key': apiKey
+			},
+			timeout: REMOTE_LOG_QUERY_TIMEOUT_MS
+		}
+	);
+	const result = response?.data?.result || {};
+	return {
+		source: 'remote',
+		logs: Array.isArray(result.logs) ? reviveIsoDateStrings(result.logs) : [],
+		count: Number.isFinite(Number(result.count)) ? Number(result.count) : 0,
+		endpoint
+	};
+}
+
+export function loadLogMethods(methodManager: MethodManager) {
+		methodManager.methods({
+			insertErrorLog: {
+				function: async function(message, data) {
+					const payloadCorrelationId = typeof data === 'object' && data
+						? (typeof data.correlationId === 'string' && data.correlationId)
+							|| (typeof data.error === 'object' && data.error && typeof data.error.correlationId === 'string' && data.error.correlationId)
+						: undefined;
+					const correlationId = getCorrelationId() || payloadCorrelationId || objectIdHexString();
+					const payload = {
+						user: this.user,
+						id_user: this.id_user,
+						message,
+					data,
+					correlationId
+				};
+
+				await ErrorReporter.report({
+					sourceApp: 'method-logs',
+					message: 'Error Detected - ' + ResolveIOServer.getClientName(),
+					environment: this.serverConfig?.ROOT_URL || ResolveIOServer.getServerConfig()?.ROOT_URL,
+					clientSlug: ResolveIOServer.getClientName(),
+					clientName: ResolveIOServer.getServerConfig()?.CLIENT_NAME,
+					severity: 'error',
+					context: payload,
+					metadata: {
+						context: 'insertErrorLog',
+						message,
+						correlationId
+					},
+					correlationId
+				});
+
+				if (
+					ResolveIOServer.shouldWriteLogsOffline()
+				) {
+					ResolveIOServer.getLocalLogManager().writeLog({
+						type: 'log',
+						data: {
+							_id: objectIdHexString(),
+							createdAt: new Date(),
+							type: 'error',
+							collection: '',
+							id_document: '',
+							payload: JSON.stringify([payload], null, 2),
+							method: '',
+							id_user: this.id_user || '',
+							user: this.user || '',
+							messageId: 0,
+							route: '',
+							instance_index: process.env.NODE_APP_INSTANCE || '0',
+							correlationId
+						}
+					});
+				}
+				else {
+					await Logs.insertOne({
+						_id: objectIdHexString(),
+						type: 'error',
+						collection: '',
+						id_document: '',
+						payload: JSON.stringify([payload], null, 2),
+						method: '',
+						id_user: this.id_user || '',
+						user: this.user || '',
+						messageId: 0,
+						route: '',
+						client: 'ResolveIO',
+						instance: ResolveIOServer.getInstanceHost(),
+						instance_index: process.env.NODE_APP_INSTANCE || '0',
+						correlationId
+					});
+				}
+
+				return true;
+			},
+			skipValidation: true
+		},
+		insertSubscriptionLog: {
+			bypassLogs: true,
+			function: async function(type, subscription, collection_name, subData) {
+				if (
+					ResolveIOServer.shouldWriteLogsOffline()
+				) {
+					ResolveIOServer.getLocalLogManager().writeLog({
+						type: 'log-subscription',
+						data: {
+							_id: objectIdHexString(),
+							__v: 0,
+							date: new Date(),
+							type: type,
+							subscription: subscription,
+							collection_name: collection_name,
+							subData: subData,
+							instance_index: process.env.NODE_APP_INSTANCE || '0'
+						}
+					});
+				}
+				else {
+					await LogSubscriptions.create({
+						_id: objectIdHexString(),
+						__v: 0,
+						date: new Date(),
+						type: type,
+						subscription: subscription,
+						collection_name: collection_name,
+						subData: subData,
+						client: 'ResolveIO',
+						instance: ResolveIOServer.getInstanceHost()
+					});
+				}
+
+				return true;
+			},
+			skipValidation: true
+		},
+		queryLogsForDashboard: {
+			function: async function(payload: any = {}) {
+				const normalized = normalizeLogQueryPayload(payload);
+				const useRemote = shouldUseRemoteLogsSource(this.serverConfig || {}, normalized.sourceMode);
+				if (useRemote) {
+					try {
+						const remote = await runRemoteLogsQuery(this.serverConfig || {}, normalized);
+						return {
+							source: remote.source,
+							logs: remote.logs,
+							count: remote.count,
+							endpoint: remote.endpoint
+						};
+					}
+					catch (error) {
+						const fallback = await runLocalLogsQuery(normalized.query, normalized.options, normalized.includeCount);
+						return {
+							source: fallback.source,
+							logs: fallback.logs,
+							count: fallback.count,
+							remote_error: (error as Error)?.message || String(error)
+						};
+					}
+				}
+				const local = await runLocalLogsQuery(normalized.query, normalized.options, normalized.includeCount);
+				return {
+					source: local.source,
+					logs: local.logs,
+					count: local.count
+				};
+			},
+			skipValidation: true
+		},
+		queryResolveIOLogsRemote: {
+			function: async function(payload: any = {}) {
+				const normalized = normalizeLogQueryPayload({
+					...(payload || {}),
+					force_source: 'remote'
+				});
+				const remote = await runRemoteLogsQuery(this.serverConfig || {}, normalized);
+				return {
+					source: remote.source,
+					logs: remote.logs,
+					count: remote.count,
+					endpoint: remote.endpoint
+				};
+			},
+			skipValidation: true
+		},
+		superadminAPM: {
+			function: async function(date_start: Date, date_end: Date, graphInterval) {
+				let latencies = await LogMethodLatencies.aggregate([
+					{
+						$match: {
+							$and: [
+								{
+									createdAt: {$gte: date_start}
+								},
+								{
+									createdAt: {$lte: date_end}
+								}
+							]
+						}
+					},
+					{
+						$group: {
+							_id: '$method',
+							method: {$first: '$method'},
+							count: {$sum: 1},
+							latency_min: {$min: '$latency_ms'},
+							latency_avg: {$avg: '$latency_ms'},
+							latency_max: {$max: '$latency_ms'}
+						}
+					}
+				], {allowDiskUse: true, readPreference: this.serverConfig['ROOT_URL'] !== 'http://localhost:4200' ? 'secondary' : 'primary'}, true);
+
+				let subscriptions = await LogSubscriptions.aggregate([
+					{
+						$match: {
+							$and: [
+								{
+									createdAt: {$gte: date_start}
+								},
+								{
+									createdAt: {$lte: date_end}
+								}
+							]
+						}
+					},
+					{
+						$group: {
+							_id: '$subscription',
+							subscription: {$first: '$subscription'},
+							count: {$sum: 1}
+						}
+					}
+				], {allowDiskUse: true, readPreference: this.serverConfig['ROOT_URL'] !== 'http://localhost:4200' ? 'secondary' : 'primary'}, true);
+
+				let groupId = {};
+				let groupSort = {};
+
+				if (graphInterval === 'seconds') {
+					groupId = {
+						second: {
+							$second: {
+								date: '$createdAt',
+								timezone: process.env.TZ_CLIENT || 'America/Chicago'
+							} 
+						},
+						minute: {
+							$minute: {
+								date: '$createdAt',
+								timezone: process.env.TZ_CLIENT || 'America/Chicago'
+							} 
+						},
+						hour: {
+							$hour: {
+								date: '$createdAt',
+								timezone: process.env.TZ_CLIENT || 'America/Chicago'
+							} 
+						},
+						day: {
+							$dayOfMonth: {
+								date: '$createdAt',
+								timezone: process.env.TZ_CLIENT || 'America/Chicago'
+							} 
+						},
+						month: {
+							$month: {
+								date: '$createdAt',
+								timezone: process.env.TZ_CLIENT || 'America/Chicago'
+							} 
+						},
+						year: {
+							$year: {
+								date: '$createdAt',
+								timezone: process.env.TZ_CLIENT || 'America/Chicago'
+							} 
+						}
+					};
+
+					groupSort = {
+						'_id.year': 1,
+						'_id.month': 1,
+						'_id.day': 1,
+						'_id.hour': 1,
+						'_id.minute': 1,
+						'_id.second': 1
+					};
+				}
+				else if (graphInterval === 'minutes') {
+					groupId = {
+						minute: {
+							$minute: {
+								date: '$createdAt',
+								timezone: process.env.TZ_CLIENT || 'America/Chicago'
+							} 
+						},
+						hour: {
+							$hour: {
+								date: '$createdAt',
+								timezone: process.env.TZ_CLIENT || 'America/Chicago'
+							} 
+						},
+						day: {
+							$dayOfMonth: {
+								date: '$createdAt',
+								timezone: process.env.TZ_CLIENT || 'America/Chicago'
+							} 
+						},
+						month: {
+							$month: {
+								date: '$createdAt',
+								timezone: process.env.TZ_CLIENT || 'America/Chicago'
+							} 
+						},
+						year: {
+							$year: {
+								date: '$createdAt',
+								timezone: process.env.TZ_CLIENT || 'America/Chicago'
+							} 
+						}
+					};
+
+					groupSort = {
+						'_id.year': 1,
+						'_id.month': 1,
+						'_id.day': 1,
+						'_id.hour': 1,
+						'_id.minute': 1
+					};
+				}
+				else if (graphInterval === 'hours') {
+					groupId = {
+						hour: {
+							$hour: {
+								date: '$createdAt',
+								timezone: process.env.TZ_CLIENT || 'America/Chicago'
+							} 
+						},
+						day: {
+							$dayOfMonth: {
+								date: '$createdAt',
+								timezone: process.env.TZ_CLIENT || 'America/Chicago'
+							} 
+						},
+						month: {
+							$month: {
+								date: '$createdAt',
+								timezone: process.env.TZ_CLIENT || 'America/Chicago'
+							} 
+						},
+						year: {
+							$year: {
+								date: '$createdAt',
+								timezone: process.env.TZ_CLIENT || 'America/Chicago'
+							} 
+						}
+					};
+
+					groupSort = {
+						'_id.year': 1,
+						'_id.month': 1,
+						'_id.day': 1,
+						'_id.hour': 1
+					};
+				}
+				else {
+					groupId = {
+						day: {
+							$dayOfMonth: {
+								date: '$createdAt',
+								timezone: process.env.TZ_CLIENT || 'America/Chicago'
+							} 
+						},
+						month: {
+							$month: {
+								date: '$createdAt',
+								timezone: process.env.TZ_CLIENT || 'America/Chicago'
+							} 
+						},
+						year: {
+							$year: {
+								date: '$createdAt',
+								timezone: process.env.TZ_CLIENT || 'America/Chicago'
+							} 
+						}
+					};
+
+					groupSort = {
+						'_id.year': 1,
+						'_id.month': 1,
+						'_id.day': 1
+					};
+				}
+
+				let methodGraphData = await LogMethodLatencies.aggregate([
+					{
+						$match: {
+							$and: [
+								{
+									createdAt: {$gte: date_start}
+								},
+								{
+									createdAt: {$lte: date_end}
+								}
+							]
+						}
+					},
+					{
+						$group: {
+							_id: groupId,
+							count: {
+								$sum: 1
+							}
+						}
+					},
+					{
+						$sort: groupSort
+					}
+				], {allowDiskUse: true, readPreference: this.serverConfig['ROOT_URL'] !== 'http://localhost:4200' ? 'secondary' : 'primary'}, true);
+
+				let subGraphData = await LogSubscriptions.aggregate([
+					{
+						$match: {
+							$and: [
+								{
+									createdAt: {$gte: date_start}
+								},
+								{
+									createdAt: {$lte: date_end}
+								}
+							]
+						}
+					},
+					{
+						$group: {
+							_id: groupId,
+							count: {
+								$sum: 1
+							}
+						}
+					},
+					{
+						$sort: groupSort
+					}
+				], {allowDiskUse: true, readPreference: this.serverConfig['ROOT_URL'] !== 'http://localhost:4200' ? 'secondary' : 'primary'}, true);
+
+				return {
+					methods: latencies,
+					subscriptions: subscriptions,
+					methodGraphData: methodGraphData,
+					subGraphData: subGraphData
+				};
+			},
+			skipValidation: true
+		}
+	});
+}