@resolveio/server-lib 22.3.220 → 22.3.221

package/src/managers/slow-query.manager.ts

@@ -0,0 +1,519 @@
+import { ResolveIOServer } from '../resolveio-server-app';
+import { objectIdHexString } from '../util/common';
+import { createHash } from 'crypto';
+import { SlowQueryReportPayload, slowQueryLogStatuses } from '../types/slow-query-report';
+
+type SlowQueryLogModel = Record<string, any>;
+
+export interface SlowQueryManagerDependencies {
+	SlowQueryLogs: any;
+}
+
+let SlowQueryLogs: any = null;
+let configuredSlowQueryManagerDependencies: SlowQueryManagerDependencies | null = null;
+
+function resolveSlowQueryManagerDependencies(
+	overrides?: Partial<SlowQueryManagerDependencies>
+): SlowQueryManagerDependencies {
+	const resolved: SlowQueryManagerDependencies = {
+		...(configuredSlowQueryManagerDependencies || {} as SlowQueryManagerDependencies),
+		...(overrides || {})
+	};
+
+	if (!resolved.SlowQueryLogs) {
+		throw new Error('SlowQueryManager dependencies are not configured.');
+	}
+
+	return resolved;
+}
+
+function applySlowQueryManagerDependencies(dependencies: SlowQueryManagerDependencies): void {
+	configuredSlowQueryManagerDependencies = dependencies;
+	SlowQueryLogs = dependencies.SlowQueryLogs;
+}
+
+export function registerSlowQueryManagerDependencies(dependencies: SlowQueryManagerDependencies): void {
+	applySlowQueryManagerDependencies(dependencies);
+	ensureSlowQueryManager();
+}
+
+interface SlowQueryConfig {
+	enabled: boolean;
+	ingestKeys: string[];
+	debugLogging: boolean;
+	configSource: string;
+}
+
+const OBJECT_ID_PATTERN = /^[a-f0-9]{24}$/i;
+const UUID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
+const ISO_DATE_PATTERN = /^\d{4}-\d{2}-\d{2}(?:[tT ][\d:.+-]+)?(?:[zZ]|[+-]\d{2}:?\d{2})?$/;
+const NUMERIC_STRING_PATTERN = /^[-+]?\d+(?:\.\d+)?$/;
+const TOKEN_LIKE_PATTERN = /^[A-Za-z0-9._-]{16,}$/;
+const DATE_HINT_KEY_PATTERN = /(date|time|timestamp|created|updated|expires|expiry|start|end|at)$/i;
+const ID_HINT_KEY_PATTERN = /(^|_)(id|uuid|guid|token|hash|key)$/i;
+
+export class SlowQueryManager {
+	private config: SlowQueryConfig;
+	private serverConfig: Record<string, any> | null;
+
+	constructor(serverConfig, dependencies?: Partial<SlowQueryManagerDependencies>) {
+		const resolvedDependencies = resolveSlowQueryManagerDependencies(dependencies);
+		applySlowQueryManagerDependencies(resolvedDependencies);
+
+		this.serverConfig = serverConfig || {};
+		this.config = this.resolveConfig();
+	}
+
+	public isReady(): boolean {
+		return !!this.config.enabled;
+	}
+
+	public validateIngestKey(candidate?: string): boolean {
+		const keys = Array.isArray(this.config.ingestKeys) ? this.config.ingestKeys.filter(Boolean) : [];
+
+		if (!keys.length) {
+			return true;
+		}
+
+		if (!candidate) {
+			return false;
+		}
+
+		return keys.includes(candidate.trim());
+	}
+
+	public async ingestSlowQuery(payload: SlowQueryReportPayload): Promise<SlowQueryLogModel> {
+		if (!this.isReady()) {
+			throw new Error('SlowQueryManager is not enabled.');
+		}
+
+		if (!payload || !payload.queryHash || !payload.collection || typeof payload.durationMs !== 'number' || Number.isNaN(payload.durationMs)) {
+			throw new Error('Invalid slow query payload.');
+		}
+
+		const now = new Date();
+		const rawFilterValue = payload.filter ?? {};
+		const rawPipelineValue = Array.isArray(payload.pipeline) ? payload.pipeline : undefined;
+		const rawOptionsValue = payload.options;
+		const provisionalFingerprint = this.buildCanonicalFingerprint(payload.collection, rawFilterValue, rawPipelineValue, rawOptionsValue);
+		const provisionalScopeKey = this.buildFingerprintScopeKey({
+			clientSlug: payload.clientSlug,
+			clientName: payload.clientName,
+			sourceApp: payload.sourceApp,
+			environment: payload.environment,
+			publication: payload.publication,
+			serverUrl: payload.server_url
+		});
+
+		let existing = await SlowQueryLogs.findOne({
+			collection: payload.collection,
+			canonical_fingerprint_hash: provisionalFingerprint.hash,
+			fingerprint_scope_key: provisionalScopeKey
+		});
+
+		if (!existing) {
+			existing = await SlowQueryLogs.findOne(this.buildLegacyDedupeQuery(payload));
+		}
+		const existingOccurrences = typeof existing?.occurrences === 'number' ? existing.occurrences : 0;
+		const occurrences = existingOccurrences + 1;
+		const existingAvg = typeof existing?.avg_duration_ms === 'number' ? existing.avg_duration_ms : 0;
+		const totalDuration = existingAvg * existingOccurrences;
+		const avgDuration = (totalDuration + payload.durationMs) / occurrences;
+
+		const previousMax = typeof existing?.max_duration_ms === 'number' ? existing.max_duration_ms : payload.durationMs;
+		const maxBase = Math.max(payload.durationMs, previousMax);
+		const maxDuration = typeof payload.maxDurationMs === 'number'
+			? Math.max(maxBase, payload.maxDurationMs)
+			: maxBase;
+
+		const resolvedStatus = payload.status && slowQueryLogStatuses.includes(payload.status)
+			? payload.status
+			: (existing?.status || 'new');
+		const resolvedIgnored = typeof payload.ignored === 'boolean'
+			? payload.ignored
+			: (typeof existing?.ignored === 'boolean' ? existing.ignored : false);
+
+		const filterValue = payload.filter ?? existing?.filter ?? {};
+		const pipelineValue = Array.isArray(payload.pipeline)
+			? payload.pipeline
+			: existing?.pipeline;
+		if (this.config.debugLogging) {
+			this.logPipelineDebug(payload, pipelineValue, Array.isArray(existing?.pipeline) ? existing.pipeline : undefined);
+		}
+		const optionsValue = payload.options ?? existing?.options;
+		const explainPlanValue = typeof payload.explainPlan !== 'undefined' ? payload.explainPlan : existing?.explain_plan;
+		const explainStatsValue = typeof payload.explainExecutionStats !== 'undefined' ? payload.explainExecutionStats : existing?.explain_execution_stats;
+		const explainGeneratedAtValue = typeof payload.explainGeneratedAt !== 'undefined'
+			? this.coerceDate(payload.explainGeneratedAt)
+			: existing?.explain_generated_at;
+		const fingerprintScopeKey = this.buildFingerprintScopeKey({
+			clientSlug: payload.clientSlug ?? existing?.client_slug,
+			clientName: payload.clientName ?? existing?.client_name,
+			sourceApp: payload.sourceApp ?? existing?.source_app,
+			environment: payload.environment ?? existing?.environment,
+			publication: payload.publication ?? existing?.publication,
+			serverUrl: payload.server_url ?? existing?.server_url
+		});
+		const canonicalFingerprint = this.buildCanonicalFingerprint(
+			payload.collection,
+			filterValue,
+			pipelineValue,
+			optionsValue
+		);
+
+		const setFields: Partial<SlowQueryLogModel> = {
+			query_hash: payload.queryHash,
+			collection: payload.collection,
+			canonical_fingerprint: canonicalFingerprint.serialized,
+			canonical_fingerprint_hash: canonicalFingerprint.hash,
+			fingerprint_scope_key: fingerprintScopeKey,
+			filter: filterValue,
+			pipeline: pipelineValue,
+			options: optionsValue,
+			duration_ms: payload.durationMs,
+			max_duration_ms: maxDuration,
+			avg_duration_ms: avgDuration,
+			threshold_ms: typeof payload.thresholdMs === 'number' ? payload.thresholdMs : existing?.threshold_ms,
+			explain_plan: explainPlanValue,
+			explain_execution_stats: explainStatsValue,
+			explain_generated_at: explainGeneratedAtValue,
+			client_slug: payload.clientSlug ?? existing?.client_slug,
+			client_name: payload.clientName ?? existing?.client_name,
+			environment: payload.environment ?? existing?.environment,
+			source_app: payload.sourceApp ?? existing?.source_app,
+			node_hostname: payload.nodeHostname ?? existing?.node_hostname,
+			server_url: payload.server_url ?? existing?.server_url,
+			occurrences,
+			last_seen_at: now,
+			status: resolvedStatus,
+			ignored: resolvedIgnored,
+			notes: typeof payload.notes === 'string' ? payload.notes : existing?.notes,
+			publication: payload.publication ?? existing?.publication,
+			subscription_data: payload.subscription_data ?? existing?.subscription_data,
+			user_id: payload.userId ?? existing?.user_id
+		};
+
+		const cleaned = this.cleanSet(setFields);
+		if (existing) {
+			cleaned.updatedAt = now;
+			await SlowQueryLogs.updateOne({_id: existing._id}, {$set: cleaned});
+			return await SlowQueryLogs.findOne({_id: existing._id});
+		}
+
+		const doc: SlowQueryLogModel = {
+			_id: objectIdHexString(),
+			...cleaned,
+			slow_query_count: 0,
+			slow_query_count_string: 'Pending',
+			auto_fix_attempt_count: 0,
+			first_seen_at: now,
+			createdAt: now,
+			updatedAt: now,
+			verification_status: 'pending',
+			verification_runs: [],
+			verification_next_run_at: now
+		} as SlowQueryLogModel;
+
+		await SlowQueryLogs.insertOne(doc as any);
+		return await SlowQueryLogs.findOne({_id: doc._id});
+	}
+
+	private buildLegacyDedupeQuery(payload: SlowQueryReportPayload): {[key: string]: any} {
+		const dedupeQuery: {[key: string]: any} = {
+			query_hash: payload.queryHash,
+			collection: payload.collection
+		};
+
+		if (payload.clientSlug) {
+			dedupeQuery.client_slug = payload.clientSlug;
+		}
+
+		if (payload.environment) {
+			dedupeQuery.environment = payload.environment;
+		}
+
+		if (payload.sourceApp) {
+			dedupeQuery.source_app = payload.sourceApp;
+		}
+
+		if (payload.publication) {
+			dedupeQuery.publication = payload.publication;
+		}
+
+		if (payload.server_url) {
+			dedupeQuery.server_url = payload.server_url;
+		}
+
+		return dedupeQuery;
+	}
+
+	private buildFingerprintScopeKey(scope: {
+		clientSlug?: string;
+		clientName?: string;
+		sourceApp?: string;
+		environment?: string;
+		publication?: string;
+		serverUrl?: string;
+	}): string {
+		const clientKey = this.normalizeScopeSegment(scope.clientSlug || scope.clientName || '');
+		const sourceApp = this.normalizeScopeSegment(scope.sourceApp || '');
+		const environment = this.normalizeScopeSegment(scope.environment || '');
+		const publication = this.normalizeScopeSegment(scope.publication || '');
+		const serverUrl = this.normalizeScopeSegment(scope.serverUrl || '');
+		return [clientKey, sourceApp, environment, publication, serverUrl].join('|');
+	}
+
+	private normalizeScopeSegment(value: any): string {
+		const normalized = String(value || '')
+			.trim()
+			.toLowerCase()
+			.replace(/\s+/g, ' ');
+		return normalized || '_';
+	}
+
+	private buildCanonicalFingerprint(
+		collection: string,
+		filterValue: any,
+		pipelineValue?: any[],
+		optionsValue?: Record<string, any>
+	): {serialized: string; hash: string} {
+		const canonicalPayload = {
+			collection: this.normalizeScopeSegment(collection),
+			filter: this.normalizeCanonicalValue(filterValue, 'filter'),
+			pipeline: this.normalizeCanonicalValue(Array.isArray(pipelineValue) ? pipelineValue : [], 'pipeline'),
+			options: this.normalizeCanonicalValue(optionsValue ?? {}, 'options')
+		};
+		const serialized = JSON.stringify(canonicalPayload);
+		const hash = createHash('sha256').update(serialized, 'utf8').digest('hex');
+		return {serialized, hash};
+	}
+
+	private normalizeCanonicalValue(value: any, parentKey = ''): any {
+		if (value === null || typeof value === 'undefined') {
+			return null;
+		}
+
+		if (value instanceof Date) {
+			return '__date__';
+		}
+
+		if (value instanceof RegExp) {
+			return '__regex__';
+		}
+
+		if (Buffer.isBuffer(value)) {
+			return '__binary__';
+		}
+
+		const valueType = typeof value;
+		if (valueType === 'string') {
+			return this.normalizeCanonicalString(value, parentKey);
+		}
+		if (valueType === 'number') {
+			return '__num__';
+		}
+		if (valueType === 'boolean') {
+			return value;
+		}
+		if (valueType === 'bigint') {
+			return '__num__';
+		}
+		if (valueType === 'function') {
+			return '__fn__';
+		}
+
+		if (Array.isArray(value)) {
+			const normalizedItems = value.map(entry => this.normalizeCanonicalValue(entry, parentKey));
+			if (['$and', '$or', '$in', '$nin', '$all'].includes(parentKey)) {
+				const sortable = normalizedItems.map(item => ({
+					item,
+					serialized: JSON.stringify(item)
+				}));
+				sortable.sort((left, right) => left.serialized.localeCompare(right.serialized));
+				return sortable.map(entry => entry.item);
+			}
+			return normalizedItems;
+		}
+
+		if (valueType === 'object') {
+			if (this.isObjectIdLikeValue(value)) {
+				return '__objectid__';
+			}
+			const normalizedObj: Record<string, any> = {};
+			Object.keys(value).sort().forEach((key) => {
+				normalizedObj[key] = this.normalizeCanonicalValue(value[key], key);
+			});
+			return normalizedObj;
+		}
+
+		return '__literal__';
+	}
+
+	private normalizeCanonicalString(value: string, parentKey = ''): string {
+		const trimmed = String(value || '').trim();
+		if (!trimmed) {
+			return '__empty__';
+		}
+
+		if (parentKey === '$regex') {
+			return '__regex__';
+		}
+
+		if (DATE_HINT_KEY_PATTERN.test(parentKey) || ISO_DATE_PATTERN.test(trimmed)) {
+			return '__date__';
+		}
+
+		if (OBJECT_ID_PATTERN.test(trimmed)) {
+			return '__objectid__';
+		}
+
+		if (UUID_PATTERN.test(trimmed)) {
+			return '__uuid__';
+		}
+
+		if (NUMERIC_STRING_PATTERN.test(trimmed)) {
+			return trimmed.length >= 10 ? '__numstr_large__' : '__numstr__';
+		}
+
+		if (ID_HINT_KEY_PATTERN.test(parentKey) || TOKEN_LIKE_PATTERN.test(trimmed)) {
+			return '__token__';
+		}
+
+		return '__str__';
+	}
+
+	private isObjectIdLikeValue(value: any): boolean {
+		if (!value || typeof value !== 'object') {
+			return false;
+		}
+
+		const bsonType = String((value as any)._bsontype || '').trim().toLowerCase();
+		if (bsonType === 'objectid') {
+			return true;
+		}
+
+		if (typeof (value as any).toHexString === 'function') {
+			const hexValue = String((value as any).toHexString() || '').trim();
+			return OBJECT_ID_PATTERN.test(hexValue);
+		}
+
+		return false;
+	}
+
+	private resolveConfig(): SlowQueryConfig {
+		const slowQueryConfig = (this.serverConfig && (this.serverConfig.slowQuery || this.serverConfig.SLOW_QUERY)) || {};
+
+		const getBoolean = (envKey: string, configKey: string, fallback = true) => {
+			if (typeof process.env[envKey] !== 'undefined') {
+				return process.env[envKey] === 'true';
+			}
+			if (typeof slowQueryConfig[configKey] !== 'undefined') {
+				return !!slowQueryConfig[configKey];
+			}
+			return fallback;
+		};
+
+		const parseKeyList = (value: any): string[] => {
+			if (Array.isArray(value)) {
+				return value.map(item => `${item || ''}`.trim()).filter(Boolean);
+			}
+
+			if (typeof value === 'string') {
+				return value.split(',').map(item => item.trim()).filter(Boolean);
+			}
+
+			return [];
+		};
+
+		const ingestSource = typeof process.env.SLOW_QUERY_INGEST_KEYS !== 'undefined'
+			? process.env.SLOW_QUERY_INGEST_KEYS
+			: slowQueryConfig.ingestKeys;
+
+		return {
+			enabled: getBoolean('SLOW_QUERY_ENABLED', 'enabled', true),
+			ingestKeys: parseKeyList(ingestSource),
+			debugLogging: getBoolean('SLOW_QUERY_DEBUG_LOGS', 'debugLogging', false),
+			configSource: process.env.SLOW_QUERY_ENABLED ? 'environment' : (Object.keys(slowQueryConfig).length ? 'serverConfig' : 'defaults')
+		};
+	}
+
+	private cleanSet<T extends Record<string, any>>(input: T): T {
+		const output: Record<string, any> = {};
+
+		Object.keys(input).forEach(key => {
+			if (typeof input[key] !== 'undefined') {
+				output[key] = input[key];
+			}
+		});
+
+		return output as T;
+	}
+
+	private coerceDate(value: any): Date | undefined {
+		if (!value) {
+			return undefined;
+		}
+
+		if (value instanceof Date && !Number.isNaN(value.getTime())) {
+			return value;
+		}
+
+		const parsed = new Date(value);
+		return Number.isNaN(parsed.getTime()) ? undefined : parsed;
+	}
+
+	private logPipelineDebug(payload: SlowQueryReportPayload, pipelineValue: any, existingPipeline?: any[]): void {
+		if (!this.config.debugLogging) {
+			return;
+		}
+
+		const source = Array.isArray(payload.pipeline)
+			? 'payload'
+			: (Array.isArray(existingPipeline) ? 'existing' : 'none');
+		const length = Array.isArray(pipelineValue) ? pipelineValue.length : 0;
+		const stages = this.describePipelineStages(pipelineValue);
+
+		console.info(`[SlowQueryManager][pipeline] collection=${payload.collection} queryHash=${payload.queryHash} source=${source} len=${length} stages=${stages}`);
+	}
+
+	private describePipelineStages(pipeline?: any[]): string {
+		if (!Array.isArray(pipeline) || pipeline.length === 0) {
+			return 'empty';
+		}
+
+		const snippet = pipeline.slice(0, 3).map(stage => {
+			if (!stage || typeof stage !== 'object') {
+				return 'unknown';
+			}
+
+			const key = Object.keys(stage)[0];
+			return key ? key : 'stage';
+		});
+
+		const additional = pipeline.length > snippet.length
+			? ` (+${pipeline.length - snippet.length} more)`
+			: '';
+
+		return `${snippet.join(', ')}${additional}`;
+	}
+}
+
+export function ensureSlowQueryManager(serverConfig?: any): SlowQueryManager | null {
+	const existing = ResolveIOServer['SlowQueryManager'] as SlowQueryManager | null;
+	if (existing) {
+		return existing;
+	}
+
+	if (!configuredSlowQueryManagerDependencies) {
+		return null;
+	}
+
+	const resolvedServerConfig = serverConfig || ResolveIOServer.getServerConfig();
+	if (!resolvedServerConfig) {
+		return null;
+	}
+
+	const manager = new SlowQueryManager(resolvedServerConfig);
+	ResolveIOServer['SlowQueryManager'] = manager;
+	return manager;
+}