@resolveio/server-lib 22.3.219 → 22.3.221

package/src/util/schema-report-builder.ts

@@ -0,0 +1,448 @@
+import { CollectionTree } from '../models/report-builder.model';
+import { ResolveIOServer } from '../resolveio-server-app';
+import { toTitleCase } from './common';
+
+function assertLookupCollection(collectionName: string, referringField?: string) {
+	let collectionModel = ResolveIOServer.getMongoManager().collection(collectionName);
+
+	if (!collectionModel) {
+		let context = referringField ? ` referenced by '${referringField}'` : '';
+		throw new Error(`Report Builder: collection '${collectionName}'${context} is not registered.`);
+	}
+
+	if (!collectionModel.useRB || !collectionModel.rbSchema) {
+		let context = referringField ? ` referenced by '${referringField}'` : '';
+		throw new Error(`Report Builder: collection '${collectionName}'${context} is not configured for Report Builder.`);
+	}
+
+	return collectionModel;
+}
+
+export function getReportLookupSchemas(collection_root: string) {
+	let lookupTrees: CollectionTree[] = [];
+	let processedCollections = new Set<string>();
+	let pendingCollections: Array<{ name: string; isRoot: boolean; referringField?: string }> = [{ name: collection_root, isRoot: true }];
+
+	while (pendingCollections.length) {
+		let current = pendingCollections.shift();
+
+		if (!current) {
+			continue;
+		}
+
+		if (processedCollections.has(current.name)) {
+			continue;
+		}
+
+		let collectionModel = assertLookupCollection(current.name, current.referringField);
+		let schemaObj = collectionModel.rbSchema;
+
+		lookupTrees.push({ collection_name: current.name, tree: schemaObj, is_root: current.isRoot });
+		processedCollections.add(current.name);
+
+		Object.keys(schemaObj).forEach(field => {
+			if (schemaObj[field].lookup_collection && !processedCollections.has(schemaObj[field].lookup_collection)) {
+				pendingCollections.push({
+					name: schemaObj[field].lookup_collection,
+					isRoot: false,
+					referringField: `${current.name}.${field}`
+				});
+			}
+		});
+	}
+
+	return lookupTrees;
+}
+
+export function buildTree(current_root, schemaTree, lookup?) {
+	let res = [];
+
+	Object.keys(schemaTree).filter(a => a !== '__v').forEach(key => {
+	// Object.keys(schemaTree).filter(a => a !== '__v').forEach(key => {
+		if (!lookup || (lookup && !schemaTree[key].lookup_collection)) {
+			let treeItemSplit = key.split('.');
+		
+			// debugger;
+
+			if (treeItemSplit.length === 1) {
+				if (schemaTree[key].rbType === 'String' || schemaTree[key].rbType === 'Number' || schemaTree[key].rbType === 'Boolean' || schemaTree[key].rbType === 'Date') {
+					res.push({
+						collection_name: current_root,
+						columnName: toTitleCase(key.replace(/\_/g, ' ')),
+						fieldName: key,
+						fieldType: schemaTree[key].rbType,
+						fieldTypeName: schemaTree[key].rbType === 'String' ? 'Text' : (schemaTree[key].rbType === 'Boolean' ? 'T/F' : schemaTree[key].rbType),
+						distinctFieldValues: [],
+						fieldPath: lookup ? lookup.lookup_as + '.$.' + key : key,
+						fieldPathName: lookup ? (toTitleCase(lookup.lookup_as.replace(/\.\$\./g, ' (List) -> ').replace(/\./g, ' (Group) -> '))) + ' (List) -> ' + (toTitleCase(key.replace(/\.\$\./g, ' (List) -> ').replace(/\./g, ' (Group) -> '))) : toTitleCase(key.replace(/\.\$\./g, ' (List) -> ').replace(/\./g, ' (Group) -> ')),
+						lookup_collection: lookup ? lookup.lookup_collection : (schemaTree[key].lookup_collection ? schemaTree[key].lookup_collection : ''),
+						lookup_local_key: lookup ? lookup.lookup_local_key : (schemaTree[key].lookup_collection ? schemaTree[key].local_key : ''),
+						lookup_foreign_key: lookup ? lookup.lookup_foreign_key : (schemaTree[key].lookup_collection ? schemaTree[key].lookup_key : ''),
+						lookup_as: lookup ? lookup.lookup_as : (schemaTree[key].lookup_collection ? schemaTree[key].lookup_as : ''),
+						text: toTitleCase(key.replace(/\_/g, ' ')),
+						value: key,
+						isLeaf: true,
+						isActive: false,
+						isSelected: false,
+						depth: 0,
+						leafValueType: '',
+						leafValueTypes: getLeafValueTypes(schemaTree[key], key),
+						leafFormatType: '',
+						leafFormatTypes: getLeafFormatTypes(schemaTree[key], key)
+					});
+				}
+				else {
+					res.push({
+						collection_name: current_root,
+						columnName: toTitleCase(key.replace(/\_/g, ' ')),
+						fieldName: key,
+						fieldType: schemaTree[key].lookup_collection ? 'Lookup' : schemaTree[key].rbType,
+						fieldTypeName: schemaTree[key].lookup_collection ? 'Lookup' : (schemaTree[key].rbType === 'Array' ? 'List' : 'Group'),
+						distinctFieldValues: [],
+						fieldPath: lookup ? lookup.lookup_as + '.$.' + key : key,
+						fieldPathName: lookup ? (toTitleCase(lookup.lookup_as.replace(/\.\$\./g, ' (List) -> ').replace(/\./g, ' (Group) -> '))) + ' (List) -> ' + (toTitleCase(key.replace(/\.\$\./g, ' (List) -> ').replace(/\./g, ' (Group) -> '))) : toTitleCase(key.replace(/\.\$\./g, ' (List) -> ').replace(/\./g, ' (Group) -> ')),
+						lookup_collection: lookup ? lookup.lookup_collection : (schemaTree[key].lookup_collection ? schemaTree[key].lookup_collection : ''),
+						lookup_local_key: lookup ? lookup.lookup_local_key : (schemaTree[key].lookup_collection ? schemaTree[key].local_key : ''),
+						lookup_foreign_key: lookup ? lookup.lookup_foreign_key : (schemaTree[key].lookup_collection ? schemaTree[key].lookup_key : ''),
+						lookup_as: lookup ? lookup.lookup_as : (schemaTree[key].lookup_collection ? schemaTree[key].lookup_as : ''),
+						text: toTitleCase(key.replace(/\_/g, ' ')),
+						value: key,
+						isLeaf: false,
+						isActive: false,
+						isSelected: false,
+						depth: 0,
+						leafValueType: '',
+						leafFormatType: '',
+						children: []
+					});
+				}
+
+				res = res.sort((a, b) => a.fieldName.localeCompare(b.fieldName));
+			}
+			else {
+				let child = null;
+
+				treeItemSplit.forEach((layer, index) => {
+					if (!child) {
+						child = res.find(a => a.fieldName === layer);
+					}
+					else {
+						if (child.children.find(a => a.fieldName === layer)) {
+							child = child.children.find(a => a.fieldName === layer);
+						}
+						else {
+							if (schemaTree[key].rbType === 'String' || schemaTree[key].rbType === 'Number' || schemaTree[key].rbType === 'Boolean' || schemaTree[key].rbType === 'Date') {							
+								child.children.push({
+									collection_name: current_root,
+									columnName: toTitleCase(key.replace(/\_/g, ' ')),
+									fieldName: layer,
+									fieldType: schemaTree[key].rbType,
+									fieldTypeName: schemaTree[key].rbType === 'String' ? 'Text' : (schemaTree[key].rbType === 'Boolean' ? 'T/F' : schemaTree[key].rbType),
+									distinctFieldValues: [],
+									fieldPath: lookup ? lookup.lookup_as + '.$.' + key : key,
+									fieldPathName: lookup ? (toTitleCase(lookup.lookup_as.replace(/\.\$\./g, ' (List) -> ').replace(/\./g, ' (Group) -> '))) + ' (List) -> ' + (toTitleCase(key.replace(/\.\$\./g, ' (List) -> ').replace(/\./g, ' (Group) -> '))) : toTitleCase(key.replace(/\.\$\./g, ' (List) -> ').replace(/\./g, ' (Group) -> ')),
+									lookup_collection: lookup ? lookup.lookup_collection : (schemaTree[key].lookup_collection ? schemaTree[key].lookup_collection : ''),
+									lookup_local_key: lookup ? lookup.lookup_local_key : (schemaTree[key].lookup_collection ? schemaTree[key].local_key : ''),
+									lookup_foreign_key: lookup ? lookup.lookup_foreign_key : (schemaTree[key].lookup_collection ? schemaTree[key].lookup_key : ''),
+									lookup_as: lookup ? lookup.lookup_as : (schemaTree[key].lookup_collection ? schemaTree[key].lookup_as : ''),
+									text: toTitleCase(layer.replace(/\_/g, ' ')),
+									value: layer,
+									isLeaf: true,
+									isActive: false,
+									isSelected: false,
+									depth: index,
+									leafValueType: '',
+									leafValueTypes: getLeafValueTypes(schemaTree[key], key),
+									leafFormatType: '',
+									leafFormatTypes: getLeafFormatTypes(schemaTree[key], key)
+								});
+							}
+							else {
+								child.children.push({
+									collection_name: current_root,
+									columnName: toTitleCase(key.replace(/\_/g, ' ')),
+									fieldName: layer,
+									fieldType: schemaTree[key].lookup_collection ? 'Lookup' : schemaTree[key].rbType,
+									fieldTypeName: schemaTree[key].lookup_collection ? 'Lookup' : (schemaTree[key].rbType === 'Array' ? 'List' : 'Group'),
+									distinctFieldValues: [],
+									fieldPath: lookup ? lookup.lookup_as + '.$.' + key : key,
+									fieldPathName: lookup ? (toTitleCase(lookup.lookup_as.replace(/\.\$\./g, ' (List) -> ').replace(/\./g, ' (Group) -> '))) + ' (List) -> ' + (toTitleCase(key.replace(/\.\$\./g, ' (List) -> ').replace(/\./g, ' (Group) -> '))) : toTitleCase(key.replace(/\.\$\./g, ' (List) -> ').replace(/\./g, ' (Group) -> ')),
+									lookup_collection: lookup ? lookup.lookup_collection : (schemaTree[key].lookup_collection ? schemaTree[key].lookup_collection : ''),
+									lookup_local_key: lookup ? lookup.lookup_local_key : (schemaTree[key].lookup_collection ? schemaTree[key].local_key : ''),
+									lookup_foreign_key: lookup ? lookup.lookup_foreign_key : (schemaTree[key].lookup_collection ? schemaTree[key].lookup_key : ''),
+									lookup_as: lookup ? lookup.lookup_as : (schemaTree[key].lookup_collection ? schemaTree[key].lookup_as : ''),
+									text: toTitleCase(layer.replace(/\_/g, ' ')),
+									value: layer,
+									isLeaf: false,
+									isActive: false,
+									isSelected: false,
+									depth: index,
+									leafValueType: '',
+									leafFormatType: '',
+									children: []
+								});
+							}
+
+							child.children = child.children.sort((a, b) => a.fieldName.localeCompare(b.fieldName));
+						}
+					}
+				});
+			}
+		}
+	});
+
+	return res;
+}
+
+// eslint-disable-next-line no-unused-vars
+export function getLeafValueTypes(schemaTree, fieldPath) {
+	let leafTypes = [];
+
+	if (schemaTree.rbType === 'String') {
+		leafTypes = [
+			{
+				text: 'As Value',
+				value: 'Value'
+			},
+			{
+				text: 'Unique',
+				value: 'Unique'
+			},
+			{
+				text: 'As Count',
+				value: 'Count'
+			},
+			{
+				text: 'First',
+				value: 'First'
+			},
+			{
+				text: 'Last',
+				value: 'Last'
+			}
+		];
+	}
+	else if (schemaTree.rbType === 'Number') {
+		leafTypes = [
+			{
+				text: 'As Value',
+				value: 'Value'
+			},
+			{
+				text: 'Unique',
+				value: 'Unique'
+			},
+			{
+				text: 'As Average',
+				value: 'Average'
+			},
+			{
+				text: 'As Count',
+				value: 'Count'
+			},
+			{
+				text: 'As Sum',
+				value: 'Sum'
+			},
+			{
+				text: 'Minimum',
+				value: 'Minimum'
+			},
+			{
+				text: 'Maximum',
+				value: 'Maximum'
+			},
+			{
+				text: 'First',
+				value: 'First'
+			},
+			{
+				text: 'Last',
+				value: 'Last'
+			}
+		];
+	}
+	else if (schemaTree.rbType === 'Boolean') {
+		leafTypes = [
+			{
+				text: 'As Value',
+				value: 'Value'
+			},
+			{
+				text: 'Unique',
+				value: 'Unique'
+			},
+			{
+				text: 'As Count',
+				value: 'Count'
+			},
+			{
+				text: 'First',
+				value: 'First'
+			},
+			{
+				text: 'Last',
+				value: 'Last'
+			}
+		];
+	}
+	else if (schemaTree.rbType === 'Date') {
+		leafTypes = [
+			{
+				text: 'As Value',
+				value: 'Value'
+			},
+			{
+				text: 'Unique',
+				value: 'Unique'
+			},
+			{
+				text: 'As Count',
+				value: 'Count'
+			},
+			{
+				text: 'Minimum',
+				value: 'Minimum'
+			},
+			{
+				text: 'Maximum',
+				value: 'Maximum'
+			},
+			{
+				text: 'First',
+				value: 'First'
+			},
+			{
+				text: 'Last',
+				value: 'Last'
+			}
+		];
+	}
+	return leafTypes;
+}
+
+// eslint-disable-next-line no-unused-vars
+export function getLeafFormatTypes(schemaTree, fieldPath) {
+	let leafTypes = [];
+
+	if (schemaTree.rbType === 'String') {
+		leafTypes = [
+			{
+				text: 'As Text',
+				value: 'String'
+			},
+			{
+				text: 'As Number',
+				value: 'Number'
+			}
+		];
+	}
+	else if (schemaTree.rbType === 'Number') {
+		leafTypes = [
+			{
+				text: 'As Number',
+				value: 'Number'
+			},
+			{
+				text: 'As Currency',
+				value: 'Currency'
+			},
+			{
+				text: 'As Text',
+				value: 'String'
+			}
+		];
+	}
+	else if (schemaTree.rbType === 'Boolean') {
+		leafTypes = [
+			{
+				text: 'As Number',
+				value: 'Number'
+			},
+			{
+				text: 'As Boolean (True/False)',
+				value: 'Boolean'
+			},
+			{
+				text: 'As Text (Yes/No)',
+				value: 'String'
+			},
+			{
+				text: 'As Number (1/0)',
+				value: 'Boolean_Number'
+			}
+		];
+	}
+	else if (schemaTree.rbType === 'Date') {
+		leafTypes = [
+			{
+				text: 'As Number',
+				value: 'Number'
+			},
+			{
+				text: 'As Date (Short Format)',
+				value: 'Date'
+			}, 
+			{
+				text: 'As Time (Short Format)',
+				value: 'Time'
+			}, 
+			{
+				text: 'As Date/Time (Short Format)',
+				value: 'DateTime'
+			}, 
+			{
+				text: 'As Date (Long Format)',
+				value: 'Date_long'
+			},
+			{
+				text: 'As Date/Time (Long Format)',
+				value: 'DateTime_long'
+			}, 
+			{
+				text: 'As Timestamp (MS since 1970)',
+				value: 'Timestamp'
+			}
+		];
+	}
+	return leafTypes;
+}
+
+
+
+
+
+export async function mongoCommand(commandstr: string, collection: string, field?: any) {
+	let tmpObj;
+	try {
+		// Count the number of non-empty fields in a given collection:
+		if (commandstr === 'count') {
+			let obj = {
+				count: collection,
+				query: {}
+			};
+			obj.query[field] = {$ne: null};
+			tmpObj = await ResolveIOServer.getMainDB().command(obj);
+		}
+		// Get all distinct values in a given field.
+		else if (commandstr === 'distinct') {
+			let obj = {
+				distinct: collection,
+				query: {},
+				key: field,
+			};
+			tmpObj = await ResolveIOServer.getMainDB().command(obj);
+		}
+		else if (commandstr === 'listIndexes') {
+			// tmpObj = await this.mainDb.db[collection].getIndexes();
+			// let obj = { 'listIndexes': collection };
+			// tmpObj = await this.mainDb.db.getIndexes(obj);
+		}
+	} catch {
+		// console.log('----------- mongoCommand err: ', err);
+		return -1;
+	}
+	if (tmpObj) {
+		return tmpObj.values;
+	}
+}

package/src/util/slow-query-reporter.ts

@@ -0,0 +1,216 @@
+import { createHash } from 'crypto';
+import { Document } from 'mongodb';
+import { hostname } from 'os';
+import { ResolveIOServer } from '../resolveio-server-app';
+import { SlowQueryReportPayload } from '../types/slow-query-report';
+import { deepCopy } from './common';
+
+const DEFAULT_SLOW_QUERY_THRESHOLD_MS = 1000;
+
+interface SlowQueryReportParams {
+	collection: string;
+	filter?: Document;
+	pipeline?: any[];
+	options?: Document;
+	durationMs: number;
+	queryType: 'find' | 'findOne' | 'aggregate' | 'countDocuments';
+	notes?: string;
+	queryHash?: string;
+	sourceApp?: string;
+	serverUrl?: string;
+	publication?: string;
+	subscriptionData?: any[];
+	userId?: string;
+}
+
+export class SlowQueryReporter {
+	public static async reportSlowQuery(params: SlowQueryReportParams): Promise<void> {
+		if (!this.isReportingEnabled()) {
+			return;
+		}
+
+		const thresholdMs = this.getThresholdMs();
+		if (thresholdMs <= 0 || params.durationMs < thresholdMs) {
+			return;
+		}
+
+		const manager = ResolveIOServer['SlowQueryManager'];
+		if (!manager || typeof manager.ingestSlowQuery !== 'function') {
+			console.error('SlowQueryReporter skipped: local SlowQueryManager is not available');
+			return;
+		}
+		if (typeof manager.isReady === 'function' && !manager.isReady()) {
+			return;
+		}
+
+		const queryHash = params.queryHash || this.generateQueryHash(params);
+		const payload: SlowQueryReportPayload = {
+			queryHash,
+			collection: params.collection,
+			pipeline: params.pipeline ? deepCopy(params.pipeline) : undefined,
+			filter: params.filter ? deepCopy(params.filter) : undefined,
+			options: params.options ? deepCopy(params.options) : undefined,
+			durationMs: params.durationMs,
+			thresholdMs,
+			clientSlug: ResolveIOServer.getClientName(),
+			clientName: this.getClientName(),
+			sourceApp: params.sourceApp || 'mongo-manager',
+			environment: this.getEnvironment(),
+			nodeHostname: hostname(),
+			notes: params.notes,
+			server_url: params.serverUrl || this.getServerUrl(),
+			publication: params.publication,
+			subscription_data: params.subscriptionData,
+			userId: params.userId
+		};
+
+		try {
+			await manager.ingestSlowQuery(payload);
+		}
+		catch (err) {
+			console.error('SlowQueryReporter failed', err);
+		}
+	}
+
+	public static reportSlowQueryFireAndForget(params: SlowQueryReportParams): void {
+		// eslint-disable-next-line no-restricted-syntax, promise/prefer-await-to-then
+		void this.reportSlowQuery(params).catch(err => {
+			console.error('SlowQueryReporter fire-and-forget error', err);
+		});
+	}
+
+	private static isReportingEnabled(): boolean {
+		const config = ResolveIOServer.getServerConfig();
+		const envValue = process.env.SLOW_QUERY_ENABLED;
+		if (typeof envValue === 'string' && envValue.length) {
+			const normalized = envValue.trim().toLowerCase();
+			if (['true', '1', 'yes'].includes(normalized)) {
+				return true;
+			}
+			else if (['false', '0', 'no'].includes(normalized)) {
+				return false;
+			}
+		}
+
+		const slowQueryConfig = (config && (config.slowQuery || config.SLOW_QUERY)) || null;
+		const configValue = typeof slowQueryConfig?.enabled !== 'undefined'
+			? slowQueryConfig.enabled
+			: config?.SLOW_QUERY_ENABLED;
+
+		if (typeof configValue === 'boolean') {
+			return configValue;
+		}
+
+		if (typeof configValue === 'string' && configValue.length) {
+			const normalized = configValue.trim().toLowerCase();
+			if (['true', '1', 'yes'].includes(normalized)) {
+				return true;
+			}
+			else if (['false', '0', 'no'].includes(normalized)) {
+				return false;
+			}
+		}
+
+		if (typeof configValue === 'number') {
+			return configValue !== 0;
+		}
+
+		return true;
+	}
+
+	private static getThresholdMs(): number {
+		const config = ResolveIOServer.getServerConfig();
+		const envValue = process.env.SLOW_QUERY_THRESHOLD_MS;
+		if (envValue && !Number.isNaN(Number(envValue))) {
+			return Number.parseInt(envValue, 10);
+		}
+
+		const configValue = config?.SLOW_QUERY_THRESHOLD_MS;
+		if (typeof configValue === 'number') {
+			return configValue;
+		}
+
+		if (typeof configValue === 'string' && configValue.length && !Number.isNaN(Number(configValue))) {
+			return Number.parseInt(configValue, 10);
+		}
+
+		return DEFAULT_SLOW_QUERY_THRESHOLD_MS;
+	}
+
+	private static getEnvironment(): string {
+		const config = ResolveIOServer.getServerConfig();
+		return (
+			config?.RESOLVEIO_ENV ||
+			config?.ENVIRONMENT ||
+			config?.ROOT_URL ||
+			process.env.RESOLVEIO_ENV ||
+			process.env.NODE_ENV ||
+			'unknown'
+		);
+	}
+
+	private static getClientName(): string | undefined {
+		const config = ResolveIOServer.getServerConfig();
+		return config?.CLIENT_NAME || ResolveIOServer.getClientName();
+	}
+
+	private static generateQueryHash(params: SlowQueryReportParams): string {
+		try {
+			const normalized = this.normalizeForHash({
+				collection: params.collection,
+				filter: params.filter || null,
+				pipeline: params.pipeline || null,
+				options: params.options || null,
+				queryType: params.queryType
+			});
+			const digest = createHash('sha256').update(JSON.stringify(normalized)).digest('hex').slice(0, 16);
+			return `${params.collection}-${digest}`;
+		}
+		catch (err) {
+			console.error('SlowQueryReporter hash generation failed', err);
+			return `${params.collection}-${Date.now()}`;
+		}
+	}
+
+	private static getServerUrl(): string | undefined {
+		const config = ResolveIOServer.getServerConfig();
+		if (config?.ROOT_URL) {
+			return `${config.ROOT_URL}`.trim();
+		}
+
+		if (process.env.ROOT_URL) {
+			return process.env.ROOT_URL.trim();
+		}
+
+		return undefined;
+	}
+
+	private static normalizeForHash(value: any): any {
+		if (value === null || value === undefined) {
+			return value;
+		}
+
+		if (Array.isArray(value)) {
+			return value.map(item => this.normalizeForHash(item));
+		}
+
+		if (value instanceof RegExp) {
+			return value.toString();
+		}
+
+		if (typeof value === 'object') {
+			if (typeof (value as any).toHexString === 'function') {
+				return (value as any).toHexString();
+			}
+
+			const ordered: Record<string, any> = {};
+			Object.keys(value).sort().forEach(key => {
+				ordered[key] = this.normalizeForHash(value[key]);
+			});
+			return ordered;
+		}
+
+		return value;
+	}
+
+}