@resolveio/server-lib 22.3.219 → 22.3.221

package/src/managers/monitor.manager.ts

@@ -0,0 +1,507 @@
+import axios from 'axios';
+import { MonitorCPUs } from '../collections/monitor-cpu.collection';
+import { MonitorFunctions } from '../collections/monitor-function.collection';
+import { MonitorMemorys } from '../collections/monitor-memory.collection';
+import { MonitorMongos } from '../collections/monitor-mongo.collection';
+import { ResolveIOServer } from '../resolveio-server-app';
+import { objectIdHexString } from '../util/common';
+import { LocalLogModel } from './local-log.manager';
+
+const os = require('os');
+const MAX_BATCH_SIZE = 8 * 1024 * 1024; // 8MB max for safety
+
+export class MonitorManager {
+	private _instanceHostname = '';
+	private _monitorData: LocalLogModel[] = [];
+
+	constructor() {}
+
+	static async create() {
+		const monitorManager = new MonitorManager();
+		await monitorManager.initialize();
+		return monitorManager;
+	}
+
+	private async initialize() {
+		this._instanceHostname = os.hostname().replace(/\./g, '-');
+		await this.setupIntervals();
+		this.setupEventLoop();
+	}
+
+	cpuAverage() {
+		var totalIdle = 0, totalTick = 0;
+		var cpus = os.cpus();
+
+		for (var i = 0, len = cpus.length; i < len; i++) {
+			var cpu = cpus[i];
+
+			for (let type in cpu.times) {
+				totalTick += cpu.times[type];
+			}
+			totalIdle += cpu.times.idle;
+		}
+
+		return {
+			idle: totalIdle / cpus.length,
+			total: totalTick / cpus.length
+		};
+	}
+
+	async setupIntervals() {
+		const processOnlyRuntime = this.isProcessOnlyManagerRuntime();
+
+		// Example: If not on production or localhost, do a health check
+		if (
+			!processOnlyRuntime
+		&&
+			ResolveIOServer.getServerConfig()['ROOT_URL'] !== 'https://resolveio.com'
+		&& ResolveIOServer.getServerConfig()['ROOT_URL'] !== 'http://localhost:4200'
+		) {
+			try {
+				let instanceId = await axios.get('http://169.254.169.254/latest/meta-data/instance-id');
+
+				setInterval(async () => {
+					try {
+						await axios.post('https://backend.resolveio.com/api/health', {
+							type: 'application',
+							id_aws_instance: instanceId.data,
+							version: typeof process.env.APP_VERSION === 'number'
+								? (<any>process.env.APP_VERSION).toString()
+								: process.env.APP_VERSION,
+							version_key: process.env.APP_VERSION_KEY
+						});
+					}
+					catch {}
+				}, 10000);
+			}
+			catch {}
+		}
+
+		let lastCPU = this.cpuAverage();
+
+		setInterval(async () => {
+			let now = new Date();
+			var endMeasure = this.cpuAverage();
+			var idleDifference = endMeasure.idle - lastCPU.idle;
+			var totalDifference = endMeasure.total - lastCPU.total;
+			var percentageCPU = 1 - idleDifference / totalDifference;
+
+			// Original local code
+			if (
+				!ResolveIOServer.shouldWriteLogsOffline()
+			) {
+				await MonitorCPUs.create({
+					_id: objectIdHexString(),
+					metadata: {
+						instance: this._instanceHostname,
+						client: ResolveIOServer.getServerConfig()['CLIENT_NAME']
+					},
+					date: now,
+					app: percentageCPU,
+					system: os.loadavg()[0]
+				});
+			}
+			else {
+				this.pushMonitorData({
+					type: 'monitor-cpu',
+					data: {
+						metadata: {
+							instance: this._instanceHostname,
+							client: ResolveIOServer.getServerConfig()['CLIENT_NAME']
+						},
+						date: now,
+						app: percentageCPU,
+						system: os.loadavg()[0]
+					}
+				});
+			}
+
+			lastCPU = endMeasure;
+
+			const memUsage = process.memoryUsage();
+
+			if (
+				!ResolveIOServer.shouldWriteLogsOffline()
+			) {
+				await MonitorMemorys.create({
+					_id: objectIdHexString(),
+					metadata: {
+						instance: this._instanceHostname,
+						client: ResolveIOServer.getServerConfig()['CLIENT_NAME']
+					},
+					date: now,
+					physical_total: memUsage.heapTotal,
+					physical_used: memUsage.heapUsed,
+					physical_free: os.freemem(),
+					virtual: memUsage.rss,
+					private: memUsage.external,
+					physical: os.totalmem()
+				});
+			}
+			else {
+				this.pushMonitorData({
+					type: 'monitor-memory',
+					data: {
+						metadata: {
+							instance: this._instanceHostname,
+							client: ResolveIOServer.getServerConfig()['CLIENT_NAME']
+						},
+						date: now,
+						physical_total: memUsage.heapTotal,
+						physical_used: memUsage.heapUsed,
+						physical_free: os.freemem(),
+						virtual: memUsage.rss,
+						private: memUsage.external,
+						physical: os.totalmem(),
+						type: 'memory'
+					}
+				});
+			}
+		}, 3000);
+	}
+
+	private isProcessOnlyManagerRuntime(): boolean {
+		const parseFlag = (value: any): boolean => {
+			const normalized = String(value ?? '').trim().toLowerCase();
+			return normalized === '1'
+				|| normalized === 'true'
+				|| normalized === 'yes'
+				|| normalized === 'on'
+				|| normalized === 'enabled';
+		};
+		return parseFlag(process.env.SUPPORT_CODEX_MANAGER_PROCESS_ONLY)
+			|| parseFlag(process.env.SUPPORT_AUTO_MANAGER_PROCESS_ONLY)
+			|| parseFlag(process.env.AI_ASSISTANT_CODEX_MANAGER_PROCESS_ONLY);
+	}
+
+	async addMongoTracker(data: MongoMonitorModel) {
+		if (
+			!ResolveIOServer.shouldWriteLogsOffline()
+		) {
+			if (!data.collection.startsWith('monitor-')) {
+				await MonitorMongos.create({
+					_id: objectIdHexString(),
+					metadata: {
+						instance: this._instanceHostname,
+						client: ResolveIOServer.getServerConfig()['CLIENT_NAME']
+					},
+					date: new Date(data.date),
+					method: data.method,
+					doc_collection: data.collection,
+					duration: data.duration,
+					query: Buffer.byteLength(data.query, 'utf8') < 15800000
+						? data.query
+						: 'Too Big'
+				});
+			}
+		}
+		else {
+			if (this._instanceHostname) {
+				this.pushMonitorData({
+					type: 'monitor-mongo',
+					data: {
+						metadata: {
+							instance: this._instanceHostname,
+							client: ResolveIOServer.getServerConfig()['CLIENT_NAME']
+						},
+						date: new Date(data.date),
+						method: data.method,
+						doc_collection: data.collection,
+						duration: data.duration,
+						query: Buffer.byteLength(data.query, 'utf8') < 15800000
+							? data.query
+							: 'Too Big'
+					}
+				});
+			}
+		}
+	}
+
+	async addFunctionTracker(data: MonitorFunction) {
+		if (typeof(data.data) !== 'string') {
+			data.data = JSON.stringify(data.data);
+		}
+
+		if (
+			!ResolveIOServer.shouldWriteLogsOffline()
+		) {
+			await MonitorFunctions.create({
+				_id: objectIdHexString(),
+				metadata: {
+					instance: this._instanceHostname,
+					client: ResolveIOServer.getServerConfig()['CLIENT_NAME']
+				},
+				date: new Date(data.startTime),
+				type: data.functionType,
+				name: data.functionName,
+				user: data.user,
+				duration: data.duration,
+				data: Buffer.byteLength(data.data, 'utf8') < 15800000
+					? data.data
+					: 'Too Big'
+			});
+		}
+		else {
+			if (this._instanceHostname) {
+				this.pushMonitorData({
+					type: 'monitor-function',
+					data: {
+						metadata: {
+							instance: this._instanceHostname,
+							client: ResolveIOServer.getServerConfig()['CLIENT_NAME']
+						},
+						date: new Date(data.startTime),
+						type: data.functionType,
+						name: data.functionName,
+						user: data.user,
+						duration: data.duration,
+						data: Buffer.byteLength(data.data, 'utf8') < 15800000
+							? data.data
+							: 'Too Big'
+					}
+				});
+			}
+		}
+	}
+
+	private pushMonitorData(data: LocalLogModel) {
+		this._monitorData.push(data);
+
+		// If data is huge, flush immediately to file
+		if (this.estimateBatchSize() >= MAX_BATCH_SIZE) {
+			this.flushMonitorData();
+		}
+	}
+
+	private estimateBatchSize(): number {
+		return this._monitorData.reduce((total, item) => {
+			return total + Buffer.byteLength(JSON.stringify(item));
+		}, 0);
+	}
+
+	setupEventLoop() {
+		// Flush every 2.5 seconds
+		setInterval(() => {
+			if (this._monitorData.length > 0) {
+				this.flushMonitorData();
+			}
+		}, 2500);
+	}
+
+	private flushMonitorData() {
+		const batch = [];
+		let batchSize = 0;
+
+		while (this._monitorData.length > 0 && batchSize < MAX_BATCH_SIZE) {
+			const item = this._monitorData.shift();
+			const itemSize = Buffer.byteLength(JSON.stringify(item));
+
+			if ((batchSize + itemSize) > MAX_BATCH_SIZE && batch.length > 0) {
+				// put the item back for next round
+				this._monitorData.unshift(item);
+				break;
+			}
+
+			batch.push(item);
+			batchSize += itemSize;
+		}
+
+		try {
+			// Now, instead of calling an external HTTP endpoint, we write the batch to disk:
+			ResolveIOServer.getLocalLogManager().writeLogs(batch);
+		}
+		catch (error) {
+			console.error(new Date(), 'MonitorManager', 'Failed to flush monitor data to file:', error);
+		}
+	}
+}
+
+export interface MongoMonitorModel {
+	date: Date;
+	method: string;
+	collection: string;
+	duration: number;
+	query: string;
+}
+
+export class MonitorMongo {
+	private _startTime = 0;
+	private _method = '';
+	private _collection = '';
+	private _query = '';
+
+	constructor() {}
+
+	static create(method: string, collection: string, query: string) {
+		const monitorMongo = new MonitorMongo();
+		monitorMongo.initialize(method, collection, query);
+		return monitorMongo;
+	}
+
+	private initialize(method: string, collection: string, query: string) {
+		this._startTime = Date.now();
+		this._method = method;
+		this._collection = collection;
+		this._query = query;
+	}
+
+	public async finish() {
+		let endTime = Date.now();
+
+		let mongoMonitor: MongoMonitorModel = {
+			date: new Date(endTime),
+			method: this._method,
+			collection: this._collection,
+			query: this._query,
+			duration: endTime - this._startTime
+		};
+
+		await ResolveIOServer.getMainServer().getMonitorManager().addMongoTracker(mongoMonitor);
+	}
+}
+
+export class MonitorManagerFunction {
+	private _functionCnt = 0;
+	private _functions: MonitorFunction[] = [];
+	private _functionLastCompletedWS: { [id_socket: string]: MonitorFunction; } = {};
+	private _functionLastCompleted: MonitorFunction;
+
+	constructor() {}
+
+	static create() {
+		const monitorManagerFunction = new MonitorManagerFunction();
+		monitorManagerFunction.initialize();
+		return monitorManagerFunction;
+	}
+
+	private initialize() {
+
+	}
+
+	startMonitorFunction(
+		functionType: MonitorFunctionType,
+		functionName: string,
+		user: string,
+		id_socket: string,
+		data: any
+	) {
+		let newMonitorFunction = MonitorFunction.create(
+			this._functionCnt++,
+			functionType,
+			functionName,
+			user,
+			id_socket,
+			data
+		);
+
+		this._functions.push(newMonitorFunction);
+		return newMonitorFunction.id;
+	}
+
+	async finishMonitorFunction(id: number) {
+		let monitor = this._functions.find(a => a.id === id);
+
+		if (monitor) {
+			await monitor.finish();
+			this._functionLastCompleted = monitor;
+			this._functionLastCompletedWS[monitor.id_socket] = monitor;
+			this._functions.splice(
+				this._functions.map(a => a.id).indexOf(id),
+				1
+			);
+		}
+	}
+
+	getActiveMonitorFunctions() {
+		return this._functions;
+	}
+
+	getLastCompletedMonitorFunction() {
+		return this._functionLastCompleted;
+	}
+
+	getLastCompletedMonitorFunctionWS() {
+		return this._functionLastCompletedWS;
+	}
+}
+
+export type MonitorFunctionType =
+	'Cron Method'
+	| 'Method'
+	| 'User Specific Publication'
+	| 'Publication';
+
+export class MonitorFunction {
+	private _timer: NodeJS.Timeout = null;
+	public duration = 0;
+	public functionType: MonitorFunctionType;
+	public functionName: string;
+	public user: string;
+	public data: any;
+	public startTime: Date = null;
+	public endTime: Date = null;
+	public lastTime: Date = null;
+	public id_socket = '';
+	public id = 0;
+
+	constructor() {}
+
+	static create(
+		id: number,
+		functionType: MonitorFunctionType,
+		functionName: string,
+		user: string,
+		id_socket: string,
+		data: any
+	) {
+		const monitorFunction = new MonitorFunction();
+		monitorFunction.initialize(id, functionType, functionName, user, id_socket, data);
+		return monitorFunction;
+	}
+
+	private initialize(
+		id: number,
+		functionType: MonitorFunctionType,
+		functionName: string,
+		user: string,
+		id_socket: string,
+		data: any
+	) {
+		this.id = id;
+		this.startTime = new Date();
+		this.functionType = functionType;
+		this.functionName = functionName;
+		this.user = user;
+		this.id_socket = id_socket;
+		this.data = data;
+		this.lastTime = new Date();
+
+		this._timer = setInterval(() => {
+			let diff = Date.now() - this.lastTime.getTime();
+
+			if (diff > this.duration) {
+				this.duration = diff;
+			}
+
+			this.lastTime = new Date();
+		}, 1000);
+	}
+
+	public async finish() {
+		if (this._timer) {
+			clearInterval(this._timer);
+			this._timer = null;
+		}
+
+		this.endTime = new Date();
+
+		if (this.duration >= 5000) {
+			// Optional slow function logging or email
+			await ResolveIOServer.getMainServer().getMonitorManager().addFunctionTracker(this);
+		}
+
+		// await ResolveIOServer.getMainServer().getMonitorManager().addFunctionTracker(this);
+
+		return Promise.resolve(true);
+	}
+}

package/src/managers/openai-usage-ledger.manager.ts

@@ -0,0 +1,112 @@
+import { OpenAIUsageLedger } from '../collections/openai-usage-ledger.collection';
+import {
+	OpenAIUsageCostBasis,
+	OpenAIUsageSource,
+	OpenAIUsageSurface
+} from '../models/openai-usage-ledger.model';
+import { estimateOpenAIUsageCost } from '../util/openai-usage-cost';
+export { estimateOpenAIUsageCost, OpenAIUsagePricingConfig } from '../util/openai-usage-cost';
+
+export interface OpenAIUsageRecordInput {
+	id_client?: string;
+	model: string;
+	input_tokens: number;
+	cached_input_tokens?: number;
+	output_tokens: number;
+	total_tokens: number;
+	billable?: boolean;
+	category?: string;
+	id_request?: string;
+	id_conversation?: string;
+	timestamp?: Date;
+	cost_estimate?: number;
+	usage_source?: OpenAIUsageSource | string;
+	usage_surface?: OpenAIUsageSurface | string;
+	usage_phase?: string;
+	cost_basis?: OpenAIUsageCostBasis | string;
+	id_run?: string;
+	id_job?: string;
+	id_app?: string;
+	id_ticket?: string;
+	is_manual?: boolean;
+	is_untracked?: boolean;
+	untracked_reason?: string;
+	metadata?: Record<string, any>;
+}
+
+const toNumber = (value: any): number => {
+	const parsed = Number(value);
+	return Number.isFinite(parsed) ? parsed : 0;
+};
+
+const cleanText = (value: any, max = 240): string => {
+	if (value === undefined || value === null) {
+		return '';
+	}
+	return String(value).trim().slice(0, max);
+};
+
+const cleanMetadata = (value: any): Record<string, any> | undefined => {
+	if (!value || typeof value !== 'object' || Array.isArray(value)) {
+		return undefined;
+	}
+	return { ...value };
+};
+
+export async function recordOpenAIUsage(input: OpenAIUsageRecordInput): Promise<void> {
+	if (!input) {
+		return;
+	}
+	const idClient = String(input.id_client || '').trim();
+	const inputTokens = toNumber(input.input_tokens);
+	const cachedInputTokens = toNumber(input.cached_input_tokens);
+	const outputTokens = toNumber(input.output_tokens);
+	const totalTokens = toNumber(input.total_tokens);
+	const normalizedInputCostEstimate = toNumber(input.cost_estimate);
+	if (!totalTokens && (!inputTokens && !outputTokens) && normalizedInputCostEstimate <= 0) {
+		return;
+	}
+	const model = String(input.model || '').trim() || 'unknown';
+	const costEstimate = normalizedInputCostEstimate > 0
+		? normalizedInputCostEstimate
+		: estimateOpenAIUsageCost(model, inputTokens, outputTokens, cachedInputTokens);
+
+	await OpenAIUsageLedger.insertOne({
+		id_client: idClient,
+		timestamp: input.timestamp || new Date(),
+		model,
+		input_tokens: inputTokens,
+		cached_input_tokens: cachedInputTokens,
+		output_tokens: outputTokens,
+		total_tokens: totalTokens || (inputTokens + outputTokens),
+		cost_estimate: costEstimate,
+		billable: input.billable !== false,
+		category: cleanText(input.category, 160),
+		id_request: cleanText(input.id_request, 240),
+		id_conversation: cleanText(input.id_conversation, 240),
+		usage_source: cleanText(input.usage_source, 80) || 'unknown',
+		usage_surface: cleanText(input.usage_surface, 80) || 'unknown',
+		usage_phase: cleanText(input.usage_phase, 120),
+		cost_basis: cleanText(input.cost_basis, 80) || (normalizedInputCostEstimate > 0 ? 'provider_usage' : 'estimated_tokens'),
+		id_run: cleanText(input.id_run, 240),
+		id_job: cleanText(input.id_job, 240),
+		id_app: cleanText(input.id_app, 240),
+		id_ticket: cleanText(input.id_ticket, 240),
+		is_manual: input.is_manual === true || cleanText(input.usage_source, 80) === 'manual_codex',
+		is_untracked: input.is_untracked === true,
+		untracked_reason: cleanText(input.untracked_reason, 500),
+		metadata: cleanMetadata(input.metadata)
+	});
+}
+
+export async function recordManualCodexUsageEstimate(input: Omit<OpenAIUsageRecordInput, 'usage_source' | 'is_manual'>): Promise<void> {
+	await recordOpenAIUsage({
+		...input,
+		usage_source: 'manual_codex',
+		usage_surface: input.usage_surface || 'manual',
+		cost_basis: input.cost_basis || 'manual_estimate',
+		is_manual: true,
+		is_untracked: input.is_untracked !== false,
+		untracked_reason: input.untracked_reason || 'Manual Codex session was outside the managed runner telemetry path.'
+	});
+}