@geekmidas/telescope 10.0.0-alpha.98 → 10.0.0

package/src/metrics/__tests__/MetricsAggregator.spec.ts

@@ -0,0 +1,298 @@
+import { beforeEach, describe, expect, it } from 'vitest';
+import type { RequestEntry } from '../../types';
+import { MetricsAggregator } from '../MetricsAggregator';
+
+function createRequest(overrides: Partial<RequestEntry> = {}): RequestEntry {
+	return {
+		id: `req-${Math.random().toString(36).slice(2)}`,
+		method: 'GET',
+		path: '/api/test',
+		url: 'http://localhost/api/test',
+		headers: {},
+		query: {},
+		status: 200,
+		responseHeaders: {},
+		duration: 50,
+		timestamp: new Date(),
+		...overrides,
+	};
+}
+
+describe('MetricsAggregator', () => {
+	let aggregator: MetricsAggregator;
+
+	beforeEach(() => {
+		aggregator = new MetricsAggregator();
+	});
+
+	describe('record', () => {
+		it('should record a request', () => {
+			const request = createRequest();
+			aggregator.record(request);
+
+			const metrics = aggregator.getMetrics();
+			expect(metrics.totalRequests).toBe(1);
+		});
+
+		it('should aggregate multiple requests', () => {
+			aggregator.record(createRequest({ duration: 100 }));
+			aggregator.record(createRequest({ duration: 200 }));
+			aggregator.record(createRequest({ duration: 300 }));
+
+			const metrics = aggregator.getMetrics();
+			expect(metrics.totalRequests).toBe(3);
+			expect(metrics.avgDuration).toBe(200);
+		});
+
+		it('should track error requests', () => {
+			aggregator.record(createRequest({ status: 200 }));
+			aggregator.record(createRequest({ status: 404 }));
+			aggregator.record(createRequest({ status: 500 }));
+			aggregator.record(createRequest({ status: 200 }));
+
+			const metrics = aggregator.getMetrics();
+			expect(metrics.totalRequests).toBe(4);
+			expect(metrics.errorRate).toBe(50); // 2 out of 4
+			expect(metrics.successRate).toBe(50);
+		});
+	});
+
+	describe('getMetrics', () => {
+		it('should return empty metrics when no requests', () => {
+			const metrics = aggregator.getMetrics();
+
+			expect(metrics.totalRequests).toBe(0);
+			expect(metrics.avgDuration).toBe(0);
+			expect(metrics.p50Duration).toBe(0);
+			expect(metrics.p95Duration).toBe(0);
+			expect(metrics.p99Duration).toBe(0);
+			expect(metrics.errorRate).toBe(0);
+			expect(metrics.successRate).toBe(100);
+			expect(metrics.requestsPerSecond).toBe(0);
+			expect(metrics.timeSeries).toEqual([]);
+		});
+
+		it('should calculate percentiles correctly', () => {
+			// Add 100 requests with durations 1-100ms
+			for (let i = 1; i <= 100; i++) {
+				aggregator.record(createRequest({ duration: i }));
+			}
+
+			const metrics = aggregator.getMetrics();
+
+			// p50 should be around 50 (median of 1-100 is 50.5)
+			expect(metrics.p50Duration).toBeGreaterThanOrEqual(49);
+			expect(metrics.p50Duration).toBeLessThanOrEqual(52);
+			// p95 should be around 95
+			expect(metrics.p95Duration).toBeGreaterThanOrEqual(94);
+			expect(metrics.p95Duration).toBeLessThanOrEqual(96);
+			// p99 should be around 99
+			expect(metrics.p99Duration).toBeGreaterThanOrEqual(98);
+			expect(metrics.p99Duration).toBeLessThanOrEqual(100);
+		});
+
+		it('should calculate requests per second', () => {
+			// Record 10 requests
+			for (let i = 0; i < 10; i++) {
+				aggregator.record(createRequest());
+			}
+
+			// Use default range (last hour) - just verify it calculates something reasonable
+			const metrics = aggregator.getMetrics();
+
+			expect(metrics.totalRequests).toBe(10);
+			// With 10 requests in the default 1 hour range, rps should be small but positive
+			expect(metrics.requestsPerSecond).toBeGreaterThan(0);
+		});
+
+		it('should generate time series data', () => {
+			const now = new Date();
+			const bucketSize = 60000; // 1 minute
+
+			// Record requests in different buckets
+			aggregator.record(
+				createRequest({
+					timestamp: new Date(now.getTime() - 2 * bucketSize),
+				}),
+			);
+			aggregator.record(
+				createRequest({
+					timestamp: new Date(now.getTime() - bucketSize),
+				}),
+			);
+			aggregator.record(createRequest({ timestamp: now }));
+
+			const metrics = aggregator.getMetrics({
+				range: {
+					start: new Date(now.getTime() - 3 * bucketSize),
+					end: now,
+				},
+			});
+
+			expect(metrics.timeSeries.length).toBeGreaterThan(0);
+		});
+	});
+
+	describe('getEndpointMetrics', () => {
+		it('should track metrics per endpoint', () => {
+			aggregator.record(createRequest({ method: 'GET', path: '/users' }));
+			aggregator.record(createRequest({ method: 'GET', path: '/users' }));
+			aggregator.record(createRequest({ method: 'POST', path: '/users' }));
+			aggregator.record(createRequest({ method: 'GET', path: '/posts' }));
+
+			const endpoints = aggregator.getEndpointMetrics();
+
+			expect(endpoints.length).toBe(3);
+
+			const getUsersEndpoint = endpoints.find(
+				(e) => e.method === 'GET' && e.path === '/users',
+			);
+			expect(getUsersEndpoint?.count).toBe(2);
+
+			const postUsersEndpoint = endpoints.find(
+				(e) => e.method === 'POST' && e.path === '/users',
+			);
+			expect(postUsersEndpoint?.count).toBe(1);
+		});
+
+		it('should sort by request count descending', () => {
+			aggregator.record(createRequest({ path: '/a' }));
+			aggregator.record(createRequest({ path: '/b' }));
+			aggregator.record(createRequest({ path: '/b' }));
+			aggregator.record(createRequest({ path: '/c' }));
+			aggregator.record(createRequest({ path: '/c' }));
+			aggregator.record(createRequest({ path: '/c' }));
+
+			const endpoints = aggregator.getEndpointMetrics();
+
+			expect(endpoints[0].path).toBe('/c');
+			expect(endpoints[0].count).toBe(3);
+			expect(endpoints[1].path).toBe('/b');
+			expect(endpoints[1].count).toBe(2);
+			expect(endpoints[2].path).toBe('/a');
+			expect(endpoints[2].count).toBe(1);
+		});
+
+		it('should respect limit option', () => {
+			for (let i = 0; i < 10; i++) {
+				aggregator.record(createRequest({ path: `/path-${i}` }));
+			}
+
+			const endpoints = aggregator.getEndpointMetrics({ limit: 5 });
+			expect(endpoints.length).toBe(5);
+		});
+
+		it('should calculate endpoint error rates', () => {
+			aggregator.record(createRequest({ path: '/error', status: 500 }));
+			aggregator.record(createRequest({ path: '/error', status: 500 }));
+			aggregator.record(createRequest({ path: '/error', status: 200 }));
+			aggregator.record(createRequest({ path: '/error', status: 200 }));
+
+			const endpoints = aggregator.getEndpointMetrics();
+			const errorEndpoint = endpoints.find((e) => e.path === '/error');
+
+			expect(errorEndpoint?.errorRate).toBe(50);
+		});
+	});
+
+	describe('getStatusDistribution', () => {
+		it('should count status codes by category', () => {
+			aggregator.record(createRequest({ status: 200 }));
+			aggregator.record(createRequest({ status: 201 }));
+			aggregator.record(createRequest({ status: 301 }));
+			aggregator.record(createRequest({ status: 400 }));
+			aggregator.record(createRequest({ status: 404 }));
+			aggregator.record(createRequest({ status: 500 }));
+
+			const dist = aggregator.getStatusDistribution();
+
+			expect(dist['2xx']).toBe(2);
+			expect(dist['3xx']).toBe(1);
+			expect(dist['4xx']).toBe(2);
+			expect(dist['5xx']).toBe(1);
+		});
+
+		it('should return zeros when no requests', () => {
+			const dist = aggregator.getStatusDistribution();
+
+			expect(dist['2xx']).toBe(0);
+			expect(dist['3xx']).toBe(0);
+			expect(dist['4xx']).toBe(0);
+			expect(dist['5xx']).toBe(0);
+		});
+	});
+
+	describe('reset', () => {
+		it('should clear all metrics', () => {
+			aggregator.record(createRequest());
+			aggregator.record(createRequest());
+			aggregator.record(createRequest());
+
+			expect(aggregator.getMetrics().totalRequests).toBe(3);
+
+			aggregator.reset();
+
+			expect(aggregator.getMetrics().totalRequests).toBe(0);
+			expect(aggregator.getEndpointMetrics().length).toBe(0);
+		});
+	});
+
+	describe('bucket pruning', () => {
+		it('should prune old buckets when maxBuckets exceeded', () => {
+			const aggregator = new MetricsAggregator({
+				bucketSize: 1000, // 1 second buckets
+				maxBuckets: 5,
+			});
+
+			// Add requests across 10 different seconds
+			const baseTime = Date.now();
+			for (let i = 0; i < 10; i++) {
+				aggregator.record(
+					createRequest({
+						timestamp: new Date(baseTime + i * 1000),
+					}),
+				);
+			}
+
+			// Query should only return data from the last 5 buckets
+			const metrics = aggregator.getMetrics({
+				range: {
+					start: new Date(baseTime),
+					end: new Date(baseTime + 10000),
+				},
+			});
+
+			// Should have at most 5 buckets worth of data
+			expect(metrics.totalRequests).toBeLessThanOrEqual(5);
+		});
+	});
+
+	describe('custom bucket size', () => {
+		it('should use custom bucket size', () => {
+			const aggregator = new MetricsAggregator({
+				bucketSize: 5000, // 5 second buckets
+			});
+
+			// Use a fixed timestamp to ensure all requests fall in the same bucket
+			const baseTime = Math.floor(Date.now() / 5000) * 5000; // Align to bucket boundary
+			const timestamp = new Date(baseTime);
+
+			// Add 3 requests within the same 5-second bucket
+			aggregator.record(createRequest({ timestamp }));
+			aggregator.record(createRequest({ timestamp }));
+			aggregator.record(createRequest({ timestamp }));
+
+			const metrics = aggregator.getMetrics({
+				range: {
+					start: new Date(baseTime - 1000),
+					end: new Date(baseTime + 6000),
+				},
+				bucketSize: 5000,
+			});
+
+			// Should have at least 1 time series point with 3 requests total
+			expect(metrics.totalRequests).toBe(3);
+			expect(metrics.timeSeries.length).toBeGreaterThanOrEqual(1);
+		});
+	});
+});

package/src/metrics/index.ts

@@ -0,0 +1,2 @@
+export type { MetricsAggregatorOptions } from './MetricsAggregator';
+export { MetricsAggregator } from './MetricsAggregator';

package/src/metrics/storage/InMemoryMetricsStorage.ts

@@ -0,0 +1,380 @@
+import type { MetricsStorage } from './MetricsStorage';
+import {
+	type AggregationInterval,
+	DEFAULT_METRICS_CONFIG,
+	type MetricBucket,
+	type MetricLabels,
+	type MetricPoint,
+	type MetricQuery,
+	type MetricResult,
+	type MetricsStorageConfig,
+	type PartitionInfo,
+	type RollupOptions,
+} from './types';
+
+/**
+ * In-memory metrics storage for development and testing.
+ *
+ * Features:
+ * - Buffered writes with configurable flush interval
+ * - Time-bucketed aggregation
+ * - Basic percentile calculation (reservoir sampling)
+ * - Automatic pruning based on retention
+ *
+ * Not suitable for production - use a proper time-series database.
+ */
+export class InMemoryMetricsStorage implements MetricsStorage {
+	private config: MetricsStorageConfig;
+	private buffer: MetricPoint[] = [];
+	private buckets: Map<string, MetricBucket> = new Map();
+	private samples: Map<string, number[]> = new Map(); // For percentile calculations
+	private flushTimer: ReturnType<typeof setInterval> | null = null;
+
+	constructor(config: Partial<MetricsStorageConfig> = {}) {
+		this.config = { ...DEFAULT_METRICS_CONFIG, ...config };
+
+		if (this.config.flushInterval > 0) {
+			this.flushTimer = setInterval(() => {
+				this.flush().catch(console.error);
+			}, this.config.flushInterval);
+		}
+	}
+
+	async write(points: MetricPoint[]): Promise<void> {
+		this.buffer.push(...points);
+
+		if (this.buffer.length >= this.config.maxBatchSize) {
+			await this.flush();
+		}
+	}
+
+	async flush(): Promise<void> {
+		if (this.buffer.length === 0) return;
+
+		const points = this.buffer.splice(0);
+
+		for (const point of points) {
+			this.aggregatePoint(point);
+		}
+
+		// Prune if auto-rollup is enabled
+		if (this.config.autoRollup) {
+			await this.prune();
+		}
+	}
+
+	private aggregatePoint(point: MetricPoint): void {
+		const bucketTime = this.getBucketTime(point.timestamp, '1m');
+		const key = this.getBucketKey(point, bucketTime);
+
+		const existing = this.buckets.get(key);
+
+		if (existing) {
+			existing.count += 1;
+			existing.sum += point.value;
+			existing.min = Math.min(existing.min, point.value);
+			existing.max = Math.max(existing.max, point.value);
+		} else {
+			this.buckets.set(key, {
+				bucket: bucketTime,
+				projectId: point.projectId,
+				service: point.service,
+				environment: point.environment,
+				name: point.name,
+				labelsHash: this.hashLabels(point.labels),
+				labels: { ...point.labels },
+				count: 1,
+				sum: point.value,
+				min: point.value,
+				max: point.value,
+			});
+		}
+
+		// Store samples for percentile calculations (reservoir sampling)
+		const sampleKey = `${point.projectId}:${point.name}:${bucketTime.getTime()}`;
+		const samples = this.samples.get(sampleKey) ?? [];
+		if (samples.length < 1000) {
+			samples.push(point.value);
+		} else {
+			// Reservoir sampling
+			const idx = Math.floor(Math.random() * (samples.length + 1));
+			if (idx < samples.length) {
+				samples[idx] = point.value;
+			}
+		}
+		this.samples.set(sampleKey, samples);
+	}
+
+	async query(query: MetricQuery): Promise<MetricResult[]> {
+		await this.flush();
+
+		const results = new Map<string, MetricResult>();
+		const startTime = query.range.start.getTime();
+		const endTime = query.range.end.getTime();
+		const interval = query.interval ?? '1m';
+		const intervalMs = this.getIntervalMs(interval);
+
+		for (const [, bucket] of this.buckets) {
+			if (bucket.projectId !== query.projectId) continue;
+			if (query.service && bucket.service !== query.service) continue;
+			if (query.environment && bucket.environment !== query.environment)
+				continue;
+			if (query.name && !this.matchesPattern(bucket.name, query.name)) continue;
+			if (query.labels && !this.matchesLabels(bucket.labels, query.labels))
+				continue;
+
+			const bucketTime = bucket.bucket.getTime();
+			if (bucketTime < startTime || bucketTime > endTime) continue;
+
+			// Group by metric name + labels
+			const groupKey = query.groupBy
+				? this.getGroupKey(bucket, query.groupBy)
+				: bucket.name;
+
+			let result = results.get(groupKey);
+			if (!result) {
+				result = {
+					name: bucket.name,
+					labels: query.groupBy
+						? this.filterLabels(bucket.labels, query.groupBy)
+						: bucket.labels,
+					timeSeries: [],
+					stats: {
+						count: 0,
+						sum: 0,
+						avg: 0,
+						min: Infinity,
+						max: -Infinity,
+						p50: 0,
+						p95: 0,
+						p99: 0,
+					},
+				};
+				results.set(groupKey, result);
+			}
+
+			// Aggregate into time series
+			const tsIndex = Math.floor((bucketTime - startTime) / intervalMs);
+			while (result.timeSeries.length <= tsIndex) {
+				result.timeSeries.push({
+					timestamp: new Date(
+						startTime + result.timeSeries.length * intervalMs,
+					),
+					count: 0,
+					sum: 0,
+					avg: 0,
+					min: Infinity,
+					max: -Infinity,
+				});
+			}
+
+			const tsPoint = result.timeSeries[tsIndex];
+			if (tsPoint) {
+				tsPoint.count += bucket.count;
+				tsPoint.sum += bucket.sum;
+				tsPoint.min = Math.min(tsPoint.min, bucket.min);
+				tsPoint.max = Math.max(tsPoint.max, bucket.max);
+				tsPoint.avg = tsPoint.sum / tsPoint.count;
+			}
+
+			// Update overall stats
+			result.stats.count += bucket.count;
+			result.stats.sum += bucket.sum;
+			result.stats.min = Math.min(result.stats.min, bucket.min);
+			result.stats.max = Math.max(result.stats.max, bucket.max);
+		}
+
+		// Calculate percentiles and finalize
+		for (const result of results.values()) {
+			if (result.stats.count > 0) {
+				result.stats.avg = result.stats.sum / result.stats.count;
+
+				// Gather all samples for this result
+				const allSamples: number[] = [];
+				for (const [key, samples] of this.samples) {
+					if (key.startsWith(`${query.projectId}:${result.name}:`)) {
+						allSamples.push(...samples);
+					}
+				}
+
+				if (allSamples.length > 0) {
+					allSamples.sort((a, b) => a - b);
+					result.stats.p50 = this.percentile(allSamples, 50);
+					result.stats.p95 = this.percentile(allSamples, 95);
+					result.stats.p99 = this.percentile(allSamples, 99);
+				}
+			}
+
+			// Fix infinity values
+			if (result.stats.min === Infinity) result.stats.min = 0;
+			if (result.stats.max === -Infinity) result.stats.max = 0;
+
+			for (const point of result.timeSeries) {
+				if (point.min === Infinity) point.min = 0;
+				if (point.max === -Infinity) point.max = 0;
+			}
+		}
+
+		const resultArray = Array.from(results.values());
+
+		if (query.limit) {
+			return resultArray.slice(0, query.limit);
+		}
+
+		return resultArray;
+	}
+
+	async getBucket(
+		projectId: string,
+		name: string,
+		bucket: Date,
+	): Promise<MetricBucket | null> {
+		await this.flush();
+
+		for (const [, b] of this.buckets) {
+			if (
+				b.projectId === projectId &&
+				b.name === name &&
+				b.bucket.getTime() === bucket.getTime()
+			) {
+				return b;
+			}
+		}
+
+		return null;
+	}
+
+	async rollup(_options: RollupOptions): Promise<number> {
+		// In-memory storage doesn't need rollup - data is already aggregated
+		return 0;
+	}
+
+	async prune(): Promise<number> {
+		const now = Date.now();
+		let deleted = 0;
+
+		// Use the shortest retention from tiers
+		const retentionMs = Math.min(
+			...this.config.tiers
+				.filter((t) => t.retention > 0)
+				.map((t) => t.retention),
+		);
+
+		if (retentionMs === Infinity) return 0;
+
+		const cutoff = now - retentionMs;
+
+		for (const [key, bucket] of this.buckets) {
+			if (bucket.bucket.getTime() < cutoff) {
+				this.buckets.delete(key);
+				deleted++;
+			}
+		}
+
+		// Also prune samples
+		for (const [key] of this.samples) {
+			const timestamp = parseInt(key.split(':').pop() ?? '0', 10);
+			if (timestamp < cutoff) {
+				this.samples.delete(key);
+			}
+		}
+
+		return deleted;
+	}
+
+	async listPartitions(_tier?: string): Promise<PartitionInfo[]> {
+		// In-memory storage doesn't use partitions
+		return [];
+	}
+
+	getConfig(): MetricsStorageConfig {
+		return this.config;
+	}
+
+	async close(): Promise<void> {
+		if (this.flushTimer) {
+			clearInterval(this.flushTimer);
+			this.flushTimer = null;
+		}
+		await this.flush();
+		this.buckets.clear();
+		this.samples.clear();
+	}
+
+	// Helper methods
+
+	private getBucketTime(timestamp: Date, interval: AggregationInterval): Date {
+		const ms = this.getIntervalMs(interval);
+		return new Date(Math.floor(timestamp.getTime() / ms) * ms);
+	}
+
+	private getBucketKey(point: MetricPoint, bucketTime: Date): string {
+		return [
+			point.projectId,
+			point.service,
+			point.environment,
+			point.name,
+			this.hashLabels(point.labels),
+			bucketTime.getTime(),
+		].join(':');
+	}
+
+	private hashLabels(labels: MetricLabels): string {
+		const sorted = Object.keys(labels)
+			.sort()
+			.map((k) => `${k}=${labels[k]}`)
+			.join(',');
+		return sorted || '_empty_';
+	}
+
+	private getIntervalMs(interval: AggregationInterval): number {
+		const map: Record<AggregationInterval, number> = {
+			'1m': 60_000,
+			'5m': 300_000,
+			'15m': 900_000,
+			'1h': 3_600_000,
+			'6h': 21_600_000,
+			'1d': 86_400_000,
+		};
+		return map[interval];
+	}
+
+	private matchesPattern(name: string, pattern: string): boolean {
+		if (!pattern.includes('*')) return name === pattern;
+		const regex = new RegExp(
+			`^${pattern.replace(/\./g, '\\.').replace(/\*/g, '.*')}$`,
+		);
+		return regex.test(name);
+	}
+
+	private matchesLabels(labels: MetricLabels, filter: MetricLabels): boolean {
+		for (const [key, value] of Object.entries(filter)) {
+			if (labels[key] !== value) return false;
+		}
+		return true;
+	}
+
+	private getGroupKey(bucket: MetricBucket, groupBy: string[]): string {
+		const parts = [bucket.name];
+		for (const key of groupBy) {
+			parts.push(`${key}=${bucket.labels[key] ?? ''}`);
+		}
+		return parts.join(':');
+	}
+
+	private filterLabels(labels: MetricLabels, keys: string[]): MetricLabels {
+		const result: MetricLabels = {};
+		for (const key of keys) {
+			if (labels[key] !== undefined) {
+				result[key] = labels[key];
+			}
+		}
+		return result;
+	}
+
+	private percentile(sorted: number[], p: number): number {
+		if (sorted.length === 0) return 0;
+		const index = Math.ceil((p / 100) * sorted.length) - 1;
+		return sorted[Math.max(0, index)] ?? 0;
+	}
+}