@gguf/coder 0.3.0 → 0.3.1

package/source/utils/logging/log-query/storage/log-storage.ts

@@ -0,0 +1,80 @@
+/**
+ * In-memory log storage for querying
+ */
+
+import {MAX_LOG_ENTRIES} from '@/constants';
+import {aggregateLogEntries} from '../aggregation/aggregator';
+import {executeQuery} from '../query/query-engine';
+import type {
+	AggregationOptions,
+	AggregationResult,
+	LogEntry,
+	LogQuery,
+	QueryResult,
+} from '../types';
+import {CircularBuffer} from './circular-buffer';
+import {IndexManager} from './index-manager';
+
+/**
+ * In-memory log storage for querying (in production, this would connect to a log database)
+ */
+export class LogStorage {
+	private buffer: CircularBuffer;
+	private indexManager: IndexManager;
+
+	constructor(maxEntries: number = MAX_LOG_ENTRIES) {
+		this.buffer = new CircularBuffer(maxEntries);
+		this.indexManager = new IndexManager();
+	}
+
+	/**
+	 * Add a log entry - O(1) instead of O(n)
+	 */
+	addEntry(entry: LogEntry): void {
+		// Add to circular buffer and get removed entry if any
+		const removed = this.buffer.add(entry);
+
+		// Update indexes
+		if (removed) {
+			this.indexManager.updateIndexes(removed, false);
+		}
+		this.indexManager.updateIndexes(entry, true);
+	}
+
+	/**
+	 * Query log entries
+	 */
+	query(query: LogQuery): QueryResult {
+		const entries = this.buffer.getAll();
+		const totalCount = this.buffer.getCount();
+		return executeQuery(entries, query, totalCount);
+	}
+
+	/**
+	 * Aggregate log entries
+	 */
+	aggregate(options: AggregationOptions): AggregationResult {
+		const entries = this.buffer.getAll();
+		return aggregateLogEntries(entries, options);
+	}
+
+	/**
+	 * Get entry count
+	 */
+	getEntryCount(): number {
+		return this.buffer.getCount();
+	}
+
+	/**
+	 * Clear all entries
+	 */
+	clear(): void {
+		this.buffer.clear();
+		this.indexManager.clear();
+	}
+}
+
+/**
+ * Global log storage instance
+ */
+export const globalLogStorage = new LogStorage();

package/source/utils/logging/log-query/types.ts

@@ -0,0 +1,163 @@
+/**
+ * Type definitions for log query system
+ */
+
+/**
+ * Log entry structure for querying
+ */
+export interface LogEntry {
+	timestamp: string;
+	level: string;
+	message: string;
+	correlationId?: string;
+	source?: string;
+	requestId?: string;
+	userId?: string;
+	sessionId?: string;
+	tags?: string[];
+	// biome-ignore lint/suspicious/noExplicitAny: Dynamic log filter
+	metadata?: Record<string, any>;
+	error?: {
+		name?: string;
+		message?: string;
+		stack?: string;
+		type?: string;
+	};
+	performance?: {
+		duration?: number;
+		// biome-ignore lint/suspicious/noExplicitAny: Dynamic log filter
+		memory?: Record<string, any>;
+		cpu?: number;
+	};
+	request?: {
+		method?: string;
+		url?: string;
+		statusCode?: number;
+		duration?: number;
+		size?: number;
+	};
+}
+
+/**
+ * Query filters for log searching
+ */
+export interface LogQuery {
+	// Time range
+	startTime?: Date;
+	endTime?: Date;
+
+	// Log level filtering
+	levels?: string[];
+	excludeLevels?: string[];
+
+	// Message filtering
+	messageContains?: string;
+	messageRegex?: RegExp;
+	messageStartsWith?: string;
+	messageEndsWith?: string;
+
+	// Correlation and request filtering
+	correlationIds?: string[];
+	requestIds?: string[];
+	userIds?: string[];
+	sessionIds?: string[];
+
+	// Source and component filtering
+	sources?: string[];
+	excludeSources?: string[];
+
+	// Tag filtering
+	tags?: string[];
+	excludeTags?: string[];
+	hasTags?: boolean;
+
+	// Metadata filtering
+	metadataKey?: string;
+	// biome-ignore lint/suspicious/noExplicitAny: Dynamic filter type
+	metadataValue?: any;
+	metadataExists?: string;
+
+	// Performance filtering
+	durationMin?: number;
+	durationMax?: number;
+	memoryThreshold?: number;
+
+	// Error filtering
+	hasErrors?: boolean;
+	errorTypes?: string[];
+
+	// Request filtering
+	requestMethods?: string[];
+	requestStatusCodes?: number[];
+	requestDurationMin?: number;
+	requestDurationMax?: number;
+
+	// Pagination
+	limit?: number;
+	offset?: number;
+	sortBy?: 'timestamp' | 'level' | 'duration' | 'memory';
+	sortOrder?: 'asc' | 'desc';
+}
+
+/**
+ * Query result with metadata
+ */
+export interface QueryResult {
+	entries: LogEntry[];
+	totalCount: number;
+	filteredCount: number;
+	queryTime: number;
+	hasMore: boolean;
+	facets?: {
+		levels: Record<string, number>;
+		sources: Record<string, number>;
+		tags: Record<string, number>;
+		errorTypes: Record<string, number>;
+		hours: Record<string, number>;
+	};
+}
+
+/**
+ * Log aggregation options
+ */
+export interface AggregationOptions {
+	groupBy: 'hour' | 'day' | 'level' | 'source' | 'correlationId' | 'errorType';
+	aggregations: (
+		| 'count'
+		| 'avgDuration'
+		| 'maxDuration'
+		| 'minDuration'
+		| 'sumDuration'
+		| 'errorRate'
+		| 'memoryUsage'
+	)[];
+	timeRange?: {
+		startTime: Date;
+		endTime: Date;
+	};
+}
+
+/**
+ * Log aggregation result
+ */
+export interface AggregationResult {
+	groups: Record<
+		string,
+		{
+			count: number;
+			avgDuration?: number;
+			maxDuration?: number;
+			minDuration?: number;
+			sumDuration?: number;
+			errorRate?: number;
+			memoryUsage?: {
+				avgHeapUsed: number;
+				maxHeapUsed: number;
+				minHeapUsed: number;
+			};
+			samples?: LogEntry[];
+		}
+	>;
+	totalGroups: number;
+	queryTime: number;
+}

package/source/utils/logging/log-query/utils/helpers.spec.ts

@@ -0,0 +1,263 @@
+/**
+ * Tests for helper functions
+ */
+
+import test from 'ava';
+import {logQueries} from './helpers.js';
+import {globalLogStorage} from '../storage/log-storage.js';
+import type {LogEntry} from '../types.js';
+
+// Helper to create test log entry
+function createLogEntry(overrides: Partial<LogEntry> = {}): LogEntry {
+	return {
+		timestamp: new Date().toISOString(),
+		level: 'info',
+		message: 'Test message',
+		...overrides,
+	};
+}
+
+// Clear storage before each test
+test.beforeEach(() => {
+	globalLogStorage.clear();
+});
+
+test('logQueries.errors: returns error and fatal logs', t => {
+	globalLogStorage.addEntry(createLogEntry({level: 'error'}));
+	globalLogStorage.addEntry(createLogEntry({level: 'fatal'}));
+	globalLogStorage.addEntry(createLogEntry({level: 'info'}));
+	globalLogStorage.addEntry(createLogEntry({level: 'error'}));
+
+	const result = logQueries.errors();
+
+	t.is(result.entries.length, 3); // 2 errors + 1 fatal
+	t.true(
+		result.entries.every(
+			e => e.level === 'error' || e.level === 'fatal',
+		),
+	);
+});
+
+test('logQueries.errors: respects custom limit', t => {
+	for (let i = 0; i < 10; i++) {
+		globalLogStorage.addEntry(createLogEntry({level: 'error'}));
+	}
+
+	const result = logQueries.errors(5);
+
+	t.is(result.entries.length, 5);
+});
+
+test('logQueries.byCorrelation: filters by correlation ID', t => {
+	globalLogStorage.addEntry(createLogEntry({correlationId: 'corr-1'}));
+	globalLogStorage.addEntry(createLogEntry({correlationId: 'corr-2'}));
+	globalLogStorage.addEntry(createLogEntry({correlationId: 'corr-1'}));
+
+	const result = logQueries.byCorrelation('corr-1');
+
+	t.is(result.entries.length, 2);
+	t.true(result.entries.every(e => e.correlationId === 'corr-1'));
+});
+
+test('logQueries.byCorrelation: sorts ascending by timestamp', t => {
+	globalLogStorage.addEntry(
+		createLogEntry({
+			correlationId: 'corr-1',
+			timestamp: '2025-01-01T12:00:00Z',
+		}),
+	);
+	globalLogStorage.addEntry(
+		createLogEntry({
+			correlationId: 'corr-1',
+			timestamp: '2025-01-01T10:00:00Z',
+		}),
+	);
+	globalLogStorage.addEntry(
+		createLogEntry({
+			correlationId: 'corr-1',
+			timestamp: '2025-01-01T11:00:00Z',
+		}),
+	);
+
+	const result = logQueries.byCorrelation('corr-1');
+
+	t.is(result.entries[0]?.timestamp, '2025-01-01T10:00:00Z');
+	t.is(result.entries[1]?.timestamp, '2025-01-01T11:00:00Z');
+	t.is(result.entries[2]?.timestamp, '2025-01-01T12:00:00Z');
+});
+
+test('logQueries.bySource: filters by source', t => {
+	globalLogStorage.addEntry(createLogEntry({source: 'api-server'}));
+	globalLogStorage.addEntry(createLogEntry({source: 'db-server'}));
+	globalLogStorage.addEntry(createLogEntry({source: 'api-server'}));
+
+	const result = logQueries.bySource('api-server');
+
+	t.is(result.entries.length, 2);
+	t.true(result.entries.every(e => e.source === 'api-server'));
+});
+
+test('logQueries.byTag: filters by tag', t => {
+	globalLogStorage.addEntry(createLogEntry({tags: ['critical', 'database']}));
+	globalLogStorage.addEntry(createLogEntry({tags: ['info']}));
+	globalLogStorage.addEntry(createLogEntry({tags: ['critical']}));
+
+	const result = logQueries.byTag('critical');
+
+	t.is(result.entries.length, 2);
+	t.true(result.entries.every(e => e.tags?.includes('critical')));
+});
+
+test('logQueries.byTag: requires entries to have tags', t => {
+	globalLogStorage.addEntry(createLogEntry({tags: ['critical']}));
+	globalLogStorage.addEntry(createLogEntry({})); // No tags
+
+	const result = logQueries.byTag('critical');
+
+	t.is(result.entries.length, 1); // Entry without tags excluded
+});
+
+test('logQueries.slowRequests: filters by duration and message', t => {
+	globalLogStorage.addEntry(
+		createLogEntry({
+			message: 'Handled request',
+			performance: {duration: 1500},
+		}),
+	);
+	globalLogStorage.addEntry(
+		createLogEntry({
+			message: 'Handled request',
+			performance: {duration: 500},
+		}),
+	);
+	globalLogStorage.addEntry(
+		createLogEntry({
+			message: 'Other operation',
+			performance: {duration: 2000},
+		}),
+	);
+
+	const result = logQueries.slowRequests(1000);
+
+	t.is(result.entries.length, 1); // Only first entry (duration >= 1000 AND message contains "request")
+	t.is(result.entries[0]?.performance?.duration, 1500);
+});
+
+test('logQueries.slowRequests: uses default duration threshold', t => {
+	globalLogStorage.addEntry(
+		createLogEntry({
+			message: 'Handled request',
+			performance: {duration: 1500},
+		}),
+	);
+	globalLogStorage.addEntry(
+		createLogEntry({
+			message: 'Handled request',
+			performance: {duration: 500},
+		}),
+	);
+
+	const result = logQueries.slowRequests(); // Default 1000ms
+
+	t.is(result.entries.length, 1);
+});
+
+test('logQueries.slowRequests: sorts by duration descending', t => {
+	globalLogStorage.addEntry(
+		createLogEntry({
+			message: 'request 1',
+			performance: {duration: 1500},
+		}),
+	);
+	globalLogStorage.addEntry(
+		createLogEntry({
+			message: 'request 2',
+			performance: {duration: 3000},
+		}),
+	);
+	globalLogStorage.addEntry(
+		createLogEntry({
+			message: 'request 3',
+			performance: {duration: 2000},
+		}),
+	);
+
+	const result = logQueries.slowRequests(1000);
+
+	t.is(result.entries[0]?.performance?.duration, 3000);
+	t.is(result.entries[1]?.performance?.duration, 2000);
+	t.is(result.entries[2]?.performance?.duration, 1500);
+});
+
+test('logQueries.memoryIntensive: filters by memory threshold', t => {
+	const threshold = 50 * 1024 * 1024; // 50MB
+
+	globalLogStorage.addEntry(
+		createLogEntry({
+			performance: {memory: {heapUsed: 60 * 1024 * 1024}},
+		}),
+	);
+	globalLogStorage.addEntry(
+		createLogEntry({
+			performance: {memory: {heapUsed: 40 * 1024 * 1024}},
+		}),
+	);
+	globalLogStorage.addEntry(
+		createLogEntry({
+			performance: {memory: {heapUsed: 70 * 1024 * 1024}},
+		}),
+	);
+
+	const result = logQueries.memoryIntensive(threshold);
+
+	t.is(result.entries.length, 2); // Only entries above threshold
+	t.true(
+		result.entries.every(
+			e => (e.performance?.memory?.heapUsed || 0) >= threshold,
+		),
+	);
+});
+
+test('logQueries.memoryIntensive: uses default threshold', t => {
+	const defaultThreshold = 50 * 1024 * 1024;
+
+	globalLogStorage.addEntry(
+		createLogEntry({
+			performance: {memory: {heapUsed: 60 * 1024 * 1024}},
+		}),
+	);
+	globalLogStorage.addEntry(
+		createLogEntry({
+			performance: {memory: {heapUsed: 40 * 1024 * 1024}},
+		}),
+	);
+
+	const result = logQueries.memoryIntensive();
+
+	t.is(result.entries.length, 1);
+	t.true((result.entries[0]?.performance?.memory?.heapUsed || 0) >= defaultThreshold);
+});
+
+test('logQueries.memoryIntensive: sorts by memory descending', t => {
+	globalLogStorage.addEntry(
+		createLogEntry({
+			performance: {memory: {heapUsed: 60 * 1024 * 1024}},
+		}),
+	);
+	globalLogStorage.addEntry(
+		createLogEntry({
+			performance: {memory: {heapUsed: 80 * 1024 * 1024}},
+		}),
+	);
+	globalLogStorage.addEntry(
+		createLogEntry({
+			performance: {memory: {heapUsed: 70 * 1024 * 1024}},
+		}),
+	);
+
+	const result = logQueries.memoryIntensive(50 * 1024 * 1024);
+
+	t.is(result.entries[0]?.performance?.memory?.heapUsed, 80 * 1024 * 1024);
+	t.is(result.entries[1]?.performance?.memory?.heapUsed, 70 * 1024 * 1024);
+	t.is(result.entries[2]?.performance?.memory?.heapUsed, 60 * 1024 * 1024);
+});

package/source/utils/logging/log-query/utils/helpers.ts

@@ -0,0 +1,72 @@
+/**
+ * Helper functions for common log query operations
+ */
+
+import {createLogQuery} from '../query/query-builder.js';
+
+/**
+ * Quick query functions for common use cases
+ */
+export const logQueries = {
+	/**
+	 * Get all error logs
+	 */
+	errors: (limit?: number) =>
+		createLogQuery()
+			.levels('error', 'fatal')
+			.sortBy('timestamp', 'desc')
+			.limit(limit || 100)
+			.execute(),
+
+	/**
+	 * Get logs by correlation ID
+	 */
+	byCorrelation: (correlationId: string, limit?: number) =>
+		createLogQuery()
+			.correlationIds(correlationId)
+			.sortBy('timestamp', 'asc')
+			.limit(limit || 100)
+			.execute(),
+
+	/**
+	 * Get logs by source
+	 */
+	bySource: (source: string, limit?: number) =>
+		createLogQuery()
+			.sources(source)
+			.sortBy('timestamp', 'desc')
+			.limit(limit || 100)
+			.execute(),
+
+	/**
+	 * Get logs with specific tag
+	 */
+	byTag: (tag: string, limit?: number) =>
+		createLogQuery()
+			.tags(tag)
+			.hasTags()
+			.sortBy('timestamp', 'desc')
+			.limit(limit || 100)
+			.execute(),
+
+	/**
+	 * Get slow requests
+	 */
+	slowRequests: (minDuration: number = 1000, limit?: number) =>
+		createLogQuery()
+			.messageContains('request')
+			.durationMin(minDuration)
+			.sortBy('duration', 'desc')
+			.limit(limit || 50)
+			.execute(),
+
+	/**
+	 * Get memory-intensive operations
+	 */
+	memoryIntensive: (minMemory: number = 50 * 1024 * 1024, limit?: number) =>
+		createLogQuery()
+			.memoryThreshold(minMemory)
+			.sortBy('memory', 'desc')
+			.limit(limit || 50)
+			.execute(),
+};