@wener/common 2.0.2 → 2.0.5

package/src/drain3/TemplateMiner.ts

@@ -0,0 +1,246 @@
+import { Drain } from './Drain';
+import type { LogCluster } from './LogCluster';
+import type { PersistenceHandler } from './persistence/PersistenceHandler';
+import type { ClusterUpdateType, ExtractedParameter, SearchStrategy } from './types';
+
+/**
+ * High-level API for log template mining with persistence support.
+ *
+ * TemplateMiner wraps the Drain algorithm and provides convenience methods
+ * for extracting parameters and managing persistence.
+ */
+export class TemplateMiner {
+	constructor(
+		private drain: Drain,
+		private readonly persistence: PersistenceHandler,
+	) {}
+
+	/**
+	 * Adds a log message and returns the update type, cluster, template, and cluster count.
+	 *
+	 * @param content The log message content
+	 * @returns Object containing updateType, cluster, templateMined, and clusterCount
+	 */
+	async addLogMessage(content: string): Promise<{
+		updateType: ClusterUpdateType;
+		cluster: LogCluster;
+		templateMined: string;
+		clusterCount: number;
+	}> {
+		const { cluster, updateType } = this.drain.addLogMessage(content);
+
+		const templateMined = cluster.getTemplate();
+		const clusterCount = this.drain.getClusters().length;
+
+		if (updateType !== 'none') {
+			await this.saveState();
+		}
+
+		return {
+			updateType,
+			cluster,
+			templateMined,
+			clusterCount,
+		};
+	}
+
+	/**
+	 * Matches a log message against an already existing cluster.
+	 *
+	 * @param content Log message to match
+	 * @param strategy Search strategy
+	 * @returns Matched cluster or null
+	 */
+	match(content: string, strategy: SearchStrategy = 'never'): LogCluster | null {
+		return this.drain.match(content, strategy);
+	}
+
+	/**
+	 * Extracts parameters from a log message according to a provided template.
+	 *
+	 * @param logTemplate Template generated by AddLogMessage
+	 * @param logMessage The log message to extract parameters from
+	 * @returns Array of extracted parameters, or null if template doesn't match
+	 */
+	extractParameters(logTemplate: string, logMessage: string): ExtractedParameter[] | null {
+		// Apply delimiters
+		let processedMessage = logMessage;
+		for (const delimiter of this.drain.extraDelimiters) {
+			// Escape delimiter for regex if it contains special characters
+			const escapedDelimiter = delimiter.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+			processedMessage = processedMessage.replace(new RegExp(escapedDelimiter, 'g'), ' ');
+		}
+
+		const { templateRegex, paramGroupNameToMaskName } = this.getTemplateParameterExtractionRegex(logTemplate);
+
+		// Create regex from template
+		const regex = new RegExp(templateRegex);
+
+		// Match the log message against the template
+		const match = processedMessage.match(regex);
+
+		// Template doesn't match
+		if (match === null) {
+			return null;
+		}
+
+		// Extract named captures
+		const extractedParameters: ExtractedParameter[] = [];
+
+		// Get all named groups from the match
+		if (match.groups) {
+			for (const [groupName, value] of Object.entries(match.groups)) {
+				if (value !== undefined) {
+					const maskName = paramGroupNameToMaskName.get(groupName);
+					if (maskName !== undefined) {
+						extractedParameters.push({
+							value,
+							maskName,
+						});
+					}
+				}
+			}
+		}
+
+		// Also check by index if groups are not available (fallback)
+		// The regex exec method provides better named group support
+		const execResult = regex.exec(processedMessage);
+		if (execResult !== null && execResult.groups === undefined) {
+			// Fallback: extract by order (less reliable but works)
+			const indices = Array.from({ length: execResult.length - 1 }, (_, i) => i + 1);
+			let paramIndex = 0;
+			for (const index of indices) {
+				const value = execResult[index];
+				if (value !== undefined) {
+					// Find corresponding mask name by order
+					// This is a fallback - ideally we should use named groups
+					const maskName = paramGroupNameToMaskName.get(`p_${paramIndex}`);
+					if (maskName !== undefined) {
+						extractedParameters.push({
+							value,
+							maskName,
+						});
+					}
+					paramIndex++;
+				}
+			}
+		}
+
+		return extractedParameters.length > 0 ? extractedParameters : null;
+	}
+
+	/**
+	 * Gets the template parameter extraction regex and mapping.
+	 *
+	 * @param logTemplate The log template
+	 * @returns Tuple of [templateRegex, paramGroupNameToMaskName]
+	 */
+	private getTemplateParameterExtractionRegex(logTemplate: string): {
+		templateRegex: string;
+		paramGroupNameToMaskName: Map<string, string>;
+	} {
+		const paramGroupNameToMaskName = new Map<string, string>();
+		let paramNameCounter = 0;
+
+		const getNextParamName = (): string => {
+			const paramGroupName = `p_${paramNameCounter}`;
+			paramNameCounter++;
+			return paramGroupName;
+		};
+
+		// Create a named group with the respective patterns for the given mask-name
+		const createCaptureRegex = (maskName: string): string => {
+			const allowedPatterns: string[] = [];
+
+			if (maskName === '*') {
+				allowedPatterns.push('.+?');
+			}
+
+			// Give each capture group a unique name to avoid conflict
+			const paramGroupName = getNextParamName();
+			paramGroupNameToMaskName.set(paramGroupName, maskName);
+			const joinedPatterns = allowedPatterns.join('|');
+			const captureRegex = `(?<${paramGroupName}>${joinedPatterns})`;
+
+			return captureRegex;
+		};
+
+		// For every mask in the template, replace it with a named group
+		const maskNames = new Set<string>();
+		// The drain catch-all mask
+		maskNames.add('*');
+
+		// Escape the template for regex
+		let templateRegex = this.escapeRegex(logTemplate);
+
+		// Replace each mask name with a proper regex that captures it
+		for (const maskName of maskNames) {
+			const searchStr = `<${this.escapeRegex(maskName)}>`;
+
+			// Replace one-by-one to get a new param group name for each replacement
+			while (true) {
+				const repStr = createCaptureRegex(maskName);
+				const templateRegexNew = templateRegex.replace(searchStr, repStr);
+				// Break when all replaces for this mask are done
+				if (templateRegexNew === templateRegex) {
+					break;
+				}
+				templateRegex = templateRegexNew;
+			}
+		}
+
+		// Match also messages with multiple spaces or other whitespace chars between tokens
+		templateRegex = templateRegex.replace(/\\ /g, '\\s+');
+		templateRegex = `^${templateRegex}$`;
+
+		return { templateRegex, paramGroupNameToMaskName };
+	}
+
+	/**
+	 * Escapes special regex characters in a string.
+	 */
+	private escapeRegex(str: string): string {
+		return str.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+	}
+
+	/**
+	 * Loads the Drain state from persistence.
+	 *
+	 * @throws Error if loading fails
+	 */
+	async loadState(): Promise<void> {
+		const state = await this.persistence.load();
+		if (state === null || (typeof state === 'string' && state.length === 0)) {
+			throw new Error('saved state not found');
+		}
+
+		try {
+			const stateStr = typeof state === 'string' ? state : new TextDecoder().decode(state);
+			const data = JSON.parse(stateStr);
+			this.drain = Drain.fromJSON(data);
+		} catch (error) {
+			throw new Error(`failed to unmarshal state: ${error instanceof Error ? error.message : String(error)}`);
+		}
+	}
+
+	/**
+	 * Saves the Drain state to persistence.
+	 *
+	 * @throws Error if saving fails
+	 */
+	async saveState(): Promise<void> {
+		try {
+			const state = JSON.stringify(this.drain.toJSON());
+			await this.persistence.save(state);
+		} catch (error) {
+			throw new Error(`failed to save state: ${error instanceof Error ? error.message : String(error)}`);
+		}
+	}
+
+	/**
+	 * Gets the underlying Drain instance.
+	 */
+	getDrain(): Drain {
+		return this.drain;
+	}
+}

package/src/drain3/index.ts

@@ -0,0 +1,36 @@
+/**
+ * Drain3 - A TypeScript implementation of the Drain log clustering algorithm.
+ *
+ * Drain is an online log parsing algorithm that groups log messages into clusters
+ * based on their structural similarity, extracting templates by parameterizing
+ * variable parts of the logs.
+ *
+ * @example
+ * ```typescript
+ * import { Drain, TemplateMiner, MemoryPersistence } from '@wener/common/drain3';
+ *
+ * // Basic usage
+ * const drain = new Drain({
+ *   logClusterDepth: 4,
+ *   simTh: 0.4,
+ *   maxClusters: 1000
+ * });
+ *
+ * const result = drain.addLogMessage('[INFO] User 123 logged in');
+ * console.log(result.cluster.getTemplate()); // [INFO] User <*> logged in
+ *
+ * // Advanced usage with persistence
+ * const miner = new TemplateMiner(drain, new MemoryPersistence());
+ * await miner.addLogMessage('[INFO] User 123 logged in');
+ * ```
+ */
+
+export { Drain } from './Drain';
+export { LogCluster } from './LogCluster';
+export { Node } from './Node';
+export { TemplateMiner } from './TemplateMiner';
+
+export type { ClusterUpdateType, DrainOptions, ExtractedParameter, SearchStrategy } from './types';
+
+export type { PersistenceHandler } from './persistence/PersistenceHandler';
+export { MemoryPersistence } from './persistence/MemoryPersistence';

package/src/drain3/persistence/FilePersistence.ts

@@ -0,0 +1,24 @@
+import { readFile, writeFile } from 'node:fs/promises';
+import type { PersistenceHandler } from './PersistenceHandler';
+
+/**
+ * File-based persistence handler that saves and loads state from the filesystem.
+ */
+export class FilePersistence implements PersistenceHandler {
+	constructor(private readonly filePath: string) {}
+
+	async save(state: Uint8Array | string): Promise<void> {
+		await writeFile(this.filePath, state, 'utf8');
+	}
+
+	async load(): Promise<string | null> {
+		try {
+			return await readFile(this.filePath, 'utf8');
+		} catch (error) {
+			if ((error as { code?: string })?.code === 'ENOENT') {
+				return null;
+			}
+			throw error;
+		}
+	}
+}

package/src/drain3/persistence/MemoryPersistence.ts

@@ -0,0 +1,23 @@
+import type { PersistenceHandler } from './PersistenceHandler';
+
+/**
+ * In-memory persistence handler that stores state in memory.
+ * Useful for testing or when persistence is not needed.
+ */
+export class MemoryPersistence implements PersistenceHandler {
+	private state: Uint8Array | string | null = null;
+
+	/**
+	 * Saves the state to memory.
+	 */
+	async save(state: Uint8Array | string): Promise<void> {
+		this.state = state;
+	}
+
+	/**
+	 * Loads the state from memory.
+	 */
+	async load(): Promise<Uint8Array | string | null> {
+		return this.state;
+	}
+}

package/src/drain3/persistence/PersistenceHandler.ts

@@ -0,0 +1,19 @@
+/**
+ * Interface for persistence handlers that can save and load Drain state.
+ */
+export interface PersistenceHandler {
+	/**
+	 * Saves the Drain state.
+	 *
+	 * @param state Serialized state data
+	 * @returns Promise that resolves when save is complete
+	 */
+	save(state: Uint8Array | string): Promise<void>;
+
+	/**
+	 * Loads the Drain state.
+	 *
+	 * @returns Promise that resolves with the loaded state, or null if no state exists
+	 */
+	load(): Promise<Uint8Array | string | null>;
+}

package/src/drain3/types.ts

@@ -0,0 +1,75 @@
+/**
+ * Cluster update type indicating what happened when a log message was processed.
+ */
+export type ClusterUpdateType = 'none' | 'created' | 'templateChanged';
+
+/**
+ * Search strategy for matching log messages against existing clusters.
+ *
+ * - 'never': Fastest, always performs tree search [O(log(n))] but might produce false negatives
+ * - 'fallback': Performs linear search [O(n)] only if tree search found no match, should not have false negatives
+ * - 'always': Slowest, always evaluates all clusters and selects the best match with least wildcard parameters
+ */
+export type SearchStrategy = 'never' | 'fallback' | 'always';
+
+/**
+ * Configuration options for Drain algorithm.
+ */
+export interface DrainOptions {
+	/**
+	 * Depth of the prefix tree for log clustering. Must be at least 3.
+	 * @default 4
+	 */
+	logClusterDepth?: number;
+
+	/**
+	 * Similarity threshold (0.0 to 1.0) for matching log messages to clusters.
+	 * @default 0.4
+	 */
+	simTh?: number;
+
+	/**
+	 * Maximum number of children nodes in the prefix tree.
+	 * @default 100
+	 */
+	maxChildren?: number;
+
+	/**
+	 * Maximum number of clusters to maintain in LRU cache.
+	 * @default 1000
+	 */
+	maxClusters?: number;
+
+	/**
+	 * Additional delimiters to replace with spaces during tokenization.
+	 * @default []
+	 */
+	extraDelimiters?: string[];
+
+	/**
+	 * String used to represent parameterized tokens in templates.
+	 * @default "<*>"
+	 */
+	paramStr?: string;
+
+	/**
+	 * Whether to automatically parameterize tokens containing numbers.
+	 * @default true
+	 */
+	parametrizeNumericTokens?: boolean;
+}
+
+/**
+ * Extracted parameter from a log message based on a template.
+ */
+export interface ExtractedParameter {
+	/**
+	 * The extracted parameter value.
+	 */
+	value: string;
+
+	/**
+	 * The mask name (e.g., "*") that matched this parameter.
+	 */
+	maskName: string;
+}

package/src/emittery/emitter.ts

@@ -0,0 +1,9 @@
+import Emittery, { type EventName } from 'emittery';
+
+export function createEmitter<EventData = Record<EventName, any>>(name: string) {
+	return new Emittery<EventData>({
+		debug: {
+			name,
+		},
+	});
+}

package/src/emittery/index.ts

@@ -0,0 +1 @@
+export { createEmitter } from './emitter';

package/src/fs/FileSystemError.ts

@@ -0,0 +1,26 @@
+export const FileSystemErrorCode = {
+	ENOENT: 'ENOENT',
+	ENOTDIR: 'ENOTDIR',
+	EEXIST: 'EEXIST',
+	EISDIR: 'EISDIR',
+	ENOTEMPTY: 'ENOTEMPTY',
+	EACCES: 'EACCES',
+	EPERM: 'EPERM',
+	EINVAL: 'EINVAL',
+} as const;
+
+export type FileSystemErrorCode = (typeof FileSystemErrorCode)[keyof typeof FileSystemErrorCode];
+
+export class FileSystemError extends Error {
+	constructor(
+		message: string,
+		public readonly code: FileSystemErrorCode | string,
+	) {
+		super(message);
+		this.name = 'FileSystemError';
+		// Maintains proper stack trace for where our error was thrown (only available on V8)
+		if (Error.captureStackTrace) {
+			Error.captureStackTrace(this, FileSystemError);
+		}
+	}
+}

package/src/fs/IFileSystem.d.ts

@@ -0,0 +1,101 @@
+import type { FileKind, FileUrlOptions } from './types';
+
+// Base operation options
+type OperationOptions = {
+	signal?: AbortSignal;
+};
+
+// Directory operations
+export type ReaddirOptions = OperationOptions & {
+	glob?: string;
+	recursive?: boolean;
+	depth?: number;
+	kind?: FileKind;
+	hidden?: boolean;
+};
+export type MkdirOptions = OperationOptions & {
+	recursive?: boolean;
+};
+// File operations
+export type ReadFileOptions = OperationOptions & {
+	encoding?: 'text' | 'binary';
+	onDownloadProgress?: (e: { loaded: number; total: number }) => void;
+};
+export type WriteFileOptions = OperationOptions & {
+	overwrite?: boolean;
+	onUploadProgress?: (e: { loaded: number; total: number }) => void;
+};
+export type RenameOptions = OperationOptions & {
+	overwrite?: boolean;
+};
+export type RmOptions = OperationOptions & {
+	recursive?: boolean;
+	force?: boolean;
+};
+export type CopyOptions = OperationOptions & {
+	overwrite?: boolean;
+	shallow?: boolean;
+};
+export type CreateReadStreamOptions = OperationOptions & {
+	range?: { start: number; end?: number };
+	signal?: AbortSignal;
+};
+export type CreateWriteStreamOptions = OperationOptions & {
+	overwrite?: boolean;
+};
+export type StatOptions = OperationOptions & {};
+
+type WritableData = string | ArrayBuffer | ArrayBufferView<ArrayBufferLike> | ReadableStream;
+
+/**
+ * Universal file system interface (browser & server compatible)
+ */
+export type IFileSystem = {
+	readdir(dir: string, options?: ReaddirOptions): Promise<IFileStat[]>;
+	stat(entry: string, options?: StatOptions): Promise<IFileStat>;
+	mkdir(path: string, options?: MkdirOptions): Promise<void>;
+	readFile(path: string, options?: ReadFileOptions & { encoding: 'text' }): Promise<string>;
+	readFile(path: string, options?: ReadFileOptions): Promise<Uint8Array>;
+	writeFile(path: string, data: WritableData, options?: WriteFileOptions): Promise<void>;
+	rm(path: string, options?: RmOptions): Promise<void>;
+	rename(oldPath: string, newPath: string, options?: RenameOptions): Promise<void>;
+	exists(path: string): Promise<boolean>;
+	copy(src: string, dest: string, options?: CopyOptions): Promise<void>;
+
+	getUrl?(path: IFileStat | string, options?: FileUrlOptions): string | undefined;
+
+	createReadableStream?(path: string, options?: CreateReadStreamOptions): ReadableStream;
+	createWritableStream?(path: string, options?: CreateWriteStreamOptions): WritableStream;
+};
+
+/**
+ * Server/Node.js specific file system interface with stream support
+ */
+export type IServerFileSystem = IFileSystem & {
+	createReadStream(path: string, options?: CreateReadStreamOptions): import('node:stream').Readable;
+	createWriteStream(path: string, options?: CreateWriteStreamOptions): import('node:stream').Writable;
+	writeFile(
+		path: string,
+		data: WritableData | Buffer | import('node:stream').Readable,
+		options?: WriteFileOptions,
+	): Promise<void>;
+};
+
+export type IFileStat = {
+	/**
+	 * parent path
+	 */
+	directory: string;
+	/**
+	 * full path
+	 */
+	path: string;
+	/**
+	 * basename
+	 */
+	name: string;
+	kind: 'directory' | 'file';
+	mtime: number;
+	meta: Record<string, any>;
+	size: number;
+};

package/src/fs/MemoryFileSystem.test.ts

@@ -0,0 +1,37 @@
+import { beforeEach, describe, test } from 'vitest';
+import { createMemoryFileSystem } from './createMemoryFileSystem';
+import { runFileSystemTest } from './tests/runFileSystemTest';
+
+describe('MemoryFileSystem', () => {
+	let fs: ReturnType<typeof createMemoryFileSystem>;
+
+	beforeEach(() => {
+		fs = createMemoryFileSystem({
+			root: {
+				path: '/',
+				directory: '',
+				name: '',
+				kind: 'directory',
+				meta: {},
+				mtime: Date.now(),
+				size: 0,
+				children: [
+					{
+						path: '/README.txt',
+						directory: '/',
+						name: 'README.txt',
+						kind: 'file',
+						content: 'Hello',
+						size: 5,
+						meta: {},
+						mtime: Date.now(),
+					},
+				],
+			},
+		});
+	});
+
+	test('common tests', async () => {
+		await runFileSystemTest(fs);
+	});
+});