@wener/common 2.0.3 → 2.0.5

package/src/drain3/TemplateMiner.ts

@@ -0,0 +1,246 @@
+import { Drain } from './Drain';
+import type { LogCluster } from './LogCluster';
+import type { PersistenceHandler } from './persistence/PersistenceHandler';
+import type { ClusterUpdateType, ExtractedParameter, SearchStrategy } from './types';
+
+/**
+ * High-level API for log template mining with persistence support.
+ *
+ * TemplateMiner wraps the Drain algorithm and provides convenience methods
+ * for extracting parameters and managing persistence.
+ */
+export class TemplateMiner {
+	constructor(
+		private drain: Drain,
+		private readonly persistence: PersistenceHandler,
+	) {}
+
+	/**
+	 * Adds a log message and returns the update type, cluster, template, and cluster count.
+	 *
+	 * @param content The log message content
+	 * @returns Object containing updateType, cluster, templateMined, and clusterCount
+	 */
+	async addLogMessage(content: string): Promise<{
+		updateType: ClusterUpdateType;
+		cluster: LogCluster;
+		templateMined: string;
+		clusterCount: number;
+	}> {
+		const { cluster, updateType } = this.drain.addLogMessage(content);
+
+		const templateMined = cluster.getTemplate();
+		const clusterCount = this.drain.getClusters().length;
+
+		if (updateType !== 'none') {
+			await this.saveState();
+		}
+
+		return {
+			updateType,
+			cluster,
+			templateMined,
+			clusterCount,
+		};
+	}
+
+	/**
+	 * Matches a log message against an already existing cluster.
+	 *
+	 * @param content Log message to match
+	 * @param strategy Search strategy
+	 * @returns Matched cluster or null
+	 */
+	match(content: string, strategy: SearchStrategy = 'never'): LogCluster | null {
+		return this.drain.match(content, strategy);
+	}
+
+	/**
+	 * Extracts parameters from a log message according to a provided template.
+	 *
+	 * @param logTemplate Template generated by AddLogMessage
+	 * @param logMessage The log message to extract parameters from
+	 * @returns Array of extracted parameters, or null if template doesn't match
+	 */
+	extractParameters(logTemplate: string, logMessage: string): ExtractedParameter[] | null {
+		// Apply delimiters
+		let processedMessage = logMessage;
+		for (const delimiter of this.drain.extraDelimiters) {
+			// Escape delimiter for regex if it contains special characters
+			const escapedDelimiter = delimiter.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+			processedMessage = processedMessage.replace(new RegExp(escapedDelimiter, 'g'), ' ');
+		}
+
+		const { templateRegex, paramGroupNameToMaskName } = this.getTemplateParameterExtractionRegex(logTemplate);
+
+		// Create regex from template
+		const regex = new RegExp(templateRegex);
+
+		// Match the log message against the template
+		const match = processedMessage.match(regex);
+
+		// Template doesn't match
+		if (match === null) {
+			return null;
+		}
+
+		// Extract named captures
+		const extractedParameters: ExtractedParameter[] = [];
+
+		// Get all named groups from the match
+		if (match.groups) {
+			for (const [groupName, value] of Object.entries(match.groups)) {
+				if (value !== undefined) {
+					const maskName = paramGroupNameToMaskName.get(groupName);
+					if (maskName !== undefined) {
+						extractedParameters.push({
+							value,
+							maskName,
+						});
+					}
+				}
+			}
+		}
+
+		// Also check by index if groups are not available (fallback)
+		// The regex exec method provides better named group support
+		const execResult = regex.exec(processedMessage);
+		if (execResult !== null && execResult.groups === undefined) {
+			// Fallback: extract by order (less reliable but works)
+			const indices = Array.from({ length: execResult.length - 1 }, (_, i) => i + 1);
+			let paramIndex = 0;
+			for (const index of indices) {
+				const value = execResult[index];
+				if (value !== undefined) {
+					// Find corresponding mask name by order
+					// This is a fallback - ideally we should use named groups
+					const maskName = paramGroupNameToMaskName.get(`p_${paramIndex}`);
+					if (maskName !== undefined) {
+						extractedParameters.push({
+							value,
+							maskName,
+						});
+					}
+					paramIndex++;
+				}
+			}
+		}
+
+		return extractedParameters.length > 0 ? extractedParameters : null;
+	}
+
+	/**
+	 * Gets the template parameter extraction regex and mapping.
+	 *
+	 * @param logTemplate The log template
+	 * @returns Tuple of [templateRegex, paramGroupNameToMaskName]
+	 */
+	private getTemplateParameterExtractionRegex(logTemplate: string): {
+		templateRegex: string;
+		paramGroupNameToMaskName: Map<string, string>;
+	} {
+		const paramGroupNameToMaskName = new Map<string, string>();
+		let paramNameCounter = 0;
+
+		const getNextParamName = (): string => {
+			const paramGroupName = `p_${paramNameCounter}`;
+			paramNameCounter++;
+			return paramGroupName;
+		};
+
+		// Create a named group with the respective patterns for the given mask-name
+		const createCaptureRegex = (maskName: string): string => {
+			const allowedPatterns: string[] = [];
+
+			if (maskName === '*') {
+				allowedPatterns.push('.+?');
+			}
+
+			// Give each capture group a unique name to avoid conflict
+			const paramGroupName = getNextParamName();
+			paramGroupNameToMaskName.set(paramGroupName, maskName);
+			const joinedPatterns = allowedPatterns.join('|');
+			const captureRegex = `(?<${paramGroupName}>${joinedPatterns})`;
+
+			return captureRegex;
+		};
+
+		// For every mask in the template, replace it with a named group
+		const maskNames = new Set<string>();
+		// The drain catch-all mask
+		maskNames.add('*');
+
+		// Escape the template for regex
+		let templateRegex = this.escapeRegex(logTemplate);
+
+		// Replace each mask name with a proper regex that captures it
+		for (const maskName of maskNames) {
+			const searchStr = `<${this.escapeRegex(maskName)}>`;
+
+			// Replace one-by-one to get a new param group name for each replacement
+			while (true) {
+				const repStr = createCaptureRegex(maskName);
+				const templateRegexNew = templateRegex.replace(searchStr, repStr);
+				// Break when all replaces for this mask are done
+				if (templateRegexNew === templateRegex) {
+					break;
+				}
+				templateRegex = templateRegexNew;
+			}
+		}
+
+		// Match also messages with multiple spaces or other whitespace chars between tokens
+		templateRegex = templateRegex.replace(/\\ /g, '\\s+');
+		templateRegex = `^${templateRegex}$`;
+
+		return { templateRegex, paramGroupNameToMaskName };
+	}
+
+	/**
+	 * Escapes special regex characters in a string.
+	 */
+	private escapeRegex(str: string): string {
+		return str.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+	}
+
+	/**
+	 * Loads the Drain state from persistence.
+	 *
+	 * @throws Error if loading fails
+	 */
+	async loadState(): Promise<void> {
+		const state = await this.persistence.load();
+		if (state === null || (typeof state === 'string' && state.length === 0)) {
+			throw new Error('saved state not found');
+		}
+
+		try {
+			const stateStr = typeof state === 'string' ? state : new TextDecoder().decode(state);
+			const data = JSON.parse(stateStr);
+			this.drain = Drain.fromJSON(data);
+		} catch (error) {
+			throw new Error(`failed to unmarshal state: ${error instanceof Error ? error.message : String(error)}`);
+		}
+	}
+
+	/**
+	 * Saves the Drain state to persistence.
+	 *
+	 * @throws Error if saving fails
+	 */
+	async saveState(): Promise<void> {
+		try {
+			const state = JSON.stringify(this.drain.toJSON());
+			await this.persistence.save(state);
+		} catch (error) {
+			throw new Error(`failed to save state: ${error instanceof Error ? error.message : String(error)}`);
+		}
+	}
+
+	/**
+	 * Gets the underlying Drain instance.
+	 */
+	getDrain(): Drain {
+		return this.drain;
+	}
+}

package/src/drain3/index.ts

@@ -0,0 +1,36 @@
+/**
+ * Drain3 - A TypeScript implementation of the Drain log clustering algorithm.
+ *
+ * Drain is an online log parsing algorithm that groups log messages into clusters
+ * based on their structural similarity, extracting templates by parameterizing
+ * variable parts of the logs.
+ *
+ * @example
+ * ```typescript
+ * import { Drain, TemplateMiner, MemoryPersistence } from '@wener/common/drain3';
+ *
+ * // Basic usage
+ * const drain = new Drain({
+ *   logClusterDepth: 4,
+ *   simTh: 0.4,
+ *   maxClusters: 1000
+ * });
+ *
+ * const result = drain.addLogMessage('[INFO] User 123 logged in');
+ * console.log(result.cluster.getTemplate()); // [INFO] User <*> logged in
+ *
+ * // Advanced usage with persistence
+ * const miner = new TemplateMiner(drain, new MemoryPersistence());
+ * await miner.addLogMessage('[INFO] User 123 logged in');
+ * ```
+ */
+
+export { Drain } from './Drain';
+export { LogCluster } from './LogCluster';
+export { Node } from './Node';
+export { TemplateMiner } from './TemplateMiner';
+
+export type { ClusterUpdateType, DrainOptions, ExtractedParameter, SearchStrategy } from './types';
+
+export type { PersistenceHandler } from './persistence/PersistenceHandler';
+export { MemoryPersistence } from './persistence/MemoryPersistence';

package/src/drain3/persistence/FilePersistence.ts

@@ -0,0 +1,24 @@
+import { readFile, writeFile } from 'node:fs/promises';
+import type { PersistenceHandler } from './PersistenceHandler';
+
+/**
+ * File-based persistence handler that saves and loads state from the filesystem.
+ */
+export class FilePersistence implements PersistenceHandler {
+	constructor(private readonly filePath: string) {}
+
+	async save(state: Uint8Array | string): Promise<void> {
+		await writeFile(this.filePath, state, 'utf8');
+	}
+
+	async load(): Promise<string | null> {
+		try {
+			return await readFile(this.filePath, 'utf8');
+		} catch (error) {
+			if ((error as { code?: string })?.code === 'ENOENT') {
+				return null;
+			}
+			throw error;
+		}
+	}
+}

package/src/drain3/persistence/MemoryPersistence.ts

@@ -0,0 +1,23 @@
+import type { PersistenceHandler } from './PersistenceHandler';
+
+/**
+ * In-memory persistence handler that stores state in memory.
+ * Useful for testing or when persistence is not needed.
+ */
+export class MemoryPersistence implements PersistenceHandler {
+	private state: Uint8Array | string | null = null;
+
+	/**
+	 * Saves the state to memory.
+	 */
+	async save(state: Uint8Array | string): Promise<void> {
+		this.state = state;
+	}
+
+	/**
+	 * Loads the state from memory.
+	 */
+	async load(): Promise<Uint8Array | string | null> {
+		return this.state;
+	}
+}

package/src/drain3/persistence/PersistenceHandler.ts

@@ -0,0 +1,19 @@
+/**
+ * Interface for persistence handlers that can save and load Drain state.
+ */
+export interface PersistenceHandler {
+	/**
+	 * Saves the Drain state.
+	 *
+	 * @param state Serialized state data
+	 * @returns Promise that resolves when save is complete
+	 */
+	save(state: Uint8Array | string): Promise<void>;
+
+	/**
+	 * Loads the Drain state.
+	 *
+	 * @returns Promise that resolves with the loaded state, or null if no state exists
+	 */
+	load(): Promise<Uint8Array | string | null>;
+}

package/src/drain3/types.ts

@@ -0,0 +1,75 @@
+/**
+ * Cluster update type indicating what happened when a log message was processed.
+ */
+export type ClusterUpdateType = 'none' | 'created' | 'templateChanged';
+
+/**
+ * Search strategy for matching log messages against existing clusters.
+ *
+ * - 'never': Fastest, always performs tree search [O(log(n))] but might produce false negatives
+ * - 'fallback': Performs linear search [O(n)] only if tree search found no match, should not have false negatives
+ * - 'always': Slowest, always evaluates all clusters and selects the best match with least wildcard parameters
+ */
+export type SearchStrategy = 'never' | 'fallback' | 'always';
+
+/**
+ * Configuration options for Drain algorithm.
+ */
+export interface DrainOptions {
+	/**
+	 * Depth of the prefix tree for log clustering. Must be at least 3.
+	 * @default 4
+	 */
+	logClusterDepth?: number;
+
+	/**
+	 * Similarity threshold (0.0 to 1.0) for matching log messages to clusters.
+	 * @default 0.4
+	 */
+	simTh?: number;
+
+	/**
+	 * Maximum number of children nodes in the prefix tree.
+	 * @default 100
+	 */
+	maxChildren?: number;
+
+	/**
+	 * Maximum number of clusters to maintain in LRU cache.
+	 * @default 1000
+	 */
+	maxClusters?: number;
+
+	/**
+	 * Additional delimiters to replace with spaces during tokenization.
+	 * @default []
+	 */
+	extraDelimiters?: string[];
+
+	/**
+	 * String used to represent parameterized tokens in templates.
+	 * @default "<*>"
+	 */
+	paramStr?: string;
+
+	/**
+	 * Whether to automatically parameterize tokens containing numbers.
+	 * @default true
+	 */
+	parametrizeNumericTokens?: boolean;
+}
+
+/**
+ * Extracted parameter from a log message based on a template.
+ */
+export interface ExtractedParameter {
+	/**
+	 * The extracted parameter value.
+	 */
+	value: string;
+
+	/**
+	 * The mask name (e.g., "*") that matched this parameter.
+	 */
+	maskName: string;
+}

package/src/fs/IFileSystem.d.ts

@@ -45,7 +45,7 @@ export type CreateWriteStreamOptions = OperationOptions & {
 };
 export type StatOptions = OperationOptions & {};
 
-type WritableData = string | ArrayBuffer | ArrayBufferView | ReadableStream;
+type WritableData = string | ArrayBuffer | ArrayBufferView<ArrayBufferLike> | ReadableStream;
 
 /**
  * Universal file system interface (browser & server compatible)
@@ -99,4 +99,3 @@ export type IFileStat = {
 	meta: Record<string, any>;
 	size: number;
 };
-

package/src/fs/createBrowserFileSystem.ts

@@ -37,7 +37,7 @@ type BrowserDirectory = IFileStat & {
 	};
 };
 
-type BrowserNode = BrowserFile | BrowserDirectory;
+type _BrowserNode = BrowserFile | BrowserDirectory;
 
 /**
  * Creates an IFileSystem instance backed by the browser's File System Access API.
@@ -136,7 +136,7 @@ class BrowserFS implements IFileSystem {
 		return !!handle;
 	}
 
-	async readdir(dir: string, options?: ReaddirOptions): Promise<IFileStat[]> {
+	async readdir(dir: string, _options?: ReaddirOptions): Promise<IFileStat[]> {
 		const [handle] = await this._getHandle(dir);
 		if (!handle) {
 			throw new BrowserFSError(`Directory not found: ${dir}`, 'ENOENT');
@@ -146,7 +146,9 @@ class BrowserFS implements IFileSystem {
 		}
 
 		const entries: IFileStat[] = [];
-		for await (const entry of (handle as FileSystemDirectoryHandle).values()) {
+		// FileSystemDirectoryHandle.values() is part of the File System Access API
+		// but not yet in TypeScript's lib.dom.d.ts
+		for await (const entry of (handle as any).values()) {
 			entries.push(await this._toFileStat(entry, join(dir, entry.name)));
 		}
 		return entries;
@@ -256,13 +258,13 @@ class BrowserFS implements IFileSystem {
 			await writable.close();
 		} else if (srcHandle.kind === 'directory') {
 			const newDirHandle = await destDirHandle.getDirectoryHandle(newName, { create: true });
-			for await (const entry of (srcHandle as FileSystemDirectoryHandle).values()) {
+			for await (const entry of (srcHandle as any).values()) {
 				await this._copyEntry(entry, newDirHandle, entry.name);
 			}
 		}
 	}
 
-	async createUrl(stat: IFileStat, options?: FileUrlOptions): Promise<string> {
+	async createUrl(stat: IFileStat, _options?: FileUrlOptions): Promise<string> {
 		// The handle is stored in the meta property in our _toFileStat method
 		const handle = stat.meta.handle as FileSystemHandle | undefined;
 		if (handle?.kind !== 'file') {

package/src/fs/createMemoryFileSystem.ts

@@ -1,6 +1,7 @@
 import type { Readable, Writable } from 'node:stream';
 import { computeIfAbsent } from '@wener/utils';
 import { basename, dirname, normalize } from 'pathe';
+import { FileSystemError } from './FileSystemError';
 import { findMimeType } from './findMimeType';
 import type {
 	CopyOptions,
@@ -27,21 +28,14 @@ type MemoryDirectory = IFileStat & {
 	children: MemoryNode[];
 };
 
-export function createMemoryFileSystem(
-	options: {
-		root?: MemoryDirectory;
-	} = {},
-): IFileSystem {
+export function createMemoryFileSystem(options: { root?: MemoryDirectory } = {}): IFileSystem {
 	return new MemFS(options);
 }
 
-class MemoryFileSystemError extends Error {
-	code?: string;
-
-	constructor(message: string, code?: string) {
-		super(message);
+class MemoryFileSystemError extends FileSystemError {
+	constructor(message: string, code: string) {
+		super(message, code);
 		this.name = 'MemoryFileSystemError';
-		this.code = code;
 	}
 }
 
@@ -408,7 +402,7 @@ class MemFS implements IFileSystem {
 
 		const self = this;
 		const stream = new Writable({
-			write(chunk: Buffer, encoding: BufferEncoding, callback: (error?: Error | null) => void) {
+			write(chunk: Buffer, _encoding: BufferEncoding, callback: (error?: Error | null) => void) {
 				if (signal?.aborted) {
 					callback(new MemoryFileSystemError('Operation aborted', 'ABORT_ERR'));
 					return;
@@ -487,7 +481,9 @@ class MemFS implements IFileSystem {
 			let c = computeIfAbsent(this.cache, mf, () => ({}) as Cache);
 			if (!c.url) {
 				let mime = findMimeType(mf.name) || 'application/octet-stream';
-				c.url = URL.createObjectURL(new Blob([mf.content], { type: mime }));
+				// Convert Buffer to Uint8Array for Blob compatibility with TypeScript 5.x strict typing
+				const blobPart = typeof mf.content === 'string' ? mf.content : new Uint8Array(mf.content);
+				c.url = URL.createObjectURL(new Blob([blobPart], { type: mime }));
 			}
 
 			return c.url;

package/src/fs/createSandboxFileSystem.ts

@@ -1,5 +1,5 @@
 import { join, normalize, relative, sep } from 'pathe';
-import type { IFileStat, IFileSystem, IServerFileSystem, ReadFileOptions } from './IFileSystem';
+import type { IFileStat, IServerFileSystem, ReadFileOptions } from './IFileSystem';
 import { getPath } from './utils';
 
 class SandboxSecurityError extends Error {

package/src/fs/createWebDavFileSystem.ts

@@ -9,6 +9,8 @@ import type {
 	ReadFileOptions,
 	RmOptions,
 	StatOptions,
+	WritableData,
+	WriteFileOptions,
 } from './IFileSystem';
 
 export function createWebDavFileSystem({ client }: { client: MaybeFunction<WebDAVClient> }): IFileSystem {
@@ -41,10 +43,10 @@ class WebdavFS implements IFileSystem {
 		const { filename: path, basename, lastmod, type: kind, etag, size, mime } = input;
 		let meta: Record<string, any> = {};
 		if (etag) {
-			meta['etag'] = etag;
+			meta.etag = etag;
 		}
 		if (mime) {
-			meta['mime'] = mime;
+			meta.mime = mime;
 		}
 		return {
 			directory: path.substring(0, path.lastIndexOf('/')) || '/',
@@ -59,12 +61,12 @@ class WebdavFS implements IFileSystem {
 
 	private getData<T>(input: ResponseDataDetailed<T> | T): T {
 		if (
-			input
-			&& typeof input === 'object'
-			&& 'data' in input
+			input &&
+			typeof input === 'object' &&
+			'data' in input &&
 			// 'headers' in input &&
-			&& 'status' in input
-			&& typeof input.status === 'number'
+			'status' in input &&
+			typeof input.status === 'number'
 		) {
 			return input.data;
 		}
@@ -135,11 +137,27 @@ class WebdavFS implements IFileSystem {
 		return this.getData(res);
 	}
 
-	async writeFile(path: string, data: string | Buffer | ArrayBuffer | Readable, options = {}): Promise<void> {
-		await this.client.putFileContents(path, data, options);
+	async writeFile(path: string, data: WritableData, options: WriteFileOptions = {}): Promise<void> {
+		// Convert web ReadableStream to something WebDAV client can handle
+		let webdavData: string | Buffer | ArrayBuffer | Readable = data as string | Buffer | ArrayBuffer | Readable;
+		if (data instanceof ReadableStream) {
+			// Convert web ReadableStream to Buffer
+			const reader = data.getReader();
+			const chunks: Uint8Array[] = [];
+			while (true) {
+				const { done, value } = await reader.read();
+				if (done) break;
+				if (value) chunks.push(value);
+			}
+			webdavData = Buffer.concat(chunks);
+		} else if (ArrayBuffer.isView(data) && !(data instanceof Buffer)) {
+			// Convert ArrayBufferView to Buffer
+			webdavData = Buffer.from(data.buffer, data.byteOffset, data.byteLength);
+		}
+		await this.client.putFileContents(path, webdavData, options);
 	}
 
-	async rm(path: string, { signal, force, recursive }: RmOptions = {}): Promise<void> {
+	async rm(path: string, { signal: _signal, force, recursive: _recursive }: RmOptions = {}): Promise<void> {
 		try {
 			await this.client.deleteFile(path);
 		} catch (e: any) {