@ananotherdeveloper/drain3js 0.9.11-rev1

package/LICENSE

@@ -0,0 +1,25 @@
+MIT License
+
+Based on Drain3 by International Business Machines and the Drain3 project contributors.
+Original: https://github.com/logpai/Drain3
+
+Copyright (c) 2020-2022 International Business Machines and the Drain3 project contributors.
+Copyright (c) 2026 ananotherdeveloper
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,237 @@
+# drain3js
+
+TypeScript port of [Drain3](https://github.com/logpai/Drain3) - a persistent and streaming log template miner that uses a fixed-depth parse tree.
+
+[![npm version](https://img.shields.io/npm/v/%40ananotherdeveloper%2Fdrain3js.svg)](https://www.npmjs.com/package/@ananotherdeveloper/drain3js)
+[![CI](https://github.com/ananotherdeveloper/drain3js/actions/workflows/ci.yml/badge.svg)](https://github.com/ananotherdeveloper/drain3js/actions/workflows/ci.yml)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+
+## What is Drain3?
+
+Drain3 extracts log templates from raw log messages in a streaming fashion. Given logs like:
+
+```
+Connected to 10.0.0.1
+Connected to 10.0.0.2
+Disk error on /dev/sda1
+Disk error on /dev/sdb2
+```
+
+It mines templates:
+
+```
+Connected to <IP>
+Disk error on <*>
+```
+
+Based on the paper: *"Drain: An Online Log Parsing Approach with Fixed Depth Tree"* by Pinjia He, Jieming Zhu, Zibin Zheng, and Michael R. Lyu (ICWS 2017).
+
+## Installation
+
+```bash
+npm install @ananotherdeveloper/drain3js
+```
+
+## Quick Start
+
+```typescript
+import { TemplateMiner, TemplateMinerConfig } from '@ananotherdeveloper/drain3js';
+
+const config = new TemplateMinerConfig();
+const miner = new TemplateMiner(null, config);
+
+const logs = [
+  'Connected to 10.0.0.1',
+  'Connected to 10.0.0.2',
+  'Disk error on /dev/sda1',
+  'Disk error on /dev/sdb2',
+];
+
+for (const log of logs) {
+  const result = miner.addLogMessage(log);
+  console.log(`Template: ${result.templateMined} (cluster #${result.clusterId})`);
+}
+```
+
+## Features
+
+- **Streaming**: Process logs one at a time - no need to batch
+- **Persistent state**: Save and restore miner state via pluggable persistence handlers
+- **Masking**: Regex-based masking to normalize IPs, numbers, hex values, etc. before mining
+- **Two algorithms**: Standard `Drain` (token-count tree) and `JaccardDrain` (Jaccard similarity)
+- **Parameter extraction**: Extract variable parameters from logs using mined templates
+- **Memory efficient**: Optional LRU-based cluster eviction with configurable max clusters
+- **Inference mode**: Match logs against existing templates without creating new clusters
+
+## Configuration
+
+Use `TemplateMinerConfig` to customize behavior:
+
+```typescript
+import { TemplateMinerConfig } from '@ananotherdeveloper/drain3js';
+
+const config = new TemplateMinerConfig();
+
+// Load from a JSON config object
+config.load({
+  drain: {
+    engine: 'Drain',         // 'Drain' or 'JaccardDrain'
+    simTh: 0.4,              // Similarity threshold (default: 0.4)
+    depth: 4,                // Parse tree depth (default: 4, minimum: 3)
+    maxChildren: 100,        // Max children per node (default: 100)
+    maxClusters: null,       // Max clusters, null = unlimited (default: null)
+    extraDelimiters: ['_'],  // Additional token delimiters (default: [])
+    parametrizeNumericTokens: true,
+  },
+  masking: {
+    instructions: [
+      { regexPattern: '((?:\\d+\\.){3}\\d+)', maskWith: 'IP' },
+      { regexPattern: '0x[a-fA-F0-9]+', maskWith: 'HEX' },
+      { regexPattern: '\\d+', maskWith: 'NUM' },
+    ],
+    maskPrefix: '<',
+    maskSuffix: '>',
+  },
+  snapshot: {
+    snapshotIntervalMinutes: 5,
+    compressState: true,
+  },
+});
+
+const miner = new TemplateMiner(null, config);
+```
+
+You can also load configuration from a JSON file:
+
+```typescript
+config.load('/path/to/config.json');
+```
+
+## Persistence
+
+Save and restore miner state using persistence handlers:
+
+```typescript
+import { TemplateMiner, TemplateMinerConfig, FilePersistence } from '@ananotherdeveloper/drain3js';
+
+const persistence = new FilePersistence('./drain3_state.bin');
+const config = new TemplateMinerConfig();
+const miner = new TemplateMiner(persistence, config);
+
+// State is automatically saved based on snapshotIntervalMinutes
+// and whenever a new cluster is created
+```
+
+Built-in persistence handlers:
+
+| Handler | Description |
+|---------|-------------|
+| `FilePersistence` | Saves state to a local file |
+| `MemoryBufferPersistence` | Stores state in memory (useful for testing) |
+
+Implement the `PersistenceHandler` interface for custom storage (Redis, S3, etc.):
+
+```typescript
+import { PersistenceHandler } from '@ananotherdeveloper/drain3js';
+
+class RedisPersistence implements PersistenceHandler {
+  saveState(state: Buffer): void { /* ... */ }
+  loadState(): Buffer | null { /* ... */ }
+}
+```
+
+## Masking
+
+Mask sensitive or variable data before template mining:
+
+```typescript
+config.load({
+  masking: {
+    instructions: [
+      { regexPattern: '((?:\\d+\\.){3}\\d+)', maskWith: 'IP' },
+      { regexPattern: '0x[a-fA-F0-9]+', maskWith: 'HEX' },
+      { regexPattern: '(?<=user=)\\w+', maskWith: 'USER' },
+    ],
+  },
+});
+```
+
+Input: `Connected to 10.0.0.1 by user=admin`
+After masking: `Connected to <IP> by user=<USER>`
+
+## Parameter Extraction
+
+Extract variable parameters from logs using mined templates:
+
+```typescript
+const result = miner.addLogMessage('Connected to 10.0.0.1');
+const params = miner.extractParameters(result.templateMined, 'Connected to 10.0.0.1');
+// [{ value: '10.0.0.1', maskName: 'IP' }]
+```
+
+## Inference Mode
+
+Match logs against existing templates without creating new clusters:
+
+```typescript
+// Training phase
+miner.addLogMessage('User alice logged in');
+miner.addLogMessage('User bob logged in');
+
+// Inference phase
+const match = miner.match('User charlie logged in');
+if (match) {
+  console.log(`Matched template: ${match.getTemplate()}`);
+}
+```
+
+The `match()` method supports three search strategies:
+- `'never'` (default) - Only search the tree path
+- `'fallback'` - Tree search first, then full search if no match
+- `'always'` - Always search all clusters for the token count
+
+## API Reference
+
+### `TemplateMiner`
+
+| Method | Description |
+|--------|-------------|
+| `addLogMessage(logMessage)` | Process a log message and return mining result |
+| `match(logMessage, strategy?)` | Match against existing templates |
+| `extractParameters(template, logMessage)` | Extract parameters with mask names |
+| `getParameterList(template, logMessage)` | Extract parameter values only |
+| `saveState(reason)` | Manually save state |
+| `loadState()` | Manually load state |
+
+### `MinerResult`
+
+| Field | Type | Description |
+|-------|------|-------------|
+| `changeType` | `string` | `'cluster_created'`, `'cluster_template_changed'`, or `'none'` |
+| `clusterId` | `number` | ID of the matched/created cluster |
+| `clusterSize` | `number` | Number of logs in the cluster |
+| `templateMined` | `string` | The current template string |
+| `clusterCount` | `number` | Total number of clusters |
+
+### Low-level API
+
+Use `Drain` or `JaccardDrain` directly for advanced use cases:
+
+```typescript
+import { Drain } from '@ananotherdeveloper/drain3js';
+
+const drain = new Drain(4, 0.4, 100);
+const [cluster, changeType] = drain.addLogMessage('Connected to 10.0.0.1');
+console.log(cluster.getTemplate());
+```
+
+## Attribution
+
+This project is a TypeScript port of [Drain3](https://github.com/logpai/Drain3) by IBM Research.
+
+The Drain algorithm is based on the paper:
+> Pinjia He, Jieming Zhu, Zibin Zheng, and Michael R. Lyu. "Drain: An Online Log Parsing Approach with Fixed Depth Tree," *Proceedings of the 2017 IEEE International Conference on Web Services (ICWS)*, 2017.
+
+## License
+
+[MIT](LICENSE)

package/dist/drain.d.ts

@@ -0,0 +1,67 @@
+import { type Profiler } from './simple-profiler.js';
+export type FullSearchStrategy = 'never' | 'fallback' | 'always';
+export declare class LogCluster {
+    logTemplateTokens: string[];
+    clusterId: number;
+    size: number;
+    constructor(logTemplateTokens: Iterable<string>, clusterId: number);
+    getTemplate(): string;
+    toString(): string;
+}
+export interface ClusterMap {
+    /** Get with LRU touch (updates recency) */
+    get(key: number): LogCluster | undefined;
+    /** Get without LRU touch (does not update recency) */
+    peek(key: number): LogCluster | undefined;
+    set(key: number, value: LogCluster): void;
+    has(key: number): boolean;
+    delete(key: number): boolean;
+    values(): IterableIterator<LogCluster>;
+    keys(): IterableIterator<number>;
+    readonly size: number;
+}
+export declare class Node {
+    keyToChildNode: Map<string, Node>;
+    clusterIds: number[];
+    constructor();
+}
+export declare abstract class DrainBase {
+    logClusterDepth: number;
+    maxNodeDepth: number;
+    simTh: number;
+    maxChildren: number;
+    rootNode: Node;
+    profiler: Profiler;
+    extraDelimiters: string[];
+    maxClusters: number | null;
+    paramStr: string;
+    parametrizeNumericTokens: boolean;
+    idToCluster: ClusterMap;
+    clustersCounter: number;
+    constructor(depth?: number, simTh?: number, maxChildren?: number, maxClusters?: number | null, extraDelimiters?: string[], profiler?: Profiler, paramStr?: string, parametrizeNumericTokens?: boolean);
+    get clusters(): LogCluster[];
+    static hasNumbers(s: string): boolean;
+    fastMatch(clusterIds: number[], tokens: string[], simTh: number, includeParams: boolean): LogCluster | null;
+    printTree(file?: {
+        write(s: string): void;
+    }, maxClusters?: number): void;
+    private printNode;
+    getContentAsTokens(content: string): string[];
+    addLogMessage(content: string): [LogCluster, string];
+    private arraysEqual;
+    getTotalClusterSize(): number;
+    getClustersIdsForSeqLen(seqFir: number | string): number[];
+    abstract treeSearch(rootNode: Node, tokens: string[], simTh: number, includeParams: boolean): LogCluster | null;
+    abstract addSeqToPrefixTree(rootNode: Node, cluster: LogCluster): void;
+    abstract getSeqDistance(seq1: string[], seq2: string[], includeParams: boolean): [number, number];
+    abstract createTemplate(seq1: string[], seq2: string[]): string[];
+    abstract match(content: string, fullSearchStrategy?: FullSearchStrategy): LogCluster | null;
+}
+export declare class Drain extends DrainBase {
+    treeSearch(rootNode: Node, tokens: string[], simTh: number, includeParams: boolean): LogCluster | null;
+    addSeqToPrefixTree(rootNode: Node, cluster: LogCluster): void;
+    getSeqDistance(seq1: string[], seq2: string[], includeParams: boolean): [number, number];
+    createTemplate(seq1: string[], seq2: string[]): string[];
+    match(content: string, fullSearchStrategy?: FullSearchStrategy): LogCluster | null;
+}
+//# sourceMappingURL=drain.d.ts.map

package/dist/drain.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"drain.d.ts","sourceRoot":"","sources":["../src/drain.ts"],"names":[],"mappings":"AAKA,OAAO,EAAgB,KAAK,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAEnE,MAAM,MAAM,kBAAkB,GAAG,OAAO,GAAG,UAAU,GAAG,QAAQ,CAAC;AAEjE,qBAAa,UAAU;IACrB,iBAAiB,EAAE,MAAM,EAAE,CAAC;IAC5B,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;gBAED,iBAAiB,EAAE,QAAQ,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,MAAM;IAMlE,WAAW,IAAI,MAAM;IAIrB,QAAQ,IAAI,MAAM;CAGnB;AAED,MAAM,WAAW,UAAU;IACzB,2CAA2C;IAC3C,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,UAAU,GAAG,SAAS,CAAC;IACzC,sDAAsD;IACtD,IAAI,CAAC,GAAG,EAAE,MAAM,GAAG,UAAU,GAAG,SAAS,CAAC;IAC1C,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,UAAU,GAAG,IAAI,CAAC;IAC1C,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;IAC1B,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;IAC7B,MAAM,IAAI,gBAAgB,CAAC,UAAU,CAAC,CAAC;IACvC,IAAI,IAAI,gBAAgB,CAAC,MAAM,CAAC,CAAC;IACjC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;CACvB;AAkFD,qBAAa,IAAI;IACf,cAAc,EAAE,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IAClC,UAAU,EAAE,MAAM,EAAE,CAAC;;CAMtB;AAED,8BAAsB,SAAS;IAC7B,eAAe,EAAE,MAAM,CAAC;IACxB,YAAY,EAAE,MAAM,CAAC;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,IAAI,CAAC;IACf,QAAQ,EAAE,QAAQ,CAAC;IACnB,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,wBAAwB,EAAE,OAAO,CAAC;IAClC,WAAW,EAAE,UAAU,CAAC;IACxB,eAAe,EAAE,MAAM,CAAC;gBAGtB,KAAK,SAAI,EACT,KAAK,SAAM,EACX,WAAW,SAAM,EACjB,WAAW,GAAE,MAAM,GAAG,IAAW,EACjC,eAAe,GAAE,MAAM,EAAO,EAC9B,QAAQ,GAAE,QAA6B,EACvC,QAAQ,SAAQ,EAChB,wBAAwB,UAAO;IAqBjC,IAAI,QAAQ,IAAI,UAAU,EAAE,CAE3B;IAED,MAAM,CAAC,UAAU,CAAC,CAAC,EAAE,MAAM,GAAG,OAAO;IAIrC,SAAS,CACP,UAAU,EAAE,MAAM,EAAE,EACpB,MAAM,EAAE,MAAM,EAAE,EAChB,KAAK,EAAE,MAAM,EACb,aAAa,EAAE,OAAO,GACrB,UAAU,GAAG,IAAI;IA2BpB,SAAS,CAAC,IAAI,CAAC,EAAE;QAAE,KAAK,CAAC,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;KAAE,EAAE,WAAW,SAAI,GAAG,IAAI;IAInE,OAAO,CAAC,SAAS;IA8CjB,kBAAkB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE;IAQ7C,aAAa,CAAC,OAAO,EAAE,MAAM,GAAG,CAAC,UAAU,EAAE,MAAM,CAAC;IAgDpD,OAAO,CAAC,WAAW;IAQnB,mBAAmB,IAAI,MAAM;IAQ7B,uBAAuB,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,EAAE;IAkB1D,QAAQ,CAAC,UAAU,CACjB,QAAQ,EAAE,IAAI,EACd,MAAM,EAAE,MAAM,EAAE,EAChB,KAAK,EAAE,MAAM,EACb,aAAa,EAAE,OAAO,GACrB,UAAU,GAAG,IAAI;IAEpB,QAAQ,CAAC,kBAAkB,CAAC,QAAQ,EAAE,IAAI,EAAE,OAAO,EAAE,UAAU,GAAG,IAAI;IAEtE,QAAQ,CAAC,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,aAAa,EAAE,OAAO,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC;IAEjG,QAAQ,CAAC,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE;IAEjE,QAAQ,CAAC,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,kBAAkB,CAAC,EAAE,kBAAkB,GAAG,UAAU,GAAG,IAAI;CAC5F;AAED,qBAAa,KAAM,SAAQ,SAAS;IAClC,UAAU,CAAC,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,aAAa,EAAE,OAAO,GAAG,UAAU,GAAG,IAAI;IAyCtG,kBAAkB,CAAC,QAAQ,EAAE,IAAI,EAAE,OAAO,EAAE,UAAU,GAAG,IAAI;IA2E7D,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,aAAa,EAAE,OAAO,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC;IAgCxF,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE;IAOxD,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,kBAAkB,GAAE,kBAA4B,GAAG,UAAU,GAAG,IAAI;CA4B5F"}