@wener/common 2.0.5 → 2.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/ai/qwen3vl/index.js +1 -1
- package/lib/ai/qwen3vl/utils.js +15 -15
- package/lib/ai/vision/DocLayoutElementTypeSchema.js +22 -22
- package/lib/ai/vision/ImageAnnotationSchema.js +63 -47
- package/lib/ai/vision/index.js +2 -2
- package/lib/ai/vision/resolveImageAnnotation.js +81 -95
- package/lib/cn/ChineseResidentIdNo.js +55 -41
- package/lib/cn/ChineseResidentIdNo.mod.js +6 -1
- package/lib/cn/ChineseResidentIdNo.test.js +22 -21
- package/lib/cn/DivisionCode.js +220 -235
- package/lib/cn/DivisionCode.mod.js +6 -1
- package/lib/cn/DivisionCode.test.js +92 -121
- package/lib/cn/Mod11.js +18 -37
- package/lib/cn/Mod31.js +23 -41
- package/lib/cn/UnifiedSocialCreditCode.js +143 -137
- package/lib/cn/UnifiedSocialCreditCode.mod.js +6 -1
- package/lib/cn/UnifiedSocialCreditCode.test.js +21 -15
- package/lib/cn/formatChineseAmount.js +46 -71
- package/lib/cn/index.js +6 -6
- package/lib/cn/mod.js +5 -3
- package/lib/cn/parseChineseNumber.js +81 -85
- package/lib/cn/parseChineseNumber.test.js +183 -261
- package/lib/cn/pinyin/cartesianProduct.js +19 -19
- package/lib/cn/pinyin/cartesianProduct.test.js +78 -178
- package/lib/cn/pinyin/loader.js +13 -11
- package/lib/cn/pinyin/preload.js +2 -1
- package/lib/cn/pinyin/toPinyin.test.js +149 -161
- package/lib/cn/pinyin/toPinyinPure.js +28 -23
- package/lib/cn/pinyin/transform.js +11 -11
- package/lib/cn/types.d.js +2 -2
- package/lib/consola/createStandardConsolaReporter.js +14 -15
- package/lib/consola/formatLogObject.js +149 -133
- package/lib/consola/formatLogObject.test.js +167 -178
- package/lib/consola/index.js +2 -2
- package/lib/data/formatSort.js +14 -12
- package/lib/data/formatSort.test.js +33 -33
- package/lib/data/index.js +3 -3
- package/lib/data/maybeNumber.js +23 -23
- package/lib/data/parseSort.js +75 -68
- package/lib/data/parseSort.test.js +196 -187
- package/lib/data/resolvePagination.js +38 -39
- package/lib/data/resolvePagination.test.js +228 -218
- package/lib/data/types.d.js +2 -2
- package/lib/dayjs/dayjs.js +20 -20
- package/lib/dayjs/formatDuration.js +56 -56
- package/lib/dayjs/formatDuration.test.js +63 -77
- package/lib/dayjs/index.js +4 -4
- package/lib/dayjs/parseDuration.js +21 -26
- package/lib/dayjs/parseRelativeTime.js +65 -66
- package/lib/dayjs/parseRelativeTime.test.js +227 -243
- package/lib/dayjs/resolveRelativeTime.js +73 -72
- package/lib/dayjs/resolveRelativeTime.test.js +296 -307
- package/lib/decimal/index.js +1 -1
- package/lib/decimal/parseDecimal.js +12 -12
- package/lib/drain3/Drain.js +303 -338
- package/lib/drain3/LogCluster.js +25 -25
- package/lib/drain3/Node.js +24 -24
- package/lib/drain3/TemplateMiner.js +197 -196
- package/lib/drain3/index.js +5 -5
- package/lib/drain3/persistence/FilePersistence.js +19 -19
- package/lib/drain3/persistence/MemoryPersistence.js +8 -8
- package/lib/drain3/persistence/PersistenceHandler.js +2 -2
- package/lib/drain3/types.js +2 -2
- package/lib/emittery/emitter.js +7 -7
- package/lib/emittery/index.js +1 -1
- package/lib/foundation/schema/SexType.js +15 -12
- package/lib/foundation/schema/index.js +1 -1
- package/lib/foundation/schema/parseSexType.js +15 -16
- package/lib/foundation/schema/types.js +8 -6
- package/lib/fs/FileSystemError.js +18 -18
- package/lib/fs/IFileSystem.d.js +2 -2
- package/lib/fs/MemoryFileSystem.test.js +172 -181
- package/lib/fs/createBrowserFileSystem.js +222 -235
- package/lib/fs/createMemoryFileSystem.js +472 -510
- package/lib/fs/createSandboxFileSystem.js +102 -101
- package/lib/fs/createWebDavFileSystem.js +162 -149
- package/lib/fs/createWebFileSystem.js +197 -220
- package/lib/fs/findMimeType.js +14 -14
- package/lib/fs/index.js +7 -7
- package/lib/fs/minio/createMinioFileSystem.js +959 -956
- package/lib/fs/minio/index.js +1 -1
- package/lib/fs/orpc/FileSystemContract.js +57 -57
- package/lib/fs/orpc/createContractClientFileSystem.js +88 -88
- package/lib/fs/orpc/index.js +2 -2
- package/lib/fs/orpc/server/createFileSystemContractImpl.js +62 -60
- package/lib/fs/orpc/server/index.js +1 -1
- package/lib/fs/s3/createS3MiniFileSystem.js +756 -737
- package/lib/fs/s3/index.js +1 -1
- package/lib/fs/s3/s3mini.test.js +524 -553
- package/lib/fs/scandir.js +56 -56
- package/lib/fs/server/createDatabaseFileSystem.js +834 -741
- package/lib/fs/server/createNodeFileSystem.js +407 -405
- package/lib/fs/server/dbfs.test.js +201 -214
- package/lib/fs/server/index.js +1 -1
- package/lib/fs/server/loadTestDatabase.js +40 -43
- package/lib/fs/tests/runFileSystemTest.js +352 -316
- package/lib/fs/types.js +17 -20
- package/lib/fs/utils/getFileUrl.js +24 -30
- package/lib/fs/utils.js +17 -17
- package/lib/fs/webdav/index.js +1 -1
- package/lib/index.js +2 -2
- package/lib/jsonschema/JsonSchema.js +216 -155
- package/lib/jsonschema/JsonSchema.test.js +123 -124
- package/lib/jsonschema/forEachJsonSchema.js +41 -41
- package/lib/jsonschema/index.js +2 -2
- package/lib/jsonschema/types.d.js +2 -2
- package/lib/meta/defineFileType.js +32 -38
- package/lib/meta/defineInit.js +39 -35
- package/lib/meta/defineMetadata.js +37 -34
- package/lib/meta/defineMetadata.test.js +13 -12
- package/lib/meta/index.js +3 -3
- package/lib/orpc/createOpenApiContractClient.js +26 -24
- package/lib/orpc/createRpcContractClient.js +37 -31
- package/lib/orpc/index.js +2 -2
- package/lib/orpc/resolveLinkPlugins.js +25 -25
- package/lib/password/PHC.js +187 -189
- package/lib/password/PHC.test.js +517 -535
- package/lib/password/Password.js +85 -80
- package/lib/password/Password.test.js +330 -364
- package/lib/password/createArgon2PasswordAlgorithm.js +50 -51
- package/lib/password/createBase64PasswordAlgorithm.js +11 -11
- package/lib/password/createBcryptPasswordAlgorithm.js +20 -18
- package/lib/password/createPBKDF2PasswordAlgorithm.js +65 -52
- package/lib/password/createScryptPasswordAlgorithm.js +74 -63
- package/lib/password/index.js +5 -5
- package/lib/password/server/index.js +1 -1
- package/lib/resource/Identifiable.js +2 -2
- package/lib/resource/ListQuery.js +42 -42
- package/lib/resource/getTitleOfResource.js +5 -5
- package/lib/resource/index.js +2 -2
- package/lib/resource/schema/AnyResourceSchema.js +91 -89
- package/lib/resource/schema/BaseResourceSchema.js +26 -26
- package/lib/resource/schema/ResourceActionType.js +117 -115
- package/lib/resource/schema/ResourceStatus.js +94 -92
- package/lib/resource/schema/ResourceType.js +25 -23
- package/lib/resource/schema/index.js +5 -5
- package/lib/resource/schema/types.js +86 -55
- package/lib/resource/schema/types.test.js +16 -13
- package/lib/s3/formatS3Url.js +60 -60
- package/lib/s3/formatS3Url.test.js +238 -261
- package/lib/s3/index.js +2 -2
- package/lib/s3/parseS3Url.js +61 -60
- package/lib/s3/parseS3Url.test.js +270 -269
- package/lib/schema/SchemaRegistry.js +41 -42
- package/lib/schema/SchemaRegistry.mod.js +1 -1
- package/lib/schema/TypeSchema.d.js +2 -2
- package/lib/schema/createSchemaData.js +113 -67
- package/lib/schema/findJsonSchemaByPath.js +28 -23
- package/lib/schema/formatZodError.js +112 -131
- package/lib/schema/formatZodError.test.js +192 -195
- package/lib/schema/getSchemaCache.js +7 -7
- package/lib/schema/getSchemaOptions.js +17 -16
- package/lib/schema/index.js +6 -6
- package/lib/schema/toJsonSchema.js +195 -189
- package/lib/schema/toJsonSchema.test.js +34 -26
- package/lib/schema/validate.js +105 -96
- package/lib/tools/generateSchema.js +40 -40
- package/lib/tools/renderJsonSchemaToMarkdownDoc.js +74 -74
- package/lib/utils/buildBaseUrl.js +8 -8
- package/lib/utils/buildRedactorFormSchema.js +54 -53
- package/lib/utils/getEstimateProcessTime.js +24 -19
- package/lib/utils/index.js +3 -3
- package/lib/utils/resolveFeatureOptions.js +9 -9
- package/package.json +14 -14
- package/src/ai/vision/index.ts +2 -2
- package/src/cn/index.ts +1 -2
- package/src/consola/index.ts +1 -1
- package/src/data/index.ts +3 -4
- package/src/data/resolvePagination.ts +2 -2
- package/src/dayjs/formatDuration.ts +8 -9
- package/src/dayjs/index.ts +1 -1
- package/src/dayjs/parseRelativeTime.ts +1 -1
- package/src/dayjs/resolveRelativeTime.ts +1 -1
- package/src/drain3/Drain.test.ts +2 -2
- package/src/drain3/index.ts +2 -4
- package/src/fs/createWebDavFileSystem.ts +2 -7
- package/src/fs/createWebFileSystem.ts +1 -1
- package/src/fs/index.ts +4 -4
- package/src/fs/minio/createMinioFileSystem.ts +2 -2
- package/src/fs/minio/index.ts +1 -1
- package/src/fs/s3/createS3MiniFileSystem.ts +1 -1
- package/src/fs/server/createDatabaseFileSystem.ts +84 -120
- package/src/fs/server/dbfs.test.ts +14 -10
- package/src/fs/server/index.ts +1 -0
- package/src/fs/server/loadTestDatabase.ts +8 -119
- package/src/jsonschema/index.ts +1 -1
- package/src/meta/index.ts +2 -3
- package/src/orm/createSqliteDialect.ts +17 -0
- package/src/orm/index.ts +1 -0
- package/src/orpc/createOpenApiContractClient.ts +1 -1
- package/src/orpc/index.ts +1 -1
- package/src/password/createArgon2PasswordAlgorithm.ts +1 -1
- package/src/password/index.ts +2 -2
- package/src/resource/index.ts +3 -3
- package/src/resource/schema/index.ts +4 -4
- package/src/s3/index.ts +1 -1
- package/src/schema/SchemaRegistry.ts +1 -1
- package/src/schema/createSchemaData.ts +1 -1
- package/src/schema/findJsonSchemaByPath.ts +1 -1
- package/src/schema/index.ts +5 -5
- package/src/schema/validate.ts +1 -1
- package/src/utils/buildRedactorFormSchema.ts +1 -1
- package/src/utils/formatNumber.ts +18 -0
- package/src/utils/formatPercent.ts +17 -0
- package/src/utils/index.ts +3 -3
- package/src/utils/resolveFeatureOptions.ts +1 -1
package/lib/drain3/LogCluster.js
CHANGED
|
@@ -1,38 +1,38 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Represents a cluster of similar log messages with a common template.
|
|
3
3
|
*/ export class LogCluster {
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
4
|
+
clusterId;
|
|
5
|
+
logTemplateTokens;
|
|
6
|
+
size;
|
|
7
|
+
constructor(clusterId, logTemplateTokens, size = 1) {
|
|
8
|
+
this.clusterId = clusterId;
|
|
9
|
+
this.logTemplateTokens = logTemplateTokens;
|
|
10
|
+
this.size = size;
|
|
11
|
+
}
|
|
12
|
+
/**
|
|
13
13
|
* Returns the template as a space-separated string.
|
|
14
14
|
*/ getTemplate() {
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
15
|
+
return this.logTemplateTokens.join(' ');
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
18
|
* Returns a string representation of the cluster.
|
|
19
19
|
*/ toString() {
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
20
|
+
return `ID=${this.clusterId.toString().padEnd(5)} : size=${this.size.toString().padEnd(10)}: ${this.getTemplate()}`;
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
23
|
* Creates a LogCluster from JSON data.
|
|
24
24
|
*/ static fromJSON(data) {
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
25
|
+
return new LogCluster(data.clusterId, data.logTemplateTokens, data.size);
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
28
|
* Converts the cluster to JSON for serialization.
|
|
29
29
|
*/ toJSON() {
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
30
|
+
return {
|
|
31
|
+
clusterId: this.clusterId,
|
|
32
|
+
logTemplateTokens: this.logTemplateTokens,
|
|
33
|
+
size: this.size,
|
|
34
|
+
};
|
|
35
|
+
}
|
|
36
36
|
}
|
|
37
37
|
|
|
38
|
-
//# sourceMappingURL=LogCluster.js.map
|
|
38
|
+
//# sourceMappingURL=LogCluster.js.map
|
package/lib/drain3/Node.js
CHANGED
|
@@ -1,39 +1,39 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Represents a node in the prefix tree used by Drain algorithm.
|
|
3
3
|
*/ export class Node {
|
|
4
|
-
|
|
4
|
+
/**
|
|
5
5
|
* Map of token keys to child nodes.
|
|
6
6
|
*/ keyToChildNode = new Map();
|
|
7
|
-
|
|
7
|
+
/**
|
|
8
8
|
* List of cluster IDs associated with this node.
|
|
9
9
|
*/ clusterIds = [];
|
|
10
|
-
|
|
10
|
+
/**
|
|
11
11
|
* Creates a new empty node.
|
|
12
12
|
*/ static newNode() {
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
13
|
+
return new Node();
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
16
|
* Converts the node to JSON for serialization.
|
|
17
17
|
*/ toJSON() {
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
18
|
+
const keyToChildNode = {};
|
|
19
|
+
for (const [key, node] of this.keyToChildNode) {
|
|
20
|
+
keyToChildNode[key] = node.toJSON();
|
|
21
|
+
}
|
|
22
|
+
return {
|
|
23
|
+
keyToChildNode,
|
|
24
|
+
clusterIds: this.clusterIds,
|
|
25
|
+
};
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
28
|
* Creates a Node from JSON data.
|
|
29
29
|
*/ static fromJSON(data) {
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
30
|
+
const node = new Node();
|
|
31
|
+
node.clusterIds = data.clusterIds;
|
|
32
|
+
for (const [key, childData] of Object.entries(data.keyToChildNode)) {
|
|
33
|
+
node.keyToChildNode.set(key, Node.fromJSON(childData));
|
|
34
|
+
}
|
|
35
|
+
return node;
|
|
36
|
+
}
|
|
37
37
|
}
|
|
38
38
|
|
|
39
|
-
//# sourceMappingURL=Node.js.map
|
|
39
|
+
//# sourceMappingURL=Node.js.map
|
|
@@ -1,204 +1,205 @@
|
|
|
1
|
-
import { Drain } from
|
|
1
|
+
import { Drain } from './Drain.js';
|
|
2
2
|
/**
|
|
3
3
|
* High-level API for log template mining with persistence support.
|
|
4
4
|
*
|
|
5
5
|
* TemplateMiner wraps the Drain algorithm and provides convenience methods
|
|
6
6
|
* for extracting parameters and managing persistence.
|
|
7
7
|
*/ export class TemplateMiner {
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
8
|
+
drain;
|
|
9
|
+
persistence;
|
|
10
|
+
constructor(drain, persistence) {
|
|
11
|
+
this.drain = drain;
|
|
12
|
+
this.persistence = persistence;
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* Adds a log message and returns the update type, cluster, template, and cluster count.
|
|
16
|
+
*
|
|
17
|
+
* @param content The log message content
|
|
18
|
+
* @returns Object containing updateType, cluster, templateMined, and clusterCount
|
|
19
|
+
*/ async addLogMessage(content) {
|
|
20
|
+
const { cluster, updateType } = this.drain.addLogMessage(content);
|
|
21
|
+
const templateMined = cluster.getTemplate();
|
|
22
|
+
const clusterCount = this.drain.getClusters().length;
|
|
23
|
+
if (updateType !== 'none') {
|
|
24
|
+
await this.saveState();
|
|
25
|
+
}
|
|
26
|
+
return {
|
|
27
|
+
updateType,
|
|
28
|
+
cluster,
|
|
29
|
+
templateMined,
|
|
30
|
+
clusterCount,
|
|
31
|
+
};
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Matches a log message against an already existing cluster.
|
|
35
|
+
*
|
|
36
|
+
* @param content Log message to match
|
|
37
|
+
* @param strategy Search strategy
|
|
38
|
+
* @returns Matched cluster or null
|
|
39
|
+
*/ match(content, strategy = 'never') {
|
|
40
|
+
return this.drain.match(content, strategy);
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* Extracts parameters from a log message according to a provided template.
|
|
44
|
+
*
|
|
45
|
+
* @param logTemplate Template generated by AddLogMessage
|
|
46
|
+
* @param logMessage The log message to extract parameters from
|
|
47
|
+
* @returns Array of extracted parameters, or null if template doesn't match
|
|
48
|
+
*/ extractParameters(logTemplate, logMessage) {
|
|
49
|
+
// Apply delimiters
|
|
50
|
+
let processedMessage = logMessage;
|
|
51
|
+
for (const delimiter of this.drain.extraDelimiters) {
|
|
52
|
+
// Escape delimiter for regex if it contains special characters
|
|
53
|
+
const escapedDelimiter = delimiter.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
54
|
+
processedMessage = processedMessage.replace(new RegExp(escapedDelimiter, 'g'), ' ');
|
|
55
|
+
}
|
|
56
|
+
const { templateRegex, paramGroupNameToMaskName } = this.getTemplateParameterExtractionRegex(logTemplate);
|
|
57
|
+
// Create regex from template
|
|
58
|
+
const regex = new RegExp(templateRegex);
|
|
59
|
+
// Match the log message against the template
|
|
60
|
+
const match = processedMessage.match(regex);
|
|
61
|
+
// Template doesn't match
|
|
62
|
+
if (match === null) {
|
|
63
|
+
return null;
|
|
64
|
+
}
|
|
65
|
+
// Extract named captures
|
|
66
|
+
const extractedParameters = [];
|
|
67
|
+
// Get all named groups from the match
|
|
68
|
+
if (match.groups) {
|
|
69
|
+
for (const [groupName, value] of Object.entries(match.groups)) {
|
|
70
|
+
if (value !== undefined) {
|
|
71
|
+
const maskName = paramGroupNameToMaskName.get(groupName);
|
|
72
|
+
if (maskName !== undefined) {
|
|
73
|
+
extractedParameters.push({
|
|
74
|
+
value,
|
|
75
|
+
maskName,
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
// Also check by index if groups are not available (fallback)
|
|
82
|
+
// The regex exec method provides better named group support
|
|
83
|
+
const execResult = regex.exec(processedMessage);
|
|
84
|
+
if (execResult !== null && execResult.groups === undefined) {
|
|
85
|
+
// Fallback: extract by order (less reliable but works)
|
|
86
|
+
const indices = Array.from(
|
|
87
|
+
{
|
|
88
|
+
length: execResult.length - 1,
|
|
89
|
+
},
|
|
90
|
+
(_, i) => i + 1,
|
|
91
|
+
);
|
|
92
|
+
let paramIndex = 0;
|
|
93
|
+
for (const index of indices) {
|
|
94
|
+
const value = execResult[index];
|
|
95
|
+
if (value !== undefined) {
|
|
96
|
+
// Find corresponding mask name by order
|
|
97
|
+
// This is a fallback - ideally we should use named groups
|
|
98
|
+
const maskName = paramGroupNameToMaskName.get(`p_${paramIndex}`);
|
|
99
|
+
if (maskName !== undefined) {
|
|
100
|
+
extractedParameters.push({
|
|
101
|
+
value,
|
|
102
|
+
maskName,
|
|
103
|
+
});
|
|
104
|
+
}
|
|
105
|
+
paramIndex++;
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
return extractedParameters.length > 0 ? extractedParameters : null;
|
|
110
|
+
}
|
|
111
|
+
/**
|
|
112
|
+
* Gets the template parameter extraction regex and mapping.
|
|
113
|
+
*
|
|
114
|
+
* @param logTemplate The log template
|
|
115
|
+
* @returns Tuple of [templateRegex, paramGroupNameToMaskName]
|
|
116
|
+
*/ getTemplateParameterExtractionRegex(logTemplate) {
|
|
117
|
+
const paramGroupNameToMaskName = new Map();
|
|
118
|
+
let paramNameCounter = 0;
|
|
119
|
+
const getNextParamName = () => {
|
|
120
|
+
const paramGroupName = `p_${paramNameCounter}`;
|
|
121
|
+
paramNameCounter++;
|
|
122
|
+
return paramGroupName;
|
|
123
|
+
};
|
|
124
|
+
// Create a named group with the respective patterns for the given mask-name
|
|
125
|
+
const createCaptureRegex = (maskName) => {
|
|
126
|
+
const allowedPatterns = [];
|
|
127
|
+
if (maskName === '*') {
|
|
128
|
+
allowedPatterns.push('.+?');
|
|
129
|
+
}
|
|
130
|
+
// Give each capture group a unique name to avoid conflict
|
|
131
|
+
const paramGroupName = getNextParamName();
|
|
132
|
+
paramGroupNameToMaskName.set(paramGroupName, maskName);
|
|
133
|
+
const joinedPatterns = allowedPatterns.join('|');
|
|
134
|
+
const captureRegex = `(?<${paramGroupName}>${joinedPatterns})`;
|
|
135
|
+
return captureRegex;
|
|
136
|
+
};
|
|
137
|
+
// For every mask in the template, replace it with a named group
|
|
138
|
+
const maskNames = new Set();
|
|
139
|
+
// The drain catch-all mask
|
|
140
|
+
maskNames.add('*');
|
|
141
|
+
// Escape the template for regex
|
|
142
|
+
let templateRegex = this.escapeRegex(logTemplate);
|
|
143
|
+
// Replace each mask name with a proper regex that captures it
|
|
144
|
+
for (const maskName of maskNames) {
|
|
145
|
+
const searchStr = `<${this.escapeRegex(maskName)}>`;
|
|
146
|
+
// Replace one-by-one to get a new param group name for each replacement
|
|
147
|
+
while (true) {
|
|
148
|
+
const repStr = createCaptureRegex(maskName);
|
|
149
|
+
const templateRegexNew = templateRegex.replace(searchStr, repStr);
|
|
150
|
+
// Break when all replaces for this mask are done
|
|
151
|
+
if (templateRegexNew === templateRegex) {
|
|
152
|
+
break;
|
|
153
|
+
}
|
|
154
|
+
templateRegex = templateRegexNew;
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
// Match also messages with multiple spaces or other whitespace chars between tokens
|
|
158
|
+
templateRegex = templateRegex.replace(/\\ /g, '\\s+');
|
|
159
|
+
templateRegex = `^${templateRegex}$`;
|
|
160
|
+
return {
|
|
161
|
+
templateRegex,
|
|
162
|
+
paramGroupNameToMaskName,
|
|
163
|
+
};
|
|
164
|
+
}
|
|
165
|
+
/**
|
|
166
|
+
* Escapes special regex characters in a string.
|
|
167
|
+
*/ escapeRegex(str) {
|
|
168
|
+
return str.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
169
|
+
}
|
|
170
|
+
/**
|
|
171
|
+
* Loads the Drain state from persistence.
|
|
172
|
+
*
|
|
173
|
+
* @throws Error if loading fails
|
|
174
|
+
*/ async loadState() {
|
|
175
|
+
const state = await this.persistence.load();
|
|
176
|
+
if (state === null || (typeof state === 'string' && state.length === 0)) {
|
|
177
|
+
throw new Error('saved state not found');
|
|
178
|
+
}
|
|
179
|
+
try {
|
|
180
|
+
const stateStr = typeof state === 'string' ? state : new TextDecoder().decode(state);
|
|
181
|
+
const data = JSON.parse(stateStr);
|
|
182
|
+
this.drain = Drain.fromJSON(data);
|
|
183
|
+
} catch (error) {
|
|
184
|
+
throw new Error(`failed to unmarshal state: ${error instanceof Error ? error.message : String(error)}`);
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
/**
|
|
188
|
+
* Saves the Drain state to persistence.
|
|
189
|
+
*
|
|
190
|
+
* @throws Error if saving fails
|
|
191
|
+
*/ async saveState() {
|
|
192
|
+
try {
|
|
193
|
+
const state = JSON.stringify(this.drain.toJSON());
|
|
194
|
+
await this.persistence.save(state);
|
|
195
|
+
} catch (error) {
|
|
196
|
+
throw new Error(`failed to save state: ${error instanceof Error ? error.message : String(error)}`);
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
/**
|
|
200
|
+
* Gets the underlying Drain instance.
|
|
201
|
+
*/ getDrain() {
|
|
202
|
+
return this.drain;
|
|
203
|
+
}
|
|
203
204
|
}
|
|
204
205
|
//# sourceMappingURL=TemplateMiner.js.map
|
package/lib/drain3/index.js
CHANGED
|
@@ -23,9 +23,9 @@
|
|
|
23
23
|
* const miner = new TemplateMiner(drain, new MemoryPersistence());
|
|
24
24
|
* await miner.addLogMessage('[INFO] User 123 logged in');
|
|
25
25
|
* ```
|
|
26
|
-
*/ export { Drain } from
|
|
27
|
-
export { LogCluster } from
|
|
28
|
-
export { Node } from
|
|
29
|
-
export {
|
|
30
|
-
export {
|
|
26
|
+
*/ export { Drain } from './Drain.js';
|
|
27
|
+
export { LogCluster } from './LogCluster.js';
|
|
28
|
+
export { Node } from './Node.js';
|
|
29
|
+
export { MemoryPersistence } from './persistence/MemoryPersistence.js';
|
|
30
|
+
export { TemplateMiner } from './TemplateMiner.js';
|
|
31
31
|
//# sourceMappingURL=index.js.map
|
|
@@ -1,24 +1,24 @@
|
|
|
1
|
-
import { readFile, writeFile } from
|
|
1
|
+
import { readFile, writeFile } from 'node:fs/promises';
|
|
2
2
|
/**
|
|
3
3
|
* File-based persistence handler that saves and loads state from the filesystem.
|
|
4
4
|
*/ export class FilePersistence {
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
5
|
+
filePath;
|
|
6
|
+
constructor(filePath) {
|
|
7
|
+
this.filePath = filePath;
|
|
8
|
+
}
|
|
9
|
+
async save(state) {
|
|
10
|
+
await writeFile(this.filePath, state, 'utf8');
|
|
11
|
+
}
|
|
12
|
+
async load() {
|
|
13
|
+
try {
|
|
14
|
+
return await readFile(this.filePath, 'utf8');
|
|
15
|
+
} catch (error) {
|
|
16
|
+
if (error?.code === 'ENOENT') {
|
|
17
|
+
return null;
|
|
18
|
+
}
|
|
19
|
+
throw error;
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
22
|
}
|
|
23
23
|
|
|
24
|
-
//# sourceMappingURL=FilePersistence.js.map
|
|
24
|
+
//# sourceMappingURL=FilePersistence.js.map
|
|
@@ -2,17 +2,17 @@
|
|
|
2
2
|
* In-memory persistence handler that stores state in memory.
|
|
3
3
|
* Useful for testing or when persistence is not needed.
|
|
4
4
|
*/ export class MemoryPersistence {
|
|
5
|
-
|
|
6
|
-
|
|
5
|
+
state = null;
|
|
6
|
+
/**
|
|
7
7
|
* Saves the state to memory.
|
|
8
8
|
*/ async save(state) {
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
9
|
+
this.state = state;
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
12
|
* Loads the state from memory.
|
|
13
13
|
*/ async load() {
|
|
14
|
-
|
|
15
|
-
|
|
14
|
+
return this.state;
|
|
15
|
+
}
|
|
16
16
|
}
|
|
17
17
|
|
|
18
|
-
//# sourceMappingURL=MemoryPersistence.js.map
|
|
18
|
+
//# sourceMappingURL=MemoryPersistence.js.map
|
package/lib/drain3/types.js
CHANGED