mynth-logger 2.1.0 → 2.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/CI.yaml +0 -3
- package/.github/workflows/publish.yml +5 -4
- package/dist/src/format.d.ts +3 -1
- package/dist/src/format.js +10 -4
- package/dist/src/logging.d.ts +2 -1
- package/dist/src/logging.js +5 -2
- package/dist/src/redact.d.ts +39 -2
- package/dist/src/redact.js +130 -62
- package/dist/tests/app.js +5 -1
- package/dist/tests/redact.test.d.ts +1 -0
- package/dist/tests/redact.test.js +14 -0
- package/package.json +2 -2
- package/src/format.ts +12 -4
- package/src/logging.ts +7 -7
- package/src/redact.ts +230 -73
- package/tests/app.ts +9 -1
- package/tests/redact.test.ts +22 -0
|
@@ -11,11 +11,13 @@ on:
|
|
|
11
11
|
- patch
|
|
12
12
|
- minor
|
|
13
13
|
|
|
14
|
+
permissions:
|
|
15
|
+
id-token: write
|
|
16
|
+
contents: read
|
|
17
|
+
|
|
14
18
|
jobs:
|
|
15
19
|
publish:
|
|
16
20
|
runs-on: ubuntu-latest
|
|
17
|
-
permissions:
|
|
18
|
-
contents: write
|
|
19
21
|
steps:
|
|
20
22
|
- name: Checkout code
|
|
21
23
|
uses: actions/checkout@v6.0.2
|
|
@@ -26,6 +28,7 @@ jobs:
|
|
|
26
28
|
uses: actions/setup-node@v6.2.0
|
|
27
29
|
with:
|
|
28
30
|
node-version: 24
|
|
31
|
+
registry-url: https://registry.npmjs.org
|
|
29
32
|
|
|
30
33
|
- uses: pnpm/action-setup@v4
|
|
31
34
|
|
|
@@ -43,8 +46,6 @@ jobs:
|
|
|
43
46
|
|
|
44
47
|
- name: Publish to npm
|
|
45
48
|
run: npm publish
|
|
46
|
-
env:
|
|
47
|
-
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
|
|
48
49
|
|
|
49
50
|
- name: Push to git repo
|
|
50
51
|
run: git push --follow-tags
|
package/dist/src/format.d.ts
CHANGED
package/dist/src/format.js
CHANGED
|
@@ -1,6 +1,12 @@
|
|
|
1
1
|
import { stringify } from "@ungap/structured-clone/json";
|
|
2
2
|
import { type } from "arktype";
|
|
3
|
-
import {
|
|
3
|
+
import { createRedact } from "./redact.js";
|
|
4
|
+
const config = {
|
|
5
|
+
redact: createRedact({}),
|
|
6
|
+
};
|
|
7
|
+
const updateConfig = (newConfig) => {
|
|
8
|
+
config.redact = createRedact(newConfig);
|
|
9
|
+
};
|
|
4
10
|
const ErrorType = type({
|
|
5
11
|
message: "string",
|
|
6
12
|
"stack?": "string",
|
|
@@ -11,7 +17,7 @@ const formatItem = (item) => {
|
|
|
11
17
|
// Remove colors from strings
|
|
12
18
|
if (typeof item === "string")
|
|
13
19
|
// biome-ignore lint/suspicious/noControlCharactersInRegex: stripping ANSI escape codes
|
|
14
|
-
return redact(item.replace(/\x1b\[[0-9;]*m/g, ""));
|
|
20
|
+
return config.redact(item.replace(/\x1b\[[0-9;]*m/g, ""));
|
|
15
21
|
// Check if this is an Error
|
|
16
22
|
const error = ErrorType(item);
|
|
17
23
|
if (!(error instanceof type.errors))
|
|
@@ -24,9 +30,9 @@ const formatItem = (item) => {
|
|
|
24
30
|
return String(item);
|
|
25
31
|
}
|
|
26
32
|
})();
|
|
27
|
-
return redact(stringified.replace(/^'|'$/g, ""));
|
|
33
|
+
return config.redact(stringified.replace(/^'|'$/g, ""));
|
|
28
34
|
};
|
|
29
35
|
const format = (items) => Array.from(items)
|
|
30
36
|
.map((item) => formatItem(item))
|
|
31
37
|
.join(" ");
|
|
32
|
-
export { format };
|
|
38
|
+
export { format, updateConfig };
|
package/dist/src/logging.d.ts
CHANGED
package/dist/src/logging.js
CHANGED
|
@@ -1,14 +1,17 @@
|
|
|
1
1
|
import { createConsola } from "consola";
|
|
2
|
+
import { updateConfig } from "./format.js";
|
|
2
3
|
import DatadogReporter from "./reporters/datadog.js";
|
|
3
4
|
import DiscordReporter from "./reporters/discord.js";
|
|
4
|
-
const setupLogging = () => {
|
|
5
|
+
const setupLogging = (config = {}) => {
|
|
6
|
+
updateConfig(config);
|
|
5
7
|
const consola = createConsola({ fancy: true, level: 5 });
|
|
6
8
|
if (process.env.NODE_ENV === "production")
|
|
7
9
|
consola.setReporters([DatadogReporter]);
|
|
8
10
|
// Set Discord reporter as first so it can remove
|
|
9
11
|
// Discord-related config before other reporters process the
|
|
10
12
|
// log
|
|
11
|
-
|
|
13
|
+
else
|
|
14
|
+
consola.setReporters([DiscordReporter, ...consola.options.reporters]);
|
|
12
15
|
consola.wrapConsole();
|
|
13
16
|
return consola;
|
|
14
17
|
};
|
package/dist/src/redact.d.ts
CHANGED
|
@@ -1,2 +1,39 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
1
|
+
/**
|
|
2
|
+
* Configurable redaction for strings that *look like secrets*:
|
|
3
|
+
* - hex (optionally 0x-prefixed)
|
|
4
|
+
* - base64 blobs
|
|
5
|
+
* - base64url blobs
|
|
6
|
+
* - base58 blobs
|
|
7
|
+
* - BIP39 mnemonics (validated)
|
|
8
|
+
*
|
|
9
|
+
* Key idea:
|
|
10
|
+
* Each detector can be given "allow" context rules that prevent redaction when
|
|
11
|
+
* extra surrounding context indicates the value is safe/expected.
|
|
12
|
+
*/
|
|
13
|
+
type ContextRule = {
|
|
14
|
+
/**
|
|
15
|
+
* Test this regex against a slice of the input around the match.
|
|
16
|
+
* If it matches, the match is NOT redacted.
|
|
17
|
+
*/
|
|
18
|
+
re: RegExp;
|
|
19
|
+
/** How many chars to include before the match when building the slice. */
|
|
20
|
+
before?: number;
|
|
21
|
+
/** How many chars to include after the match when building the slice. */
|
|
22
|
+
after?: number;
|
|
23
|
+
};
|
|
24
|
+
type DetectorConfig = {
|
|
25
|
+
/**
|
|
26
|
+
* Any rule match => do not redact that specific match.
|
|
27
|
+
*/
|
|
28
|
+
allow?: ContextRule[];
|
|
29
|
+
};
|
|
30
|
+
type RedactConfig = {
|
|
31
|
+
hex?: DetectorConfig;
|
|
32
|
+
base64?: DetectorConfig;
|
|
33
|
+
base64url?: DetectorConfig;
|
|
34
|
+
base58?: DetectorConfig;
|
|
35
|
+
mnemonic?: DetectorConfig;
|
|
36
|
+
};
|
|
37
|
+
declare const createRedact: (config: RedactConfig) => (text: string) => string;
|
|
38
|
+
export { createRedact };
|
|
39
|
+
export type { RedactConfig };
|
package/dist/src/redact.js
CHANGED
|
@@ -2,67 +2,135 @@ import { DeepRedact } from "@hackylabs/deep-redact/index.ts";
|
|
|
2
2
|
import { validateMnemonic } from "@scure/bip39";
|
|
3
3
|
import { wordlist } from "@scure/bip39/wordlists/english.js";
|
|
4
4
|
const replacement = "[REDACTED]";
|
|
5
|
+
const cloneRegex = (re) => {
|
|
6
|
+
const flags = re.flags.includes("g") ? re.flags : `${re.flags}g`;
|
|
7
|
+
return new RegExp(re.source, flags);
|
|
8
|
+
};
|
|
9
|
+
const sliceAround = (value, offset, matchLen, rule) => {
|
|
10
|
+
const before = rule.before ?? 10;
|
|
11
|
+
const after = rule.after ?? 0;
|
|
12
|
+
const start = Math.max(0, offset - before);
|
|
13
|
+
const end = Math.min(value.length, offset + matchLen + after);
|
|
14
|
+
return value.slice(start, end);
|
|
15
|
+
};
|
|
16
|
+
const shouldAllowByRules = (value, match, offset, rules) => {
|
|
17
|
+
if (!rules?.length)
|
|
18
|
+
return false;
|
|
19
|
+
for (const rule of rules) {
|
|
20
|
+
const chunk = sliceAround(value, offset, match.length, rule);
|
|
21
|
+
// If a consumer provides a /g or /y regex, .test() mutates lastIndex.
|
|
22
|
+
// Reset to ensure consistent behavior.
|
|
23
|
+
rule.re.lastIndex = 0;
|
|
24
|
+
if (rule.re.test(chunk))
|
|
25
|
+
return true;
|
|
26
|
+
}
|
|
27
|
+
return false;
|
|
28
|
+
};
|
|
29
|
+
const getReplaceMeta = (args) => {
|
|
30
|
+
const offset = args.at(-2);
|
|
31
|
+
const whole = args.at(-1);
|
|
32
|
+
if (typeof offset !== "number")
|
|
33
|
+
return null;
|
|
34
|
+
if (typeof whole !== "string")
|
|
35
|
+
return null;
|
|
36
|
+
return { offset, whole };
|
|
37
|
+
};
|
|
38
|
+
const replaceAllMatchesWithContext = (value, pattern, replacement, allow) => {
|
|
39
|
+
const re = cloneRegex(pattern);
|
|
40
|
+
return value.replace(re, (...args) => {
|
|
41
|
+
const match = args[0];
|
|
42
|
+
if (typeof match !== "string")
|
|
43
|
+
return replacement;
|
|
44
|
+
const meta = getReplaceMeta(args);
|
|
45
|
+
if (!meta)
|
|
46
|
+
return replacement;
|
|
47
|
+
if (shouldAllowByRules(meta.whole, match, meta.offset, allow))
|
|
48
|
+
return match;
|
|
49
|
+
return replacement;
|
|
50
|
+
});
|
|
51
|
+
};
|
|
5
52
|
/**
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
* -
|
|
9
|
-
* - long base64 blobs
|
|
10
|
-
* - long base58 blobs
|
|
11
|
-
* - mnemonic seed phrases (validated via BIP39 english wordlist)
|
|
53
|
+
* BIP39 contextual replacer:
|
|
54
|
+
* - Only redacts if the captured phrase validates as a BIP39 mnemonic
|
|
55
|
+
* - Still supports allow-rules (to suppress redaction in “safe” contexts)
|
|
12
56
|
*/
|
|
13
|
-
|
|
14
|
-
const
|
|
15
|
-
|
|
16
|
-
const
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
const
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
const
|
|
41
|
-
const
|
|
42
|
-
const
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
const
|
|
46
|
-
|
|
47
|
-
String.raw `(
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
{
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
57
|
+
const replaceBip39MnemonicMatchesWithContext = (value, pattern, replacement, allow) => {
|
|
58
|
+
const re = cloneRegex(pattern);
|
|
59
|
+
return value.replace(re, (...args) => {
|
|
60
|
+
const match = args[0];
|
|
61
|
+
const phrase = args[1];
|
|
62
|
+
if (typeof match !== "string")
|
|
63
|
+
return replacement;
|
|
64
|
+
if (typeof phrase !== "string")
|
|
65
|
+
return match;
|
|
66
|
+
const meta = getReplaceMeta(args);
|
|
67
|
+
if (!meta)
|
|
68
|
+
return match;
|
|
69
|
+
if (shouldAllowByRules(meta.whole, match, meta.offset, allow))
|
|
70
|
+
return match;
|
|
71
|
+
const normalized = phrase.trim().toLowerCase().replace(/\s+/g, " ");
|
|
72
|
+
if (!validateMnemonic(normalized, wordlist))
|
|
73
|
+
return match;
|
|
74
|
+
return match.replace(phrase, replacement);
|
|
75
|
+
});
|
|
76
|
+
};
|
|
77
|
+
const createRedactor = (config = {}) => {
|
|
78
|
+
const HEX_MIN_LEN = 16;
|
|
79
|
+
const BASE64_MIN_BLOCKS = 4;
|
|
80
|
+
const BASE58_MIN_LEN = 16;
|
|
81
|
+
const HEX = new RegExp(String.raw `(?<![a-fA-F0-9])(?:0x)?[a-fA-F0-9]{${HEX_MIN_LEN},}(?![a-fA-F0-9])`, "g");
|
|
82
|
+
const hexAllow = config.hex?.allow ?? [];
|
|
83
|
+
const BASE64 = new RegExp(String.raw `(?<![A-Za-z0-9+/=])(?:[A-Za-z0-9+/]{4}){${BASE64_MIN_BLOCKS},}(?:[A-Za-z0-9+/]{2,3})?(?:={0,2})(?![A-Za-z0-9+/=])`, "g");
|
|
84
|
+
const base64Allow = config.base64?.allow ?? [];
|
|
85
|
+
const BASE64URL = new RegExp(String.raw `(?<![A-Za-z0-9\-_])[A-Za-z0-9\-_]{16,}(?:={0,2})?(?![A-Za-z0-9\-_])`, "g");
|
|
86
|
+
const base64urlAllow = config.base64url?.allow ?? [];
|
|
87
|
+
const BASE58 = new RegExp(String.raw `(?<![1-9A-HJ-NP-Za-km-z])[1-9A-HJ-NP-Za-km-z]{${BASE58_MIN_LEN},}(?![1-9A-HJ-NP-Za-km-z])`, "g");
|
|
88
|
+
const base58Allow = config.base58?.allow ?? [];
|
|
89
|
+
const WORD = "[a-zA-Z]{2,8}";
|
|
90
|
+
const PHRASE_12_TO_24 = `(?:${WORD}\\s+){11,23}${WORD}`;
|
|
91
|
+
const MNEMONIC = new RegExp(String.raw `(?<![A-Za-z])(${PHRASE_12_TO_24})(?![A-Za-z])`, "gi");
|
|
92
|
+
const mnemonicAllow = config.mnemonic?.allow ?? [];
|
|
93
|
+
const stringTests = [
|
|
94
|
+
{
|
|
95
|
+
pattern: HEX,
|
|
96
|
+
replacer: (v, p) => replaceAllMatchesWithContext(v, p, replacement, hexAllow),
|
|
97
|
+
},
|
|
98
|
+
{
|
|
99
|
+
pattern: BASE64URL,
|
|
100
|
+
replacer: (v, p) => replaceAllMatchesWithContext(v, p, replacement, base64urlAllow),
|
|
101
|
+
},
|
|
102
|
+
{
|
|
103
|
+
pattern: BASE64,
|
|
104
|
+
replacer: (v, p) => replaceAllMatchesWithContext(v, p, replacement, base64Allow),
|
|
105
|
+
},
|
|
106
|
+
{
|
|
107
|
+
pattern: BASE58,
|
|
108
|
+
replacer: (v, p) => replaceAllMatchesWithContext(v, p, replacement, base58Allow),
|
|
109
|
+
},
|
|
110
|
+
{
|
|
111
|
+
pattern: MNEMONIC,
|
|
112
|
+
replacer: (v, p) => replaceBip39MnemonicMatchesWithContext(v, p, replacement, mnemonicAllow),
|
|
113
|
+
},
|
|
114
|
+
];
|
|
115
|
+
return new DeepRedact({
|
|
116
|
+
stringTests,
|
|
117
|
+
replacement,
|
|
118
|
+
serialise: false,
|
|
119
|
+
});
|
|
120
|
+
};
|
|
121
|
+
const createRedact = (config) => {
|
|
122
|
+
const redactor = createRedactor(config);
|
|
123
|
+
// DeepRedact applies only the first matching stringTest per pass.
|
|
124
|
+
// Run multiple passes until stable so different detectors can redact
|
|
125
|
+
// different tokens in the same string.
|
|
126
|
+
return (text) => {
|
|
127
|
+
let out = text;
|
|
128
|
+
while (true) {
|
|
129
|
+
const next = redactor.redact(out);
|
|
130
|
+
if (next === out)
|
|
131
|
+
return out;
|
|
132
|
+
out = next;
|
|
133
|
+
}
|
|
134
|
+
};
|
|
135
|
+
};
|
|
136
|
+
export { createRedact };
|
package/dist/tests/app.js
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import { setupLogging } from "../src/index.js";
|
|
2
2
|
const run = async () => {
|
|
3
|
-
setupLogging(
|
|
3
|
+
setupLogging({
|
|
4
|
+
hex: { allow: [{ re: /\b(intent|hash)\b/i }] },
|
|
5
|
+
});
|
|
4
6
|
console.log("Hello, this is a log");
|
|
5
7
|
console.info("Hello, this is an info log");
|
|
6
8
|
console.debug("Hello, this is a debug log");
|
|
@@ -8,6 +10,8 @@ const run = async () => {
|
|
|
8
10
|
console.error("Hello, this is an error log");
|
|
9
11
|
console.log("This is my seed phrase: ordinary quality amount solid fox guess peasant merit midnight noodle final brown pretty stable six fox beef engage waste uniform evoke flat survey crane");
|
|
10
12
|
console.log("This is my private key: 4fa7614bdd07ab15b11d2365466536ba160c06050ade0888a3aa985a5522ab22");
|
|
13
|
+
console.log("This is log for intent c8adcfef7255ad2c117f68111500997ab66de3c923e9ea9e71aaed5829d0acb8 and private key c99960eaeecb17d77d7c88f440383ccbaf93237471236a739185c0ebd62c5741");
|
|
14
|
+
console.log("this is my Discord webhook (https://discord.com/api/webhooks/1473664170267246593/V9xRwhrxrgEIJZL8inMMiEC8wDTwrMQye-VxLyBmkH6vbeTGNNBCjqkbtnUIA_dIVh3d) URL");
|
|
11
15
|
try {
|
|
12
16
|
throw new Error("An Error was thrown");
|
|
13
17
|
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { describe, expect, it } from "vitest";
|
|
2
|
+
import { createRedact } from "../src/redact.js";
|
|
3
|
+
describe("redact", () => {
|
|
4
|
+
it("redacts base64url", () => {
|
|
5
|
+
const redact = createRedact({});
|
|
6
|
+
const result = redact("Should base64url-ish V9xRwhrxrgEIJZL8inMMiEC8wDTwrMQye-VxLyBmkH6vbeTGNNBCjqkbtnUIA_dIVh3d data");
|
|
7
|
+
expect(result).toBe("Should base64url-ish [REDACTED] data");
|
|
8
|
+
});
|
|
9
|
+
it("redacts hex and base64url in same string", () => {
|
|
10
|
+
const redact = createRedact({});
|
|
11
|
+
const result = redact("Should redact private key (b49bd63e67e2cd11aba17befead483934939df828cb833a846c58661726d3b00) and API key (djOqmjzVb0GAGdqS0p0NtiEwvb6u1lx509JEkpDJLgnvMhmOMtBc9vqolpktd1OK7Xas)");
|
|
12
|
+
expect(result).toBe("Should redact private key ([REDACTED]) and API key ([REDACTED])");
|
|
13
|
+
});
|
|
14
|
+
});
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "mynth-logger",
|
|
3
|
-
"version": "2.1.
|
|
3
|
+
"version": "2.1.2",
|
|
4
4
|
"description": "Package to format logs for mynth microservices.",
|
|
5
5
|
"main": "dist/src/index.js",
|
|
6
6
|
"typings": "dist/src/index.d.ts",
|
|
@@ -32,7 +32,7 @@
|
|
|
32
32
|
"license": "MIT",
|
|
33
33
|
"repository": {
|
|
34
34
|
"type": "git",
|
|
35
|
-
"url": "
|
|
35
|
+
"url": "https://github.com/MynthAI/mynth-logger.git"
|
|
36
36
|
},
|
|
37
37
|
"peerDependencies": {
|
|
38
38
|
"arktype": "^2.1.29"
|
package/src/format.ts
CHANGED
|
@@ -1,6 +1,14 @@
|
|
|
1
1
|
import { stringify } from "@ungap/structured-clone/json";
|
|
2
2
|
import { type } from "arktype";
|
|
3
|
-
import {
|
|
3
|
+
import { createRedact, type RedactConfig } from "./redact.js";
|
|
4
|
+
|
|
5
|
+
const config = {
|
|
6
|
+
redact: createRedact({}),
|
|
7
|
+
};
|
|
8
|
+
|
|
9
|
+
const updateConfig = (newConfig: RedactConfig) => {
|
|
10
|
+
config.redact = createRedact(newConfig);
|
|
11
|
+
};
|
|
4
12
|
|
|
5
13
|
const ErrorType = type({
|
|
6
14
|
message: "string",
|
|
@@ -13,7 +21,7 @@ const formatItem = (item: unknown): string => {
|
|
|
13
21
|
// Remove colors from strings
|
|
14
22
|
if (typeof item === "string")
|
|
15
23
|
// biome-ignore lint/suspicious/noControlCharactersInRegex: stripping ANSI escape codes
|
|
16
|
-
return redact(item.replace(/\x1b\[[0-9;]*m/g, ""));
|
|
24
|
+
return config.redact(item.replace(/\x1b\[[0-9;]*m/g, ""));
|
|
17
25
|
|
|
18
26
|
// Check if this is an Error
|
|
19
27
|
const error = ErrorType(item);
|
|
@@ -27,7 +35,7 @@ const formatItem = (item: unknown): string => {
|
|
|
27
35
|
}
|
|
28
36
|
})();
|
|
29
37
|
|
|
30
|
-
return redact(stringified.replace(/^'|'$/g, ""));
|
|
38
|
+
return config.redact(stringified.replace(/^'|'$/g, ""));
|
|
31
39
|
};
|
|
32
40
|
|
|
33
41
|
const format = (items: unknown[]): string =>
|
|
@@ -35,4 +43,4 @@ const format = (items: unknown[]): string =>
|
|
|
35
43
|
.map((item) => formatItem(item))
|
|
36
44
|
.join(" ");
|
|
37
45
|
|
|
38
|
-
export { format };
|
|
46
|
+
export { format, updateConfig };
|
package/src/logging.ts
CHANGED
|
@@ -1,22 +1,22 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { createConsola } from "consola";
|
|
2
|
+
import { updateConfig } from "./format.js";
|
|
3
|
+
import type { RedactConfig } from "./redact.js";
|
|
2
4
|
import DatadogReporter from "./reporters/datadog.js";
|
|
3
5
|
import DiscordReporter from "./reporters/discord.js";
|
|
4
6
|
|
|
5
|
-
const setupLogging = () => {
|
|
6
|
-
|
|
7
|
+
const setupLogging = (config: RedactConfig = {}) => {
|
|
8
|
+
updateConfig(config);
|
|
9
|
+
const consola = createConsola({ fancy: true, level: 5 });
|
|
7
10
|
|
|
8
11
|
if (process.env.NODE_ENV === "production")
|
|
9
12
|
consola.setReporters([DatadogReporter]);
|
|
10
|
-
|
|
11
13
|
// Set Discord reporter as first so it can remove
|
|
12
14
|
// Discord-related config before other reporters process the
|
|
13
15
|
// log
|
|
14
|
-
consola.setReporters([DiscordReporter, ...consola.options.reporters]);
|
|
16
|
+
else consola.setReporters([DiscordReporter, ...consola.options.reporters]);
|
|
15
17
|
|
|
16
18
|
consola.wrapConsole();
|
|
17
19
|
return consola;
|
|
18
20
|
};
|
|
19
21
|
|
|
20
|
-
type Options = ConsolaOptions & { fancy?: boolean };
|
|
21
|
-
|
|
22
22
|
export { setupLogging };
|
package/src/redact.ts
CHANGED
|
@@ -2,88 +2,245 @@ import { DeepRedact } from "@hackylabs/deep-redact/index.ts";
|
|
|
2
2
|
import { validateMnemonic } from "@scure/bip39";
|
|
3
3
|
import { wordlist } from "@scure/bip39/wordlists/english.js";
|
|
4
4
|
|
|
5
|
+
/**
|
|
6
|
+
* Configurable redaction for strings that *look like secrets*:
|
|
7
|
+
* - hex (optionally 0x-prefixed)
|
|
8
|
+
* - base64 blobs
|
|
9
|
+
* - base64url blobs
|
|
10
|
+
* - base58 blobs
|
|
11
|
+
* - BIP39 mnemonics (validated)
|
|
12
|
+
*
|
|
13
|
+
* Key idea:
|
|
14
|
+
* Each detector can be given "allow" context rules that prevent redaction when
|
|
15
|
+
* extra surrounding context indicates the value is safe/expected.
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
type ContextRule = {
|
|
19
|
+
/**
|
|
20
|
+
* Test this regex against a slice of the input around the match.
|
|
21
|
+
* If it matches, the match is NOT redacted.
|
|
22
|
+
*/
|
|
23
|
+
re: RegExp;
|
|
24
|
+
/** How many chars to include before the match when building the slice. */
|
|
25
|
+
before?: number; // default 10
|
|
26
|
+
/** How many chars to include after the match when building the slice. */
|
|
27
|
+
after?: number; // default 0
|
|
28
|
+
};
|
|
29
|
+
|
|
30
|
+
type DetectorConfig = {
|
|
31
|
+
/**
|
|
32
|
+
* Any rule match => do not redact that specific match.
|
|
33
|
+
*/
|
|
34
|
+
allow?: ContextRule[];
|
|
35
|
+
};
|
|
36
|
+
|
|
37
|
+
type RedactConfig = {
|
|
38
|
+
hex?: DetectorConfig;
|
|
39
|
+
base64?: DetectorConfig;
|
|
40
|
+
base64url?: DetectorConfig;
|
|
41
|
+
base58?: DetectorConfig;
|
|
42
|
+
mnemonic?: DetectorConfig;
|
|
43
|
+
};
|
|
44
|
+
|
|
5
45
|
const replacement = "[REDACTED]";
|
|
6
46
|
|
|
47
|
+
const cloneRegex = (re: RegExp): RegExp => {
|
|
48
|
+
const flags = re.flags.includes("g") ? re.flags : `${re.flags}g`;
|
|
49
|
+
return new RegExp(re.source, flags);
|
|
50
|
+
};
|
|
51
|
+
|
|
52
|
+
const sliceAround = (
|
|
53
|
+
value: string,
|
|
54
|
+
offset: number,
|
|
55
|
+
matchLen: number,
|
|
56
|
+
rule: ContextRule,
|
|
57
|
+
) => {
|
|
58
|
+
const before = rule.before ?? 10;
|
|
59
|
+
const after = rule.after ?? 0;
|
|
60
|
+
|
|
61
|
+
const start = Math.max(0, offset - before);
|
|
62
|
+
const end = Math.min(value.length, offset + matchLen + after);
|
|
63
|
+
return value.slice(start, end);
|
|
64
|
+
};
|
|
65
|
+
|
|
66
|
+
const shouldAllowByRules = (
|
|
67
|
+
value: string,
|
|
68
|
+
match: string,
|
|
69
|
+
offset: number,
|
|
70
|
+
rules?: ContextRule[],
|
|
71
|
+
): boolean => {
|
|
72
|
+
if (!rules?.length) return false;
|
|
73
|
+
for (const rule of rules) {
|
|
74
|
+
const chunk = sliceAround(value, offset, match.length, rule);
|
|
75
|
+
|
|
76
|
+
// If a consumer provides a /g or /y regex, .test() mutates lastIndex.
|
|
77
|
+
// Reset to ensure consistent behavior.
|
|
78
|
+
rule.re.lastIndex = 0;
|
|
79
|
+
|
|
80
|
+
if (rule.re.test(chunk)) return true;
|
|
81
|
+
}
|
|
82
|
+
return false;
|
|
83
|
+
};
|
|
84
|
+
|
|
85
|
+
type ReplaceMeta = {
|
|
86
|
+
offset: number;
|
|
87
|
+
whole: string;
|
|
88
|
+
};
|
|
89
|
+
|
|
90
|
+
const getReplaceMeta = (args: unknown[]): ReplaceMeta | null => {
|
|
91
|
+
const offset = args.at(-2);
|
|
92
|
+
const whole = args.at(-1);
|
|
93
|
+
if (typeof offset !== "number") return null;
|
|
94
|
+
if (typeof whole !== "string") return null;
|
|
95
|
+
return { offset, whole };
|
|
96
|
+
};
|
|
97
|
+
|
|
98
|
+
const replaceAllMatchesWithContext = (
|
|
99
|
+
value: string,
|
|
100
|
+
pattern: RegExp,
|
|
101
|
+
replacement: string,
|
|
102
|
+
allow?: ContextRule[],
|
|
103
|
+
) => {
|
|
104
|
+
const re = cloneRegex(pattern);
|
|
105
|
+
return value.replace(re, (...args: unknown[]) => {
|
|
106
|
+
const match = args[0];
|
|
107
|
+
if (typeof match !== "string") return replacement;
|
|
108
|
+
|
|
109
|
+
const meta = getReplaceMeta(args);
|
|
110
|
+
if (!meta) return replacement;
|
|
111
|
+
|
|
112
|
+
if (shouldAllowByRules(meta.whole, match, meta.offset, allow)) return match;
|
|
113
|
+
return replacement;
|
|
114
|
+
});
|
|
115
|
+
};
|
|
116
|
+
|
|
7
117
|
/**
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
* -
|
|
11
|
-
* - long base64 blobs
|
|
12
|
-
* - long base58 blobs
|
|
13
|
-
* - mnemonic seed phrases (validated via BIP39 english wordlist)
|
|
118
|
+
* BIP39 contextual replacer:
|
|
119
|
+
* - Only redacts if the captured phrase validates as a BIP39 mnemonic
|
|
120
|
+
* - Still supports allow-rules (to suppress redaction in “safe” contexts)
|
|
14
121
|
*/
|
|
122
|
+
const replaceBip39MnemonicMatchesWithContext = (
|
|
123
|
+
value: string,
|
|
124
|
+
pattern: RegExp,
|
|
125
|
+
replacement: string,
|
|
126
|
+
allow?: ContextRule[],
|
|
127
|
+
) => {
|
|
128
|
+
const re = cloneRegex(pattern);
|
|
129
|
+
return value.replace(re, (...args: unknown[]) => {
|
|
130
|
+
const match = args[0];
|
|
131
|
+
const phrase = args[1];
|
|
132
|
+
|
|
133
|
+
if (typeof match !== "string") return replacement;
|
|
134
|
+
if (typeof phrase !== "string") return match;
|
|
135
|
+
|
|
136
|
+
const meta = getReplaceMeta(args);
|
|
137
|
+
if (!meta) return match;
|
|
15
138
|
|
|
16
|
-
|
|
17
|
-
const replaceAllMatches = (value: string, pattern: RegExp) =>
|
|
18
|
-
value.replace(pattern, replacement);
|
|
139
|
+
if (shouldAllowByRules(meta.whole, match, meta.offset, allow)) return match;
|
|
19
140
|
|
|
20
|
-
// Mnemonic replacer: only redact if the captured phrase is a valid BIP39 mnemonic.
|
|
21
|
-
const replaceBip39MnemonicMatches = (value: string, pattern: RegExp) =>
|
|
22
|
-
value.replace(pattern, (match: string, phrase: string) => {
|
|
23
|
-
// Normalize spacing; validateMnemonic expects words separated by single spaces
|
|
24
141
|
const normalized = phrase.trim().toLowerCase().replace(/\s+/g, " ");
|
|
25
142
|
if (!validateMnemonic(normalized, wordlist)) return match;
|
|
26
143
|
|
|
27
|
-
// Replace only the phrase portion (preserves surrounding quotes/keywords if any)
|
|
28
144
|
return match.replace(phrase, replacement);
|
|
29
145
|
});
|
|
146
|
+
};
|
|
147
|
+
|
|
148
|
+
const createRedactor = (config: RedactConfig = {}) => {
|
|
149
|
+
const HEX_MIN_LEN = 16;
|
|
150
|
+
const BASE64_MIN_BLOCKS = 4;
|
|
151
|
+
const BASE58_MIN_LEN = 16;
|
|
152
|
+
|
|
153
|
+
const HEX = new RegExp(
|
|
154
|
+
String.raw`(?<![a-fA-F0-9])(?:0x)?[a-fA-F0-9]{${HEX_MIN_LEN},}(?![a-fA-F0-9])`,
|
|
155
|
+
"g",
|
|
156
|
+
);
|
|
157
|
+
const hexAllow: ContextRule[] = config.hex?.allow ?? [];
|
|
158
|
+
|
|
159
|
+
const BASE64 = new RegExp(
|
|
160
|
+
String.raw`(?<![A-Za-z0-9+/=])(?:[A-Za-z0-9+/]{4}){${BASE64_MIN_BLOCKS},}(?:[A-Za-z0-9+/]{2,3})?(?:={0,2})(?![A-Za-z0-9+/=])`,
|
|
161
|
+
"g",
|
|
162
|
+
);
|
|
163
|
+
const base64Allow = config.base64?.allow ?? [];
|
|
164
|
+
|
|
165
|
+
const BASE64URL = new RegExp(
|
|
166
|
+
String.raw`(?<![A-Za-z0-9\-_])[A-Za-z0-9\-_]{16,}(?:={0,2})?(?![A-Za-z0-9\-_])`,
|
|
167
|
+
"g",
|
|
168
|
+
);
|
|
169
|
+
const base64urlAllow = config.base64url?.allow ?? [];
|
|
170
|
+
|
|
171
|
+
const BASE58 = new RegExp(
|
|
172
|
+
String.raw`(?<![1-9A-HJ-NP-Za-km-z])[1-9A-HJ-NP-Za-km-z]{${BASE58_MIN_LEN},}(?![1-9A-HJ-NP-Za-km-z])`,
|
|
173
|
+
"g",
|
|
174
|
+
);
|
|
175
|
+
const base58Allow = config.base58?.allow ?? [];
|
|
176
|
+
|
|
177
|
+
const WORD = "[a-zA-Z]{2,8}";
|
|
178
|
+
const PHRASE_12_TO_24 = `(?:${WORD}\\s+){11,23}${WORD}`;
|
|
179
|
+
const MNEMONIC = new RegExp(
|
|
180
|
+
String.raw`(?<![A-Za-z])(${PHRASE_12_TO_24})(?![A-Za-z])`,
|
|
181
|
+
"gi",
|
|
182
|
+
);
|
|
183
|
+
const mnemonicAllow = config.mnemonic?.allow ?? [];
|
|
184
|
+
|
|
185
|
+
const stringTests: Array<{
|
|
186
|
+
pattern: RegExp;
|
|
187
|
+
replacer: (v: string, p: RegExp) => string;
|
|
188
|
+
}> = [
|
|
189
|
+
{
|
|
190
|
+
pattern: HEX,
|
|
191
|
+
replacer: (v, p) =>
|
|
192
|
+
replaceAllMatchesWithContext(v, p, replacement, hexAllow),
|
|
193
|
+
},
|
|
194
|
+
{
|
|
195
|
+
pattern: BASE64URL,
|
|
196
|
+
replacer: (v, p) =>
|
|
197
|
+
replaceAllMatchesWithContext(v, p, replacement, base64urlAllow),
|
|
198
|
+
},
|
|
199
|
+
{
|
|
200
|
+
pattern: BASE64,
|
|
201
|
+
replacer: (v, p) =>
|
|
202
|
+
replaceAllMatchesWithContext(v, p, replacement, base64Allow),
|
|
203
|
+
},
|
|
204
|
+
{
|
|
205
|
+
pattern: BASE58,
|
|
206
|
+
replacer: (v, p) =>
|
|
207
|
+
replaceAllMatchesWithContext(v, p, replacement, base58Allow),
|
|
208
|
+
},
|
|
209
|
+
{
|
|
210
|
+
pattern: MNEMONIC,
|
|
211
|
+
replacer: (v, p) =>
|
|
212
|
+
replaceBip39MnemonicMatchesWithContext(
|
|
213
|
+
v,
|
|
214
|
+
p,
|
|
215
|
+
replacement,
|
|
216
|
+
mnemonicAllow,
|
|
217
|
+
),
|
|
218
|
+
},
|
|
219
|
+
];
|
|
220
|
+
|
|
221
|
+
return new DeepRedact({
|
|
222
|
+
stringTests,
|
|
223
|
+
replacement,
|
|
224
|
+
serialise: false,
|
|
225
|
+
});
|
|
226
|
+
};
|
|
227
|
+
|
|
228
|
+
const createRedact = (config: RedactConfig) => {
|
|
229
|
+
const redactor = createRedactor(config);
|
|
230
|
+
|
|
231
|
+
// DeepRedact applies only the first matching stringTest per pass.
|
|
232
|
+
// Run multiple passes until stable so different detectors can redact
|
|
233
|
+
// different tokens in the same string.
|
|
234
|
+
return (text: string) => {
|
|
235
|
+
let out = text;
|
|
236
|
+
|
|
237
|
+
while (true) {
|
|
238
|
+
const next = redactor.redact(out) as string;
|
|
239
|
+
if (next === out) return out;
|
|
240
|
+
out = next;
|
|
241
|
+
}
|
|
242
|
+
};
|
|
243
|
+
};
|
|
30
244
|
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
// - 32+ hex chars (optionally 0x)
|
|
34
|
-
// - word boundaries help avoid eating normal words
|
|
35
|
-
const HEX = /\b(?:0x)?[a-fA-F0-9]{32,}\b/g;
|
|
36
|
-
|
|
37
|
-
// 2) Base64 blobs (JWT parts, keys, encoded payloads)
|
|
38
|
-
// - requires a decent minimum length to reduce false positives
|
|
39
|
-
// - supports optional padding
|
|
40
|
-
const BASE64 =
|
|
41
|
-
/\b(?:[A-Za-z0-9+/]{4}){8,}(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?\b/g;
|
|
42
|
-
|
|
43
|
-
// 3) Base58 blobs (common in crypto keys/ids)
|
|
44
|
-
// - base58 alphabet excludes 0, O, I, l
|
|
45
|
-
// - require 32+ chars to reduce false positives
|
|
46
|
-
const BASE58 = /\b[1-9A-HJ-NP-Za-km-z]{32,}\b/g;
|
|
47
|
-
|
|
48
|
-
// 4) Mnemonic seed phrases
|
|
49
|
-
// We validate candidates with @scure/bip39 to avoid false positives.
|
|
50
|
-
// - 12 to 24 words separated by whitespace
|
|
51
|
-
const WORD = "[a-zA-Z]{2,8}";
|
|
52
|
-
const PHRASE_12_TO_24 = `(?:${WORD}\\s+){11,23}${WORD}`;
|
|
53
|
-
|
|
54
|
-
const MNEMONIC_WITH_KEYWORD = new RegExp(
|
|
55
|
-
// keyword then up to ~40 chars (like ":" or whitespace) then the phrase
|
|
56
|
-
String.raw`\b(?:mnemonic|seed|recovery\s+phrase|secret\s+phrase)\b[\s:=-]{0,40}(${PHRASE_12_TO_24})\b`,
|
|
57
|
-
"gi",
|
|
58
|
-
);
|
|
59
|
-
|
|
60
|
-
const MNEMONIC_QUOTED = new RegExp(
|
|
61
|
-
// quoted/bracketed phrase alone
|
|
62
|
-
String.raw`(?:["'(\[])\s*(${PHRASE_12_TO_24})\s*(?:["')\]])`,
|
|
63
|
-
"gi",
|
|
64
|
-
);
|
|
65
|
-
|
|
66
|
-
// NEW: Bare mnemonic phrases (no keyword, no quotes).
|
|
67
|
-
// This is what your console.log example is: just the phrase by itself.
|
|
68
|
-
// Validation keeps false-positives low.
|
|
69
|
-
const MNEMONIC_BARE = new RegExp(String.raw`\b(${PHRASE_12_TO_24})\b`, "gi");
|
|
70
|
-
|
|
71
|
-
const redactor = new DeepRedact({
|
|
72
|
-
// stringTests runs regex checks against string values (including flat strings)
|
|
73
|
-
// and lets us partially redact via replacer.
|
|
74
|
-
stringTests: [
|
|
75
|
-
{ pattern: HEX, replacer: replaceAllMatches },
|
|
76
|
-
{ pattern: BASE64, replacer: replaceAllMatches },
|
|
77
|
-
{ pattern: BASE58, replacer: replaceAllMatches },
|
|
78
|
-
{ pattern: MNEMONIC_WITH_KEYWORD, replacer: replaceBip39MnemonicMatches },
|
|
79
|
-
{ pattern: MNEMONIC_QUOTED, replacer: replaceBip39MnemonicMatches },
|
|
80
|
-
{ pattern: MNEMONIC_BARE, replacer: replaceBip39MnemonicMatches },
|
|
81
|
-
],
|
|
82
|
-
replacement,
|
|
83
|
-
// serialise mainly matters for objects; keeping it false avoids surprises.
|
|
84
|
-
serialise: false,
|
|
85
|
-
});
|
|
86
|
-
|
|
87
|
-
const redact = (text: string) => redactor.redact(text) as string;
|
|
88
|
-
|
|
89
|
-
export { redact };
|
|
245
|
+
export { createRedact };
|
|
246
|
+
export type { RedactConfig };
|
package/tests/app.ts
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
import { setupLogging } from "../src/index.js";
|
|
2
2
|
|
|
3
3
|
const run = async () => {
|
|
4
|
-
setupLogging(
|
|
4
|
+
setupLogging({
|
|
5
|
+
hex: { allow: [{ re: /\b(intent|hash)\b/i }] },
|
|
6
|
+
});
|
|
5
7
|
|
|
6
8
|
console.log("Hello, this is a log");
|
|
7
9
|
console.info("Hello, this is an info log");
|
|
@@ -14,6 +16,12 @@ const run = async () => {
|
|
|
14
16
|
console.log(
|
|
15
17
|
"This is my private key: 4fa7614bdd07ab15b11d2365466536ba160c06050ade0888a3aa985a5522ab22",
|
|
16
18
|
);
|
|
19
|
+
console.log(
|
|
20
|
+
"This is log for intent c8adcfef7255ad2c117f68111500997ab66de3c923e9ea9e71aaed5829d0acb8 and private key c99960eaeecb17d77d7c88f440383ccbaf93237471236a739185c0ebd62c5741",
|
|
21
|
+
);
|
|
22
|
+
console.log(
|
|
23
|
+
"this is my Discord webhook (https://discord.com/api/webhooks/1473664170267246593/V9xRwhrxrgEIJZL8inMMiEC8wDTwrMQye-VxLyBmkH6vbeTGNNBCjqkbtnUIA_dIVh3d) URL",
|
|
24
|
+
);
|
|
17
25
|
|
|
18
26
|
try {
|
|
19
27
|
throw new Error("An Error was thrown");
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import { describe, expect, it } from "vitest";
|
|
2
|
+
import { createRedact } from "../src/redact.js";
|
|
3
|
+
|
|
4
|
+
describe("redact", () => {
|
|
5
|
+
it("redacts base64url", () => {
|
|
6
|
+
const redact = createRedact({});
|
|
7
|
+
const result = redact(
|
|
8
|
+
"Should base64url-ish V9xRwhrxrgEIJZL8inMMiEC8wDTwrMQye-VxLyBmkH6vbeTGNNBCjqkbtnUIA_dIVh3d data",
|
|
9
|
+
);
|
|
10
|
+
expect(result).toBe("Should base64url-ish [REDACTED] data");
|
|
11
|
+
});
|
|
12
|
+
|
|
13
|
+
it("redacts hex and base64url in same string", () => {
|
|
14
|
+
const redact = createRedact({});
|
|
15
|
+
const result = redact(
|
|
16
|
+
"Should redact private key (b49bd63e67e2cd11aba17befead483934939df828cb833a846c58661726d3b00) and API key (djOqmjzVb0GAGdqS0p0NtiEwvb6u1lx509JEkpDJLgnvMhmOMtBc9vqolpktd1OK7Xas)",
|
|
17
|
+
);
|
|
18
|
+
expect(result).toBe(
|
|
19
|
+
"Should redact private key ([REDACTED]) and API key ([REDACTED])",
|
|
20
|
+
);
|
|
21
|
+
});
|
|
22
|
+
});
|