@upx-us/shield 0.2.16-beta → 0.3.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +178 -53
- package/dist/index.d.ts +13 -15
- package/dist/index.js +593 -245
- package/dist/src/config.d.ts +1 -15
- package/dist/src/config.js +3 -39
- package/dist/src/counters.d.ts +14 -0
- package/dist/src/counters.js +96 -0
- package/dist/src/events/base.d.ts +0 -25
- package/dist/src/events/base.js +0 -15
- package/dist/src/events/browser/enrich.js +1 -1
- package/dist/src/events/exec/enrich.js +0 -2
- package/dist/src/events/exec/redactions.d.ts +0 -1
- package/dist/src/events/exec/redactions.js +0 -1
- package/dist/src/events/file/enrich.js +0 -3
- package/dist/src/events/generic/index.d.ts +0 -1
- package/dist/src/events/generic/index.js +0 -1
- package/dist/src/events/index.d.ts +0 -13
- package/dist/src/events/index.js +1 -13
- package/dist/src/events/message/validations.js +0 -3
- package/dist/src/events/sessions-spawn/enrich.js +0 -1
- package/dist/src/events/sessions-spawn/event.d.ts +0 -1
- package/dist/src/events/tool-result/enrich.js +0 -1
- package/dist/src/events/tool-result/redactions.js +0 -1
- package/dist/src/events/web/enrich.d.ts +0 -4
- package/dist/src/events/web/enrich.js +6 -14
- package/dist/src/events/web/redactions.js +1 -3
- package/dist/src/fetcher.d.ts +1 -0
- package/dist/src/fetcher.js +28 -19
- package/dist/src/index.js +51 -16
- package/dist/src/log.d.ts +0 -26
- package/dist/src/log.js +1 -27
- package/dist/src/redactor/base.d.ts +0 -23
- package/dist/src/redactor/base.js +0 -7
- package/dist/src/redactor/index.d.ts +0 -15
- package/dist/src/redactor/index.js +8 -27
- package/dist/src/redactor/strategies/command.js +0 -3
- package/dist/src/redactor/strategies/hostname.js +0 -1
- package/dist/src/redactor/strategies/index.d.ts +0 -5
- package/dist/src/redactor/strategies/index.js +0 -5
- package/dist/src/redactor/strategies/path.js +3 -3
- package/dist/src/redactor/strategies/secret-key.js +33 -9
- package/dist/src/redactor/vault.d.ts +0 -19
- package/dist/src/redactor/vault.js +7 -35
- package/dist/src/sender.d.ts +12 -20
- package/dist/src/sender.js +40 -36
- package/dist/src/setup.d.ts +11 -9
- package/dist/src/setup.js +33 -32
- package/dist/src/transformer.d.ts +1 -12
- package/dist/src/transformer.js +73 -48
- package/dist/src/validator.d.ts +0 -11
- package/dist/src/validator.js +19 -25
- package/dist/src/version.js +1 -2
- package/openclaw.plugin.json +10 -2
- package/package.json +8 -3
- package/dist/src/host-collector.d.ts +0 -1
- package/dist/src/host-collector.js +0 -200
- package/skills/shield/SKILL.md +0 -38
package/dist/src/fetcher.js
CHANGED
|
@@ -33,6 +33,7 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
33
33
|
};
|
|
34
34
|
})();
|
|
35
35
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
+
exports.MAX_ENTRIES_PER_POLL = void 0;
|
|
36
37
|
exports.fetchNewEntries = fetchNewEntries;
|
|
37
38
|
exports.commitCursors = commitCursors;
|
|
38
39
|
const fs_1 = require("fs");
|
|
@@ -40,15 +41,16 @@ const path_1 = require("path");
|
|
|
40
41
|
const log = __importStar(require("./log"));
|
|
41
42
|
function loadCursors(cursorFile) {
|
|
42
43
|
if ((0, fs_1.existsSync)(cursorFile)) {
|
|
43
|
-
|
|
44
|
+
try {
|
|
45
|
+
return JSON.parse((0, fs_1.readFileSync)(cursorFile, 'utf-8'));
|
|
46
|
+
}
|
|
47
|
+
catch {
|
|
48
|
+
log.warn('fetcher', 'Cursor file is corrupt or unreadable — resetting to empty state. Some events may be re-processed.');
|
|
49
|
+
return {};
|
|
50
|
+
}
|
|
44
51
|
}
|
|
45
52
|
return {};
|
|
46
53
|
}
|
|
47
|
-
/**
|
|
48
|
-
* Initialize cursors for a new session directory.
|
|
49
|
-
* Sets all file offsets to current file size (skip historical data).
|
|
50
|
-
* Only called for directories that have NO existing cursor entries.
|
|
51
|
-
*/
|
|
52
54
|
function initCursorsForDir(sessionDir, agentId, existingCursors) {
|
|
53
55
|
const init = {};
|
|
54
56
|
let files;
|
|
@@ -62,15 +64,12 @@ function initCursorsForDir(sessionDir, agentId, existingCursors) {
|
|
|
62
64
|
let skipped = 0;
|
|
63
65
|
for (const file of files) {
|
|
64
66
|
const cursorKey = `${agentId}/${file}`;
|
|
65
|
-
// Also check legacy key (without agent prefix) for migration
|
|
66
67
|
if (existingCursors[cursorKey] !== undefined || existingCursors[file] !== undefined) {
|
|
67
|
-
// Already tracked — migrate legacy key if needed
|
|
68
68
|
if (existingCursors[file] !== undefined && existingCursors[cursorKey] === undefined) {
|
|
69
69
|
init[cursorKey] = existingCursors[file];
|
|
70
70
|
}
|
|
71
71
|
continue;
|
|
72
72
|
}
|
|
73
|
-
// New file — set cursor to current size (skip history)
|
|
74
73
|
const filePath = (0, path_1.join)(sessionDir, file);
|
|
75
74
|
const fd = (0, fs_1.openSync)(filePath, 'r');
|
|
76
75
|
try {
|
|
@@ -95,10 +94,6 @@ function saveCursors(cursorFile, state) {
|
|
|
95
94
|
(0, fs_1.writeFileSync)(tmp, JSON.stringify(state, null, 2));
|
|
96
95
|
(0, fs_1.renameSync)(tmp, cursorFile);
|
|
97
96
|
}
|
|
98
|
-
/**
|
|
99
|
-
* Read only the new bytes from a file starting at the given byte offset.
|
|
100
|
-
* Uses low-level fd + read to avoid loading the entire file into memory.
|
|
101
|
-
*/
|
|
102
97
|
function readNewBytes(filePath, offset) {
|
|
103
98
|
const fd = (0, fs_1.openSync)(filePath, 'r');
|
|
104
99
|
try {
|
|
@@ -116,18 +111,19 @@ function readNewBytes(filePath, offset) {
|
|
|
116
111
|
(0, fs_1.closeSync)(fd);
|
|
117
112
|
}
|
|
118
113
|
}
|
|
119
|
-
// In-memory cursor state for the current poll cycle (committed only after successful send)
|
|
120
114
|
let _pendingCursors = {};
|
|
115
|
+
exports.MAX_ENTRIES_PER_POLL = 5000;
|
|
121
116
|
async function fetchNewEntries(config) {
|
|
122
117
|
const cursors = loadCursors(config.cursorFile);
|
|
123
118
|
_pendingCursors = { ...cursors };
|
|
124
119
|
const results = [];
|
|
120
|
+
let capReached = false;
|
|
125
121
|
for (const sessionDir of config.sessionDirs) {
|
|
122
|
+
if (capReached)
|
|
123
|
+
break;
|
|
126
124
|
if (!(0, fs_1.existsSync)(sessionDir))
|
|
127
125
|
continue;
|
|
128
|
-
// Derive agent name from directory path: .../agents/{agentId}/sessions
|
|
129
126
|
const agentId = (0, path_1.basename)((0, path_1.dirname)(sessionDir));
|
|
130
|
-
// Initialize cursors for newly discovered agents (skip historical data)
|
|
131
127
|
const initCursors = initCursorsForDir(sessionDir, agentId, _pendingCursors);
|
|
132
128
|
Object.assign(_pendingCursors, initCursors);
|
|
133
129
|
let files;
|
|
@@ -138,9 +134,10 @@ async function fetchNewEntries(config) {
|
|
|
138
134
|
continue;
|
|
139
135
|
}
|
|
140
136
|
for (const file of files) {
|
|
137
|
+
if (capReached)
|
|
138
|
+
break;
|
|
141
139
|
const filePath = (0, path_1.join)(sessionDir, file);
|
|
142
140
|
const sessionId = (0, path_1.basename)(file, '.jsonl');
|
|
143
|
-
// Cursor key includes agent to avoid collisions across agents
|
|
144
141
|
const cursorKey = `${agentId}/${file}`;
|
|
145
142
|
const offset = _pendingCursors[cursorKey] ?? cursors[cursorKey] ?? 0;
|
|
146
143
|
const { content, newSize } = readNewBytes(filePath, offset);
|
|
@@ -149,7 +146,9 @@ async function fetchNewEntries(config) {
|
|
|
149
146
|
continue;
|
|
150
147
|
}
|
|
151
148
|
const lines = content.split('\n').filter(l => l.trim());
|
|
149
|
+
let bytesConsumed = 0;
|
|
152
150
|
for (const line of lines) {
|
|
151
|
+
bytesConsumed += Buffer.byteLength(line, 'utf-8') + 1;
|
|
153
152
|
try {
|
|
154
153
|
const entry = JSON.parse(line);
|
|
155
154
|
if (entry.type !== 'message')
|
|
@@ -167,9 +166,19 @@ async function fetchNewEntries(config) {
|
|
|
167
166
|
results.push({ ...entry, _sessionId: sessionId, _agentId: agentId });
|
|
168
167
|
}
|
|
169
168
|
}
|
|
170
|
-
catch {
|
|
169
|
+
catch { }
|
|
170
|
+
if (results.length >= exports.MAX_ENTRIES_PER_POLL) {
|
|
171
|
+
capReached = true;
|
|
172
|
+
log.warn('fetcher', `Entry cap reached (${exports.MAX_ENTRIES_PER_POLL}) — remaining entries deferred to next poll`);
|
|
173
|
+
break;
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
if (capReached) {
|
|
177
|
+
_pendingCursors[cursorKey] = offset + bytesConsumed;
|
|
178
|
+
}
|
|
179
|
+
else {
|
|
180
|
+
_pendingCursors[cursorKey] = newSize;
|
|
171
181
|
}
|
|
172
|
-
_pendingCursors[cursorKey] = newSize;
|
|
173
182
|
}
|
|
174
183
|
}
|
|
175
184
|
return results;
|
package/dist/src/index.js
CHANGED
|
@@ -45,8 +45,10 @@ const version_1 = require("./version");
|
|
|
45
45
|
let running = true;
|
|
46
46
|
let lastTelemetryAt = 0;
|
|
47
47
|
let consecutiveFailures = 0;
|
|
48
|
-
|
|
49
|
-
const
|
|
48
|
+
let registrationOk = false;
|
|
49
|
+
const TELEMETRY_INTERVAL_MS = 5 * 60 * 1000;
|
|
50
|
+
const MAX_BACKOFF_MS = 5 * 60 * 1000;
|
|
51
|
+
const MAX_REGISTRATION_FAILURES = 10;
|
|
50
52
|
function getBackoffInterval(baseMs) {
|
|
51
53
|
if (consecutiveFailures === 0)
|
|
52
54
|
return baseMs;
|
|
@@ -64,11 +66,8 @@ async function poll() {
|
|
|
64
66
|
let entries = await (0, fetcher_1.fetchNewEntries)(config);
|
|
65
67
|
const now = Date.now();
|
|
66
68
|
if (now - lastTelemetryAt >= TELEMETRY_INTERVAL_MS) {
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
// platform backend (Cloud Run → uss.upx.com). This avoids noisy
|
|
70
|
-
// GENERIC_EVENT floods that trigger false-positive YARA-L rules.
|
|
71
|
-
(0, transformer_1.generateHostTelemetry)(); // still collect locally (for score / future use)
|
|
69
|
+
const hostSnapshot = config.collectHostMetrics ? (0, transformer_1.generateHostTelemetry)() : null;
|
|
70
|
+
const hostMeta = hostSnapshot?.event?.tool_metadata;
|
|
72
71
|
const instancePayload = {
|
|
73
72
|
machine: {
|
|
74
73
|
hostname: config.hostname,
|
|
@@ -80,12 +79,36 @@ async function poll() {
|
|
|
80
79
|
plugin_version: version_1.VERSION,
|
|
81
80
|
openclaw_version: (0, transformer_1.resolveOpenClawVersion)(),
|
|
82
81
|
agent_label: (0, transformer_1.resolveAgentLabel)('main'),
|
|
82
|
+
...(hostMeta && {
|
|
83
|
+
gateway_bind: hostMeta['openclaw.gateway_bind'],
|
|
84
|
+
webhook_configured: hostMeta['openclaw.webhook_configured'],
|
|
85
|
+
browser_auth_required: hostMeta['openclaw.browser_auth_required'],
|
|
86
|
+
}),
|
|
83
87
|
},
|
|
84
88
|
};
|
|
85
|
-
const
|
|
86
|
-
log.info('bridge', `Instance report → Platform: success=${
|
|
87
|
-
if (
|
|
89
|
+
const result = await (0, sender_1.reportInstance)(instancePayload, config.credentials);
|
|
90
|
+
log.info('bridge', `Instance report → Platform: success=${result.ok}`);
|
|
91
|
+
if (result.ok) {
|
|
92
|
+
registrationOk = true;
|
|
88
93
|
lastTelemetryAt = now;
|
|
94
|
+
if (result.score) {
|
|
95
|
+
log.info('bridge', `Protection score: ${result.score.badge} ${result.score.total}/100 (${result.score.grade})`);
|
|
96
|
+
if (result.score.recommendations?.length) {
|
|
97
|
+
for (const rec of result.score.recommendations) {
|
|
98
|
+
log.warn('bridge', `⚠ ${rec}`);
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
else if (!registrationOk) {
|
|
104
|
+
consecutiveFailures++;
|
|
105
|
+
if (consecutiveFailures >= MAX_REGISTRATION_FAILURES) {
|
|
106
|
+
log.error('bridge', `reportInstance failed ${consecutiveFailures} consecutive times — instance not recognized. Re-run setup wizard. Exiting.`);
|
|
107
|
+
process.exit(1);
|
|
108
|
+
}
|
|
109
|
+
log.warn('bridge', `reportInstance failed (attempt ${consecutiveFailures}/${MAX_REGISTRATION_FAILURES}) — skipping events this cycle (platform may still be syncing)`);
|
|
110
|
+
continue;
|
|
111
|
+
}
|
|
89
112
|
}
|
|
90
113
|
if (entries.length > 0) {
|
|
91
114
|
let envelopes = (0, transformer_1.transformEntries)(entries);
|
|
@@ -115,6 +138,24 @@ async function poll() {
|
|
|
115
138
|
log.error('bridge', `Result: FAILED status=${r.statusCode} events=${r.eventCount} body=${r.body?.slice(0, 200)}`);
|
|
116
139
|
}
|
|
117
140
|
}
|
|
141
|
+
if (results.some(r => r.needsRegistration)) {
|
|
142
|
+
consecutiveFailures++;
|
|
143
|
+
registrationOk = false;
|
|
144
|
+
lastTelemetryAt = 0;
|
|
145
|
+
if (consecutiveFailures >= MAX_REGISTRATION_FAILURES) {
|
|
146
|
+
log.error('bridge', `Instance not recognized by platform after ${consecutiveFailures} attempts. Re-run the setup wizard. Exiting.`);
|
|
147
|
+
process.exit(1);
|
|
148
|
+
}
|
|
149
|
+
log.warn('bridge', `Instance not registered on platform (attempt ${consecutiveFailures}/${MAX_REGISTRATION_FAILURES}) — will re-register on next cycle`);
|
|
150
|
+
continue;
|
|
151
|
+
}
|
|
152
|
+
const pendingResult = results.find(r => r.pendingNamespace);
|
|
153
|
+
if (pendingResult) {
|
|
154
|
+
const retryAfterMs = Math.min(pendingResult.retryAfterMs ?? 300_000, MAX_BACKOFF_MS);
|
|
155
|
+
log.warn('bridge', `Namespace pending — holding cursors, retrying in ${Math.round(retryAfterMs / 1000)}s`);
|
|
156
|
+
await new Promise(r => setTimeout(r, retryAfterMs));
|
|
157
|
+
continue;
|
|
158
|
+
}
|
|
118
159
|
const allSuccess = results.every(r => r.success);
|
|
119
160
|
if (allSuccess) {
|
|
120
161
|
(0, fetcher_1.commitCursors)(config, entries);
|
|
@@ -134,11 +175,6 @@ async function poll() {
|
|
|
134
175
|
}
|
|
135
176
|
}
|
|
136
177
|
else {
|
|
137
|
-
// No new entries this poll — but still commit cursors so that
|
|
138
|
-
// initCursorsForDir positions (set to current file sizes) are
|
|
139
|
-
// persisted. Without this, the next poll re-initialises cursors
|
|
140
|
-
// to the NEW current size and silently skips any events that
|
|
141
|
-
// arrived between polls.
|
|
142
178
|
(0, fetcher_1.commitCursors)(config, []);
|
|
143
179
|
consecutiveFailures = 0;
|
|
144
180
|
}
|
|
@@ -162,7 +198,6 @@ async function poll() {
|
|
|
162
198
|
function checkConfiguration() {
|
|
163
199
|
if (!(0, fs_1.existsSync)(config_1.SHIELD_CONFIG_PATH)) {
|
|
164
200
|
const { SHIELD_API_URL, SHIELD_INSTANCE_ID, SHIELD_HMAC_SECRET, SHIELD_FINGERPRINT, SHIELD_SECRET } = process.env;
|
|
165
|
-
// Support both new and legacy env var names
|
|
166
201
|
if (SHIELD_API_URL && (SHIELD_INSTANCE_ID || SHIELD_FINGERPRINT) && (SHIELD_HMAC_SECRET || SHIELD_SECRET))
|
|
167
202
|
return true;
|
|
168
203
|
console.log('🛡️ OpenClaw Shield — First Time Setup Required');
|
package/dist/src/log.d.ts
CHANGED
|
@@ -1,37 +1,11 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* log.ts — Structured logger for the Shield plugin
|
|
3
|
-
*
|
|
4
|
-
* Supports two backends via the LogAdapter interface:
|
|
5
|
-
* 1. ConsoleLogAdapter (default) — used in standalone/docker bridge mode
|
|
6
|
-
* 2. Gateway adapter — injected by the plugin entry via setAdapter()
|
|
7
|
-
*
|
|
8
|
-
* All internal modules use `import * as log from './log'` unchanged.
|
|
9
|
-
* Standalone mode never calls setAdapter(), so console logging works
|
|
10
|
-
* identically to a direct console.log/warn/error setup.
|
|
11
|
-
*
|
|
12
|
-
* Log level is controlled by LOG_LEVEL env var: debug | info | warn | error
|
|
13
|
-
* (default: info). In debug mode, pipeline stages emit detailed per-event
|
|
14
|
-
* output (transformer, validator, redactor, sender).
|
|
15
|
-
*/
|
|
16
1
|
export interface LogAdapter {
|
|
17
2
|
debug(tag: string, msg: string, data?: unknown): void;
|
|
18
3
|
info(tag: string, msg: string): void;
|
|
19
4
|
warn(tag: string, msg: string): void;
|
|
20
5
|
error(tag: string, msg: string, err?: unknown): void;
|
|
21
6
|
}
|
|
22
|
-
/**
|
|
23
|
-
* Swap the log backend. Plugin mode calls this with a Gateway-backed adapter.
|
|
24
|
-
* Standalone/docker mode never calls this — console logging remains the default.
|
|
25
|
-
* Also invalidates the cached log level so it is re-resolved on next call.
|
|
26
|
-
*/
|
|
27
7
|
export declare function setAdapter(adapter: LogAdapter): void;
|
|
28
|
-
/** Reset to the default console adapter (useful for tests). */
|
|
29
8
|
export declare function resetAdapter(): void;
|
|
30
|
-
/**
|
|
31
|
-
* isDebug — dynamic property check.
|
|
32
|
-
* Accessed as `log.isDebug` (property, not function call) throughout the codebase.
|
|
33
|
-
* Using Object.defineProperty to make it a live getter on the module exports.
|
|
34
|
-
*/
|
|
35
9
|
export declare let isDebug: boolean;
|
|
36
10
|
export declare function debug(tag: string, msg: string, data?: unknown): void;
|
|
37
11
|
export declare function info(tag: string, msg: string): void;
|
package/dist/src/log.js
CHANGED
|
@@ -1,19 +1,4 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
-
/**
|
|
3
|
-
* log.ts — Structured logger for the Shield plugin
|
|
4
|
-
*
|
|
5
|
-
* Supports two backends via the LogAdapter interface:
|
|
6
|
-
* 1. ConsoleLogAdapter (default) — used in standalone/docker bridge mode
|
|
7
|
-
* 2. Gateway adapter — injected by the plugin entry via setAdapter()
|
|
8
|
-
*
|
|
9
|
-
* All internal modules use `import * as log from './log'` unchanged.
|
|
10
|
-
* Standalone mode never calls setAdapter(), so console logging works
|
|
11
|
-
* identically to a direct console.log/warn/error setup.
|
|
12
|
-
*
|
|
13
|
-
* Log level is controlled by LOG_LEVEL env var: debug | info | warn | error
|
|
14
|
-
* (default: info). In debug mode, pipeline stages emit detailed per-event
|
|
15
|
-
* output (transformer, validator, redactor, sender).
|
|
16
|
-
*/
|
|
17
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
18
3
|
exports.isDebug = void 0;
|
|
19
4
|
exports.setAdapter = setAdapter;
|
|
@@ -36,7 +21,7 @@ function getLevel() {
|
|
|
36
21
|
return cachedLevel;
|
|
37
22
|
}
|
|
38
23
|
function fmt(level, tag, msg) {
|
|
39
|
-
const ts = new Date().toISOString().slice(11, 23);
|
|
24
|
+
const ts = new Date().toISOString().slice(11, 23);
|
|
40
25
|
return `[${ts}] [${level.toUpperCase().padEnd(5)}] [${tag}] ${msg}`;
|
|
41
26
|
}
|
|
42
27
|
const consoleAdapter = {
|
|
@@ -64,25 +49,14 @@ const consoleAdapter = {
|
|
|
64
49
|
},
|
|
65
50
|
};
|
|
66
51
|
let activeAdapter = consoleAdapter;
|
|
67
|
-
/**
|
|
68
|
-
* Swap the log backend. Plugin mode calls this with a Gateway-backed adapter.
|
|
69
|
-
* Standalone/docker mode never calls this — console logging remains the default.
|
|
70
|
-
* Also invalidates the cached log level so it is re-resolved on next call.
|
|
71
|
-
*/
|
|
72
52
|
function setAdapter(adapter) {
|
|
73
53
|
activeAdapter = adapter;
|
|
74
54
|
cachedLevel = null;
|
|
75
55
|
}
|
|
76
|
-
/** Reset to the default console adapter (useful for tests). */
|
|
77
56
|
function resetAdapter() {
|
|
78
57
|
activeAdapter = consoleAdapter;
|
|
79
58
|
cachedLevel = null;
|
|
80
59
|
}
|
|
81
|
-
/**
|
|
82
|
-
* isDebug — dynamic property check.
|
|
83
|
-
* Accessed as `log.isDebug` (property, not function call) throughout the codebase.
|
|
84
|
-
* Using Object.defineProperty to make it a live getter on the module exports.
|
|
85
|
-
*/
|
|
86
60
|
exports.isDebug = LEVEL_RANK[getLevel()] <= LEVEL_RANK.debug;
|
|
87
61
|
Object.defineProperty(exports, 'isDebug', {
|
|
88
62
|
get: () => LEVEL_RANK[getLevel()] <= LEVEL_RANK.debug,
|
|
@@ -1,29 +1,6 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* src/redactor/base.ts — Core types for the strategy-based redaction system.
|
|
3
|
-
*
|
|
4
|
-
* Every redaction strategy implements RedactionStrategy. Strategies are pure
|
|
5
|
-
* data transforms: they receive a value and an injected hmac function, and
|
|
6
|
-
* return a redacted string. No I/O, no key management, no side effects.
|
|
7
|
-
*/
|
|
8
|
-
/**
|
|
9
|
-
* Produces a deterministic token (e.g. 'user:a3f9b2c1d4e5') and records the
|
|
10
|
-
* original-to-token mapping in the vault. Injected into strategies so they
|
|
11
|
-
* never own key management or storage.
|
|
12
|
-
*/
|
|
13
1
|
export type HmacFn = (category: string, value: string) => string;
|
|
14
|
-
/**
|
|
15
|
-
* Contract every redaction strategy must satisfy.
|
|
16
|
-
* The `key` string is what event `FieldRedaction.strategy` references.
|
|
17
|
-
*/
|
|
18
2
|
export interface RedactionStrategy {
|
|
19
|
-
/** Unique identifier — must match the `strategy` field in FieldRedaction rules */
|
|
20
3
|
key: string;
|
|
21
|
-
/** Human-readable description for documentation and debugging */
|
|
22
4
|
description: string;
|
|
23
|
-
/**
|
|
24
|
-
* Apply redaction to a single string value.
|
|
25
|
-
* Must be pure: same inputs always produce same outputs.
|
|
26
|
-
* Must not throw on empty strings or unusual inputs.
|
|
27
|
-
*/
|
|
28
5
|
redact: (value: string, hmac: HmacFn) => string;
|
|
29
6
|
}
|
|
@@ -1,9 +1,2 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
-
/**
|
|
3
|
-
* src/redactor/base.ts — Core types for the strategy-based redaction system.
|
|
4
|
-
*
|
|
5
|
-
* Every redaction strategy implements RedactionStrategy. Strategies are pure
|
|
6
|
-
* data transforms: they receive a value and an injected hmac function, and
|
|
7
|
-
* return a redacted string. No I/O, no key management, no side effects.
|
|
8
|
-
*/
|
|
9
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
@@ -1,20 +1,5 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* src/redactor/index.ts — Strategy engine, event redaction, and public API.
|
|
3
|
-
*
|
|
4
|
-
* This module wires vault + strategies into a single redaction engine.
|
|
5
|
-
* All imports of './redactor' resolve here (TypeScript: directory + index.ts).
|
|
6
|
-
*/
|
|
7
1
|
import type { FieldRedaction } from '../events/base';
|
|
8
|
-
/**
|
|
9
|
-
* Apply a single FieldRedaction rule to an object via dot-notation path.
|
|
10
|
-
* Missing fields and null values are silently skipped.
|
|
11
|
-
* Unknown strategy keys throw a descriptive error.
|
|
12
|
-
*/
|
|
13
2
|
export declare function applyFieldRedaction(obj: any, rule: FieldRedaction): void;
|
|
14
|
-
/**
|
|
15
|
-
* Deep-redact a Shield envelope. Returns a new object (does not mutate input).
|
|
16
|
-
* Applies base redactions + schema-specific redactions based on tool_category.
|
|
17
|
-
*/
|
|
18
3
|
export declare function redactEvent(envelope: any): any;
|
|
19
4
|
export declare function redactUsername(value: string): string;
|
|
20
5
|
export declare function redactHostname(value: string): string;
|
|
@@ -1,10 +1,4 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
-
/**
|
|
3
|
-
* src/redactor/index.ts — Strategy engine, event redaction, and public API.
|
|
4
|
-
*
|
|
5
|
-
* This module wires vault + strategies into a single redaction engine.
|
|
6
|
-
* All imports of './redactor' resolve here (TypeScript: directory + index.ts).
|
|
7
|
-
*/
|
|
8
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
9
3
|
exports.applyFieldRedaction = applyFieldRedaction;
|
|
10
4
|
exports.redactEvent = redactEvent;
|
|
@@ -21,14 +15,8 @@ const strategies_1 = require("./strategies");
|
|
|
21
15
|
const vault_1 = require("./vault");
|
|
22
16
|
const events_1 = require("../events");
|
|
23
17
|
const base_1 = require("../events/base");
|
|
24
|
-
|
|
18
|
+
const counters_1 = require("../counters");
|
|
25
19
|
const strategyMap = new Map(strategies_1.strategies.map(s => [s.key, s]));
|
|
26
|
-
// ─── Engine ───────────────────────────────────────────────────────────────────
|
|
27
|
-
/**
|
|
28
|
-
* Apply a single FieldRedaction rule to an object via dot-notation path.
|
|
29
|
-
* Missing fields and null values are silently skipped.
|
|
30
|
-
* Unknown strategy keys throw a descriptive error.
|
|
31
|
-
*/
|
|
32
20
|
function applyFieldRedaction(obj, rule) {
|
|
33
21
|
const strategy = strategyMap.get(rule.strategy);
|
|
34
22
|
if (!strategy)
|
|
@@ -43,16 +31,15 @@ function applyFieldRedaction(obj, rule) {
|
|
|
43
31
|
const last = parts[parts.length - 1];
|
|
44
32
|
if (cur == null || typeof cur !== 'object' || cur[last] == null)
|
|
45
33
|
return;
|
|
46
|
-
|
|
34
|
+
const original = String(cur[last]);
|
|
35
|
+
const redacted = strategy.redact(original, vault_1.hmacHash);
|
|
36
|
+
cur[last] = redacted;
|
|
37
|
+
if (strategy.key !== 'secret-key' && redacted !== original) {
|
|
38
|
+
(0, counters_1.recordRedaction)(strategy.key.toUpperCase().replace(/-/g, '_'));
|
|
39
|
+
}
|
|
47
40
|
}
|
|
48
|
-
// ─── High-Level API ───────────────────────────────────────────────────────────
|
|
49
|
-
/**
|
|
50
|
-
* Deep-redact a Shield envelope. Returns a new object (does not mutate input).
|
|
51
|
-
* Applies base redactions + schema-specific redactions based on tool_category.
|
|
52
|
-
*/
|
|
53
41
|
function redactEvent(envelope) {
|
|
54
|
-
const e =
|
|
55
|
-
// Source-level: hostname in the source block always gets redacted
|
|
42
|
+
const e = structuredClone(envelope);
|
|
56
43
|
if (e.source?.hostname) {
|
|
57
44
|
const hs = strategyMap.get('hostname');
|
|
58
45
|
if (hs)
|
|
@@ -60,11 +47,9 @@ function redactEvent(envelope) {
|
|
|
60
47
|
}
|
|
61
48
|
if (!e.event)
|
|
62
49
|
return e;
|
|
63
|
-
// Base redactions (principal.hostname, principal.user) — always apply
|
|
64
50
|
for (const rule of base_1.baseRedactions) {
|
|
65
51
|
applyFieldRedaction(e.event, rule);
|
|
66
52
|
}
|
|
67
|
-
// Schema-specific redactions via tool_category discriminator
|
|
68
53
|
const category = e.event.tool_category;
|
|
69
54
|
if (category) {
|
|
70
55
|
const schema = events_1.schemas.find(s => s.category === category);
|
|
@@ -76,7 +61,6 @@ function redactEvent(envelope) {
|
|
|
76
61
|
}
|
|
77
62
|
return e;
|
|
78
63
|
}
|
|
79
|
-
// ─── Individual Strategy Functions (backwards-compat + direct use) ────────────
|
|
80
64
|
function redactUsername(value) {
|
|
81
65
|
const s = strategyMap.get('username');
|
|
82
66
|
return s ? s.redact(value, vault_1.hmacHash) : value;
|
|
@@ -93,17 +77,14 @@ function redactCommand(value) {
|
|
|
93
77
|
const s = strategyMap.get('command');
|
|
94
78
|
return s ? s.redact(value, vault_1.hmacHash) : value;
|
|
95
79
|
}
|
|
96
|
-
// ─── Lifecycle ────────────────────────────────────────────────────────────────
|
|
97
80
|
function init() { (0, vault_1.initVault)(); }
|
|
98
81
|
function flush() { (0, vault_1.flushVault)(); }
|
|
99
|
-
// ─── Reverse Lookup ───────────────────────────────────────────────────────────
|
|
100
82
|
function reverseLookup(token) {
|
|
101
83
|
return (0, vault_1.reverseLookup)(token);
|
|
102
84
|
}
|
|
103
85
|
function getAllMappings() {
|
|
104
86
|
return (0, vault_1.getAllMappings)();
|
|
105
87
|
}
|
|
106
|
-
// ─── Testing ──────────────────────────────────────────────────────────────────
|
|
107
88
|
function _initForTesting(secret) {
|
|
108
89
|
(0, vault_1._initVaultForTesting)(secret);
|
|
109
90
|
}
|
|
@@ -8,11 +8,8 @@ exports.commandStrategy = {
|
|
|
8
8
|
if (!value || value.length === 0)
|
|
9
9
|
return value;
|
|
10
10
|
let result = value;
|
|
11
|
-
// SSH user@host patterns (e.g. ssh alice@192.168.1.1, scp bob@server:/path)
|
|
12
11
|
result = result.replace(/(\b)([\w.-]+)@([\d.]+|[\w.-]+\.\w+)/g, (_, pre, user, host) => `${pre}${hmac('user', user)}@${host}`);
|
|
13
|
-
// macOS home paths
|
|
14
12
|
result = result.replace(/\/Users\/([\w.-]+)\//g, (_, user) => `/Users/${hmac('user', user)}/`);
|
|
15
|
-
// Linux home paths
|
|
16
13
|
result = result.replace(/\/home\/([\w.-]+)\//g, (_, user) => `/home/${hmac('user', user)}/`);
|
|
17
14
|
return result;
|
|
18
15
|
},
|
|
@@ -7,7 +7,6 @@ exports.hostnameStrategy = {
|
|
|
7
7
|
redact(value, hmac) {
|
|
8
8
|
if (!value || value.length === 0)
|
|
9
9
|
return value;
|
|
10
|
-
// IPv4 addresses are not redacted — they have detection value for lateral movement rules
|
|
11
10
|
if (/^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$/.test(value))
|
|
12
11
|
return value;
|
|
13
12
|
return hmac('host', value);
|
|
@@ -5,9 +5,4 @@ import { commandStrategy } from './command';
|
|
|
5
5
|
import { secretKeyStrategy } from './secret-key';
|
|
6
6
|
import type { RedactionStrategy } from '../base';
|
|
7
7
|
export { usernameStrategy, hostnameStrategy, pathStrategy, commandStrategy, secretKeyStrategy };
|
|
8
|
-
/**
|
|
9
|
-
* Ordered registry of all redaction strategies.
|
|
10
|
-
* The engine resolves strategies by their `key` field.
|
|
11
|
-
* Order here does not affect execution — strategies are looked up by key.
|
|
12
|
-
*/
|
|
13
8
|
export declare const strategies: RedactionStrategy[];
|
|
@@ -11,11 +11,6 @@ const command_1 = require("./command");
|
|
|
11
11
|
Object.defineProperty(exports, "commandStrategy", { enumerable: true, get: function () { return command_1.commandStrategy; } });
|
|
12
12
|
const secret_key_1 = require("./secret-key");
|
|
13
13
|
Object.defineProperty(exports, "secretKeyStrategy", { enumerable: true, get: function () { return secret_key_1.secretKeyStrategy; } });
|
|
14
|
-
/**
|
|
15
|
-
* Ordered registry of all redaction strategies.
|
|
16
|
-
* The engine resolves strategies by their `key` field.
|
|
17
|
-
* Order here does not affect execution — strategies are looked up by key.
|
|
18
|
-
*/
|
|
19
14
|
exports.strategies = [
|
|
20
15
|
username_1.usernameStrategy,
|
|
21
16
|
hostname_1.hostnameStrategy,
|
|
@@ -2,9 +2,9 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.pathStrategy = void 0;
|
|
4
4
|
const HOME_PATTERNS = [
|
|
5
|
-
[/^(\/Users\/)([\w.-]+)(\/.*|$)/, '$1', '$3'],
|
|
6
|
-
[/^(\/home\/)([\w.-]+)(\/.*|$)/, '$1', '$3'],
|
|
7
|
-
[/^(C:\\Users\\)([\w.-]+)(\\.*|$)/i, '$1', '$3'],
|
|
5
|
+
[/^(\/Users\/)([\w.-]+)(\/.*|$)/, '$1', '$3'],
|
|
6
|
+
[/^(\/home\/)([\w.-]+)(\/.*|$)/, '$1', '$3'],
|
|
7
|
+
[/^(C:\\Users\\)([\w.-]+)(\\.*|$)/i, '$1', '$3'],
|
|
8
8
|
];
|
|
9
9
|
exports.pathStrategy = {
|
|
10
10
|
key: 'path',
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.secretKeyStrategy = void 0;
|
|
4
|
+
const counters_1 = require("../../counters");
|
|
4
5
|
exports.secretKeyStrategy = {
|
|
5
6
|
key: 'secret-key',
|
|
6
7
|
description: 'Detects and redacts API keys, bearer tokens, and secrets in free-text fields (command strings, arguments). Produces secret:HASH tokens.',
|
|
@@ -8,15 +9,38 @@ exports.secretKeyStrategy = {
|
|
|
8
9
|
if (!value || value.length === 0)
|
|
9
10
|
return value;
|
|
10
11
|
let result = value;
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
result = result.replace(/(--(?:token|api[_-]?key|secret|password)[=\s]+)(\S+)/gi, (_, flag, secret) =>
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
12
|
+
result = result.replace(/(AKIA[0-9A-Z]{16})/g, (_, key) => {
|
|
13
|
+
(0, counters_1.recordRedaction)('AWS_KEY');
|
|
14
|
+
return hmac('secret', key);
|
|
15
|
+
});
|
|
16
|
+
result = result.replace(/(--(?:token|api[_-]?key|secret|password)[=\s]+)(\S+)/gi, (_, flag, secret) => {
|
|
17
|
+
const flagLower = flag.toLowerCase();
|
|
18
|
+
if (/password/.test(flagLower))
|
|
19
|
+
(0, counters_1.recordRedaction)('PASSWORD');
|
|
20
|
+
else if (/api[_-]?key/.test(flagLower))
|
|
21
|
+
(0, counters_1.recordRedaction)('API_KEY');
|
|
22
|
+
else if (/secret/.test(flagLower))
|
|
23
|
+
(0, counters_1.recordRedaction)('SECRET_KEY');
|
|
24
|
+
else
|
|
25
|
+
(0, counters_1.recordRedaction)('TOKEN');
|
|
26
|
+
return `${flag}${hmac('secret', secret)}`;
|
|
27
|
+
});
|
|
28
|
+
result = result.replace(/(Bearer\s+)(\S+)/gi, (_, prefix, token) => {
|
|
29
|
+
(0, counters_1.recordRedaction)('BEARER_TOKEN');
|
|
30
|
+
return `${prefix}${hmac('secret', token)}`;
|
|
31
|
+
});
|
|
32
|
+
result = result.replace(/((?:SECRET|TOKEN|KEY|PASSWORD|API_KEY)=)(\S+)/gi, (_, prefix, secret) => {
|
|
33
|
+
const prefixUpper = prefix.toUpperCase();
|
|
34
|
+
if (prefixUpper.startsWith('PASSWORD'))
|
|
35
|
+
(0, counters_1.recordRedaction)('PASSWORD');
|
|
36
|
+
else if (prefixUpper.startsWith('API_KEY') || prefixUpper.startsWith('KEY'))
|
|
37
|
+
(0, counters_1.recordRedaction)('API_KEY');
|
|
38
|
+
else if (prefixUpper.startsWith('TOKEN'))
|
|
39
|
+
(0, counters_1.recordRedaction)('TOKEN');
|
|
40
|
+
else
|
|
41
|
+
(0, counters_1.recordRedaction)('SECRET_KEY');
|
|
42
|
+
return `${prefix}${hmac('secret', secret)}`;
|
|
43
|
+
});
|
|
20
44
|
return result;
|
|
21
45
|
},
|
|
22
46
|
};
|
|
@@ -1,25 +1,6 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* src/redactor/vault.ts — Key management and encrypted mapping store.
|
|
3
|
-
*
|
|
4
|
-
* The vault owns all I/O. Key lifecycle, map persistence, reverse lookup —
|
|
5
|
-
* everything involving files or crypto state lives here. No other module in
|
|
6
|
-
* the redactor touches the filesystem.
|
|
7
|
-
*
|
|
8
|
-
* Security layers:
|
|
9
|
-
* - File permissions: 0o600 on key and vault files
|
|
10
|
-
* - Encryption at rest: AES-256-GCM with scrypt-derived key
|
|
11
|
-
* - Atomic writes: write to .tmp then rename (prevents mid-write corruption)
|
|
12
|
-
* - In-memory isolation: state fully encapsulated in this module
|
|
13
|
-
* - Test isolation: _initVaultForTesting() uses deterministic secret, no file I/O
|
|
14
|
-
*/
|
|
15
1
|
export declare function initVault(): void;
|
|
16
2
|
export declare function flushVault(): void;
|
|
17
|
-
/**
|
|
18
|
-
* Produce a deterministic token for a (category, value) pair and record the
|
|
19
|
-
* reverse mapping. This is the HmacFn injected into every strategy.
|
|
20
|
-
*/
|
|
21
3
|
export declare function hmacHash(category: string, value: string): string;
|
|
22
4
|
export declare function reverseLookup(token: string): string | null;
|
|
23
5
|
export declare function getAllMappings(): Record<string, string>;
|
|
24
|
-
/** For testing only — deterministic secret, zero file I/O. */
|
|
25
6
|
export declare function _initVaultForTesting(secret: string): void;
|