adonisjs-server-stats 1.16.0 → 1.16.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/dashboard/dashboard_store.d.ts +1 -2
- package/dist/src/dashboard/dashboard_store.js +0 -3
- package/dist/src/dashboard/dashboard_types.d.ts +2 -0
- package/dist/src/dashboard/flush_manager.d.ts +1 -6
- package/dist/src/dashboard/flush_manager.js +3 -10
- package/dist/src/dashboard/integrations/config_inspector.js +9 -49
- package/dist/src/dashboard/sensitive_patterns.d.ts +39 -0
- package/dist/src/dashboard/sensitive_patterns.js +118 -0
- package/dist/src/dashboard/write_queue.d.ts +25 -13
- package/dist/src/dashboard/write_queue.js +63 -37
- package/dist/src/debug/event_collector.d.ts +8 -0
- package/dist/src/debug/event_collector.js +12 -0
- package/dist/src/provider/dashboard_init.js +9 -0
- package/package.json +1 -1
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { DevToolbarConfig,
|
|
1
|
+
import type { DevToolbarConfig, EmailRecord } from '../debug/types.js';
|
|
2
2
|
import type { StorageStatsResult } from './storage_stats.js';
|
|
3
3
|
import type { Knex } from 'knex';
|
|
4
4
|
export type { RequestInput, PersistRequestInput, RequestFilters, QueryFilters, EventFilters, EmailFilters, LogFilters, TraceFilters, PaginatedResult, PaginateOptions, } from './dashboard_types.js';
|
|
@@ -26,7 +26,6 @@ export declare class DashboardStore {
|
|
|
26
26
|
isReady(): boolean;
|
|
27
27
|
getStorageStats(): Promise<StorageStatsResult>;
|
|
28
28
|
persistRequest(input: PersistRequestInput): Promise<number | null>;
|
|
29
|
-
queueEvents(requestIndex: number, events: EventRecord[]): void;
|
|
30
29
|
recordLog(entry: Record<string, unknown>): void;
|
|
31
30
|
recordEmail(record: EmailRecord): void;
|
|
32
31
|
flushWriteQueue(): Promise<void>;
|
|
@@ -139,9 +139,6 @@ export class DashboardStore {
|
|
|
139
139
|
this.flushMgr.persistRequest(input, this.dashboardPath);
|
|
140
140
|
return Promise.resolve(null);
|
|
141
141
|
}
|
|
142
|
-
queueEvents(requestIndex, events) {
|
|
143
|
-
this.flushMgr.queueEvents(requestIndex, events);
|
|
144
|
-
}
|
|
145
142
|
recordLog(entry) {
|
|
146
143
|
this.flushMgr.recordLog(entry);
|
|
147
144
|
}
|
|
@@ -16,6 +16,8 @@ export interface RequestInput {
|
|
|
16
16
|
}
|
|
17
17
|
export interface PersistRequestInput extends RequestInput {
|
|
18
18
|
queries: import('../debug/types.js').QueryRecord[];
|
|
19
|
+
/** Events emitted since the previous request completed. */
|
|
20
|
+
events?: import('../debug/types.js').EventRecord[];
|
|
19
21
|
trace: import('../debug/types.js').TraceRecord | null;
|
|
20
22
|
httpRequestId?: string | null;
|
|
21
23
|
}
|
|
@@ -1,12 +1,8 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { EmailRecord } from '../debug/types.js';
|
|
2
2
|
import type { PersistRequestInput } from './dashboard_types.js';
|
|
3
3
|
import type { Knex } from 'knex';
|
|
4
4
|
export declare class FlushManager {
|
|
5
5
|
writeQueue: PersistRequestInput[];
|
|
6
|
-
pendingEvents: {
|
|
7
|
-
requestIndex: number;
|
|
8
|
-
events: EventRecord[];
|
|
9
|
-
}[];
|
|
10
6
|
pendingLogs: Record<string, unknown>[];
|
|
11
7
|
pendingEmails: EmailRecord[];
|
|
12
8
|
private flushTimer;
|
|
@@ -15,7 +11,6 @@ export declare class FlushManager {
|
|
|
15
11
|
private db;
|
|
16
12
|
constructor(getDb: () => Knex | null);
|
|
17
13
|
persistRequest(input: PersistRequestInput, dashboardPath: string): void;
|
|
18
|
-
queueEvents(requestIndex: number, events: EventRecord[]): void;
|
|
19
14
|
recordLog(entry: Record<string, unknown>): void;
|
|
20
15
|
recordEmail(record: EmailRecord): void;
|
|
21
16
|
stop(): Promise<void>;
|
|
@@ -1,10 +1,9 @@
|
|
|
1
1
|
import { log } from '../utils/logger.js';
|
|
2
|
-
import { prepareRequestRows, prepareLogRows, flushRequests,
|
|
2
|
+
import { prepareRequestRows, prepareLogRows, flushRequests, flushEmails, flushLogs, hasWarned, markWarned, } from './write_queue.js';
|
|
3
3
|
const FLUSH_MS = 500;
|
|
4
4
|
const MAX_Q = 200;
|
|
5
5
|
export class FlushManager {
|
|
6
6
|
writeQueue = [];
|
|
7
|
-
pendingEvents = [];
|
|
8
7
|
pendingLogs = [];
|
|
9
8
|
pendingEmails = [];
|
|
10
9
|
flushTimer = null;
|
|
@@ -21,10 +20,6 @@ export class FlushManager {
|
|
|
21
20
|
this.writeQueue.push(input);
|
|
22
21
|
this.scheduleFlush();
|
|
23
22
|
}
|
|
24
|
-
queueEvents(requestIndex, events) {
|
|
25
|
-
if (events.length > 0)
|
|
26
|
-
this.pendingEvents.push({ requestIndex, events });
|
|
27
|
-
}
|
|
28
23
|
recordLog(entry) {
|
|
29
24
|
if (!this.db())
|
|
30
25
|
return;
|
|
@@ -94,7 +89,6 @@ export class FlushManager {
|
|
|
94
89
|
const pl = prepareLogRows(snap.logs);
|
|
95
90
|
await db.transaction(async (trx) => {
|
|
96
91
|
await flushRequests(trx, pr);
|
|
97
|
-
await flushEvents(trx, snap.events);
|
|
98
92
|
await flushEmails(trx, snap.emails);
|
|
99
93
|
await flushLogs(trx, pl);
|
|
100
94
|
});
|
|
@@ -115,10 +109,9 @@ export class FlushManager {
|
|
|
115
109
|
takeSnapshot() {
|
|
116
110
|
const requests = this.writeQueue.splice(0);
|
|
117
111
|
const logs = this.pendingLogs.splice(0);
|
|
118
|
-
const events = this.pendingEvents.splice(0);
|
|
119
112
|
const emails = this.pendingEmails.splice(0);
|
|
120
|
-
if (requests.length === 0 && logs.length === 0 &&
|
|
113
|
+
if (requests.length === 0 && logs.length === 0 && emails.length === 0)
|
|
121
114
|
return null;
|
|
122
|
-
return { requests, logs,
|
|
115
|
+
return { requests, logs, emails };
|
|
123
116
|
}
|
|
124
117
|
}
|
|
@@ -1,48 +1,9 @@
|
|
|
1
|
+
import { isSensitiveConfigName, looksLikeCredentialValue } from '../sensitive_patterns.js';
|
|
1
2
|
// ---------------------------------------------------------------------------
|
|
2
|
-
// Sensitive key
|
|
3
|
+
// Sensitive key detection
|
|
3
4
|
// ---------------------------------------------------------------------------
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
*
|
|
7
|
-
* Uses `(?:^|[_.-])` and `(?:$|[_.-])` as boundaries instead of `\b`
|
|
8
|
-
* because env vars use `_` as separators and `_` is a word character
|
|
9
|
-
* in regex, so `\b` won't match between `CLIENT` and `SECRET` in
|
|
10
|
-
* `GOOGLE_CLIENT_SECRET`.
|
|
11
|
-
*/
|
|
12
|
-
const B = '(?:^|[_.\\-])'; // boundary before
|
|
13
|
-
const A = '(?:$|[_.\\-])'; // boundary after
|
|
14
|
-
const SENSITIVE_PATTERNS = [
|
|
15
|
-
new RegExp(`${B}password${A}`, 'i'),
|
|
16
|
-
new RegExp(`${B}secret${A}`, 'i'),
|
|
17
|
-
new RegExp(`${B}token${A}`, 'i'),
|
|
18
|
-
new RegExp(`${B}credential${A}`, 'i'),
|
|
19
|
-
new RegExp(`${B}private${A}`, 'i'),
|
|
20
|
-
new RegExp(`${B}auth${A}`, 'i'),
|
|
21
|
-
// API keys: `api_key`, `apiKey`, `API_KEY`
|
|
22
|
-
/api[_-]?key/i,
|
|
23
|
-
// `_KEY` at end or `_KEY_` in middle (AWS_ACCESS_KEY_ID, ENCRYPTION_KEY, etc.)
|
|
24
|
-
/[_-]key([_-]|$)/i,
|
|
25
|
-
// ACCESS_KEY pattern (AWS credentials)
|
|
26
|
-
/access[_-]?key/i,
|
|
27
|
-
// Exact match for just "key" (standalone)
|
|
28
|
-
/^key$/i,
|
|
29
|
-
// Connection strings and DSNs
|
|
30
|
-
new RegExp(`${B}dsn${A}`, 'i'),
|
|
31
|
-
/connection[_-]?string/i,
|
|
32
|
-
// Email addresses in env var names
|
|
33
|
-
new RegExp(`${B}email${A}`, 'i'),
|
|
34
|
-
new RegExp(`${B}smtp${A}`, 'i'),
|
|
35
|
-
// Database/service URLs (often contain embedded credentials)
|
|
36
|
-
/database[_-]?url/i,
|
|
37
|
-
/redis[_-]?url/i,
|
|
38
|
-
// Webhook secrets
|
|
39
|
-
/webhook[_-]?secret/i,
|
|
40
|
-
// Signing / encryption
|
|
41
|
-
new RegExp(`${B}signing${A}`, 'i'),
|
|
42
|
-
new RegExp(`${B}encryption${A}`, 'i'),
|
|
43
|
-
// App key / app secret
|
|
44
|
-
/app[_-]key/i,
|
|
45
|
-
];
|
|
5
|
+
// The name patterns live in `../sensitive_patterns.js` so the config inspector
|
|
6
|
+
// and the SQL-binding writer share one definition of "looks like a secret".
|
|
46
7
|
const REDACTED_DISPLAY = '••••••••';
|
|
47
8
|
function redact(_value) {
|
|
48
9
|
// Never include the plaintext value: the redacted object is serialized
|
|
@@ -121,20 +82,19 @@ export class ConfigInspector {
|
|
|
121
82
|
* Check if a key name matches any sensitive pattern.
|
|
122
83
|
*/
|
|
123
84
|
function isSensitiveKey(key) {
|
|
124
|
-
return
|
|
85
|
+
return isSensitiveConfigName(key);
|
|
125
86
|
}
|
|
126
87
|
/**
|
|
127
88
|
* Check if a value looks sensitive based on its content.
|
|
128
89
|
* Catches email addresses and URLs with embedded credentials.
|
|
129
90
|
*/
|
|
130
91
|
function isSensitiveValue(value) {
|
|
131
|
-
// Email addresses
|
|
92
|
+
// Email addresses are config-sensitive (SMTP accounts) even though they are
|
|
93
|
+
// ordinary data as a query binding — hence the check lives here, not in the
|
|
94
|
+
// shared shape helper.
|
|
132
95
|
if (/^[^\s@]+@[^\s@]+\.[^\s@]+$/.test(value))
|
|
133
96
|
return true;
|
|
134
|
-
|
|
135
|
-
if (/^[a-z][a-z0-9+.-]*:\/\/[^/]*:[^/]*@/i.test(value))
|
|
136
|
-
return true;
|
|
137
|
-
return false;
|
|
97
|
+
return looksLikeCredentialValue(value);
|
|
138
98
|
}
|
|
139
99
|
/** Sanitize a single key-value pair, redacting sensitive strings. */
|
|
140
100
|
function sanitizeValue(key, value, seen) {
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared server-side rules for recognising credentials.
|
|
3
|
+
*
|
|
4
|
+
* Both the config inspector and the SQL-binding writer need to answer "does
|
|
5
|
+
* this look like a secret?", and they used to answer it differently — config
|
|
6
|
+
* values were redacted against a real word list while query bindings were only
|
|
7
|
+
* truncated by length. One list, used by both, so the same secret cannot be
|
|
8
|
+
* masked in one view and printed in full in another.
|
|
9
|
+
*/
|
|
10
|
+
/**
|
|
11
|
+
* Names that identify a credential — applies to env vars, config keys, and SQL
|
|
12
|
+
* identifiers alike.
|
|
13
|
+
*/
|
|
14
|
+
export declare const SECRET_NAME_PATTERNS: RegExp[];
|
|
15
|
+
/**
|
|
16
|
+
* Names that matter for env vars and config keys but NOT for SQL identifiers.
|
|
17
|
+
*
|
|
18
|
+
* An `email` env var is usually an SMTP account; an `email` *column* is ordinary
|
|
19
|
+
* application data, and redacting every binding of every query that touches it
|
|
20
|
+
* would make the query pane useless for debugging auth. Same for the service
|
|
21
|
+
* URLs, which are env-shaped names rather than column names.
|
|
22
|
+
*/
|
|
23
|
+
export declare const CONFIG_ONLY_NAME_PATTERNS: RegExp[];
|
|
24
|
+
/** Whether a name identifies a credential. */
|
|
25
|
+
export declare function isSecretName(name: string): boolean;
|
|
26
|
+
/** Whether a name is sensitive in a config/env context (credentials plus contact/service names). */
|
|
27
|
+
export declare function isSensitiveConfigName(name: string): boolean;
|
|
28
|
+
/**
|
|
29
|
+
* Whether a SQL statement mentions a credential-shaped identifier.
|
|
30
|
+
*
|
|
31
|
+
* Tokenised first: the name patterns above use `_`/`.`/`-` boundaries, so
|
|
32
|
+
* running them across raw SQL would miss `password` sitting between spaces.
|
|
33
|
+
*
|
|
34
|
+
* Positional bindings cannot be mapped back to specific columns reliably, so a
|
|
35
|
+
* hit means every binding for that statement is redacted. Coarse on purpose —
|
|
36
|
+
* over-redacting one statement's parameters beats storing a password.
|
|
37
|
+
*/
|
|
38
|
+
export declare function sqlMentionsSecret(sql: string): boolean;
|
|
39
|
+
export declare function looksLikeCredentialValue(value: string): boolean;
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared server-side rules for recognising credentials.
|
|
3
|
+
*
|
|
4
|
+
* Both the config inspector and the SQL-binding writer need to answer "does
|
|
5
|
+
* this look like a secret?", and they used to answer it differently — config
|
|
6
|
+
* values were redacted against a real word list while query bindings were only
|
|
7
|
+
* truncated by length. One list, used by both, so the same secret cannot be
|
|
8
|
+
* masked in one view and printed in full in another.
|
|
9
|
+
*/
|
|
10
|
+
// Custom word boundaries: `\b` does not match between `CLIENT` and `SECRET` in
|
|
11
|
+
// `GOOGLE_CLIENT_SECRET`, because `_` is a word character. These treat `_`,
|
|
12
|
+
// `.`, and `-` as separators, and also match a bare token on its own.
|
|
13
|
+
const B = '(?:^|[_.\\-])'; // boundary before
|
|
14
|
+
const A = '(?:$|[_.\\-])'; // boundary after
|
|
15
|
+
/**
|
|
16
|
+
* Names that identify a credential — applies to env vars, config keys, and SQL
|
|
17
|
+
* identifiers alike.
|
|
18
|
+
*/
|
|
19
|
+
export const SECRET_NAME_PATTERNS = [
|
|
20
|
+
new RegExp(`${B}password${A}`, 'i'),
|
|
21
|
+
new RegExp(`${B}secret${A}`, 'i'),
|
|
22
|
+
new RegExp(`${B}token${A}`, 'i'),
|
|
23
|
+
new RegExp(`${B}credential${A}`, 'i'),
|
|
24
|
+
new RegExp(`${B}private${A}`, 'i'),
|
|
25
|
+
new RegExp(`${B}auth${A}`, 'i'),
|
|
26
|
+
// API keys: `api_key`, `apiKey`, `API_KEY`
|
|
27
|
+
/api[_-]?key/i,
|
|
28
|
+
// `_KEY` at end or `_KEY_` in middle (AWS_ACCESS_KEY_ID, ENCRYPTION_KEY, etc.)
|
|
29
|
+
/[_-]key([_-]|$)/i,
|
|
30
|
+
// ACCESS_KEY pattern (AWS credentials)
|
|
31
|
+
/access[_-]?key/i,
|
|
32
|
+
// Exact match for just "key" (standalone)
|
|
33
|
+
/^key$/i,
|
|
34
|
+
// Connection strings and DSNs
|
|
35
|
+
new RegExp(`${B}dsn${A}`, 'i'),
|
|
36
|
+
/connection[_-]?string/i,
|
|
37
|
+
// Webhook secrets
|
|
38
|
+
/webhook[_-]?secret/i,
|
|
39
|
+
// Signing / encryption
|
|
40
|
+
new RegExp(`${B}signing${A}`, 'i'),
|
|
41
|
+
new RegExp(`${B}encryption${A}`, 'i'),
|
|
42
|
+
// App key / app secret
|
|
43
|
+
/app[_-]key/i,
|
|
44
|
+
// One-time codes
|
|
45
|
+
new RegExp(`${B}otp${A}`, 'i'),
|
|
46
|
+
];
|
|
47
|
+
/**
|
|
48
|
+
* Names that matter for env vars and config keys but NOT for SQL identifiers.
|
|
49
|
+
*
|
|
50
|
+
* An `email` env var is usually an SMTP account; an `email` *column* is ordinary
|
|
51
|
+
* application data, and redacting every binding of every query that touches it
|
|
52
|
+
* would make the query pane useless for debugging auth. Same for the service
|
|
53
|
+
* URLs, which are env-shaped names rather than column names.
|
|
54
|
+
*/
|
|
55
|
+
export const CONFIG_ONLY_NAME_PATTERNS = [
|
|
56
|
+
new RegExp(`${B}email${A}`, 'i'),
|
|
57
|
+
new RegExp(`${B}smtp${A}`, 'i'),
|
|
58
|
+
/database[_-]?url/i,
|
|
59
|
+
/redis[_-]?url/i,
|
|
60
|
+
];
|
|
61
|
+
/** Whether a name identifies a credential. */
|
|
62
|
+
export function isSecretName(name) {
|
|
63
|
+
return SECRET_NAME_PATTERNS.some((pattern) => pattern.test(name));
|
|
64
|
+
}
|
|
65
|
+
/** Whether a name is sensitive in a config/env context (credentials plus contact/service names). */
|
|
66
|
+
export function isSensitiveConfigName(name) {
|
|
67
|
+
return isSecretName(name) || CONFIG_ONLY_NAME_PATTERNS.some((pattern) => pattern.test(name));
|
|
68
|
+
}
|
|
69
|
+
/** Identifier-ish tokens in a SQL statement: table names, column names, aliases. */
|
|
70
|
+
const SQL_IDENTIFIER_RE = /[A-Za-z_][A-Za-z0-9_$]*/g;
|
|
71
|
+
/**
|
|
72
|
+
* Whether a SQL statement mentions a credential-shaped identifier.
|
|
73
|
+
*
|
|
74
|
+
* Tokenised first: the name patterns above use `_`/`.`/`-` boundaries, so
|
|
75
|
+
* running them across raw SQL would miss `password` sitting between spaces.
|
|
76
|
+
*
|
|
77
|
+
* Positional bindings cannot be mapped back to specific columns reliably, so a
|
|
78
|
+
* hit means every binding for that statement is redacted. Coarse on purpose —
|
|
79
|
+
* over-redacting one statement's parameters beats storing a password.
|
|
80
|
+
*/
|
|
81
|
+
export function sqlMentionsSecret(sql) {
|
|
82
|
+
for (const match of sql.matchAll(SQL_IDENTIFIER_RE)) {
|
|
83
|
+
if (isSecretName(match[0]))
|
|
84
|
+
return true;
|
|
85
|
+
}
|
|
86
|
+
return false;
|
|
87
|
+
}
|
|
88
|
+
/**
|
|
89
|
+
* Whether a value looks like a credential from its shape alone, independent of
|
|
90
|
+
* any name.
|
|
91
|
+
*
|
|
92
|
+
* Deliberately excludes bare email addresses: they are sensitive as *config*
|
|
93
|
+
* (see {@link CONFIG_ONLY_NAME_PATTERNS}) but are ordinary query parameters.
|
|
94
|
+
*/
|
|
95
|
+
const CREDENTIAL_VALUE_PATTERNS = [
|
|
96
|
+
// URL with userinfo — credentials embedded in the URL
|
|
97
|
+
/^[a-z][a-z0-9+.-]*:\/\/[^/]*:[^/]*@/i,
|
|
98
|
+
// bcrypt / argon2 / scrypt password hashes
|
|
99
|
+
/^\$(?:2[aby]|argon2[a-z]*|scrypt|s?crypt)\$/i,
|
|
100
|
+
// JWT
|
|
101
|
+
/^ey[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\./,
|
|
102
|
+
// Well-known provider key prefixes. Underscores allowed in the tail so
|
|
103
|
+
// `sk_live_…` / `pk_test_…` match (Stripe, GitHub, Slack, AWS).
|
|
104
|
+
/^(?:sk|pk|rk|whsec)_[A-Za-z0-9_]{8,}/,
|
|
105
|
+
/^gh[pousr]_[A-Za-z0-9]{8,}/,
|
|
106
|
+
/^xox[baprs]-[A-Za-z0-9-]{8,}/,
|
|
107
|
+
/^(?:AKIA|ASIA)[A-Z0-9]{12,}$/,
|
|
108
|
+
// Long hex digests — session ids, reset tokens, sha hashes
|
|
109
|
+
/^[0-9a-f]{32,}$/i,
|
|
110
|
+
];
|
|
111
|
+
/** Minimum length before a base64url-shaped blob is treated as high-entropy. */
|
|
112
|
+
const MIN_BLOB_LEN = 40;
|
|
113
|
+
export function looksLikeCredentialValue(value) {
|
|
114
|
+
// Long high-entropy base64url blobs — length-gated, so kept out of the table.
|
|
115
|
+
if (value.length >= MIN_BLOB_LEN && /^[A-Za-z0-9_-]+={0,2}$/.test(value))
|
|
116
|
+
return true;
|
|
117
|
+
return CREDENTIAL_VALUE_PATTERNS.some((pattern) => pattern.test(value));
|
|
118
|
+
}
|
|
@@ -21,13 +21,26 @@ export declare function markWarned(path: string): void;
|
|
|
21
21
|
*/
|
|
22
22
|
export declare function normalizeSql(sql: string): string;
|
|
23
23
|
/**
|
|
24
|
-
* Redact
|
|
25
|
-
* (long tokens, hashes, keys) are not stored in cleartext.
|
|
24
|
+
* Redact and truncate SQL bindings before persistence.
|
|
26
25
|
*
|
|
27
|
-
*
|
|
28
|
-
*
|
|
26
|
+
* Two independent rules, because neither catches everything on its own:
|
|
27
|
+
*
|
|
28
|
+
* 1. **By statement.** If the SQL mentions a credential-shaped identifier
|
|
29
|
+
* (`password`, `remember_token`, `otp`, ...), every binding for that
|
|
30
|
+
* statement is redacted. Positional bindings cannot be mapped back to
|
|
31
|
+
* columns reliably, so this is all-or-nothing per statement — coarse, but
|
|
32
|
+
* it is the only thing that catches a short secret like a 6-digit OTP.
|
|
33
|
+
* 2. **By value shape.** Hashes, JWTs, provider key prefixes, long hex
|
|
34
|
+
* digests, and URLs with embedded credentials are redacted wherever they
|
|
35
|
+
* appear, regardless of the statement.
|
|
36
|
+
*
|
|
37
|
+
* Anything that survives both is truncated at {@link MAX_BINDING_LEN} so an
|
|
38
|
+
* oversized payload cannot bloat the row.
|
|
39
|
+
*
|
|
40
|
+
* Ordinary parameters — ids, flags, emails, timestamps — still pass through, so
|
|
41
|
+
* the query pane stays useful for debugging.
|
|
29
42
|
*/
|
|
30
|
-
export declare function sanitizeBindings(bindings: unknown): unknown;
|
|
43
|
+
export declare function sanitizeBindings(bindings: unknown, sqlText?: string): unknown;
|
|
31
44
|
export interface PreparedQuery {
|
|
32
45
|
sql_text: string;
|
|
33
46
|
sql_normalized: string;
|
|
@@ -51,6 +64,7 @@ export interface PreparedRequest {
|
|
|
51
64
|
input: PersistRequestInput;
|
|
52
65
|
filteredQueries: PreparedQuery[];
|
|
53
66
|
traceRow: PreparedTraceRow | null;
|
|
67
|
+
eventRows: EventRow[];
|
|
54
68
|
}
|
|
55
69
|
export interface PreparedLog {
|
|
56
70
|
[key: string]: unknown;
|
|
@@ -75,7 +89,6 @@ export interface EmailRow {
|
|
|
75
89
|
}
|
|
76
90
|
export interface EventRow {
|
|
77
91
|
[key: string]: unknown;
|
|
78
|
-
request_id: null;
|
|
79
92
|
event_name: string;
|
|
80
93
|
data: string | null;
|
|
81
94
|
}
|
|
@@ -96,19 +109,18 @@ export declare function buildEmailRow(record: EmailRecord): EmailRow;
|
|
|
96
109
|
/**
|
|
97
110
|
* Transform EventRecords into SQLite-ready row objects.
|
|
98
111
|
*/
|
|
112
|
+
/**
|
|
113
|
+
* Build event rows. `request_id` is attached at insert time, once the owning
|
|
114
|
+
* request row has an id — leaving it null here (as this did previously) meant
|
|
115
|
+
* retention never reclaimed them, since events are only pruned via the
|
|
116
|
+
* `server_stats_requests` foreign-key cascade.
|
|
117
|
+
*/
|
|
99
118
|
export declare function buildEventRows(events: EventRecord[]): EventRow[];
|
|
100
119
|
/**
|
|
101
120
|
* Insert rows into a table in batches of 50.
|
|
102
121
|
*/
|
|
103
122
|
export declare function batchInsert(trx: Knex.Transaction, table: string, rows: Record<string, unknown>[]): Promise<void>;
|
|
104
123
|
export declare function flushRequests(trx: Knex.Transaction, preparedRequests: PreparedRequest[]): Promise<void>;
|
|
105
|
-
/**
|
|
106
|
-
* Flush pending events into the database.
|
|
107
|
-
*/
|
|
108
|
-
export declare function flushEvents(trx: Knex.Transaction, events: {
|
|
109
|
-
requestIndex: number;
|
|
110
|
-
events: EventRecord[];
|
|
111
|
-
}[]): Promise<void>;
|
|
112
124
|
/**
|
|
113
125
|
* Flush pending emails into the database.
|
|
114
126
|
*/
|
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
* be tested in isolation.
|
|
7
7
|
*/
|
|
8
8
|
import { round } from '../utils/math_helpers.js';
|
|
9
|
+
import { isSecretName, looksLikeCredentialValue, sqlMentionsSecret } from './sensitive_patterns.js';
|
|
9
10
|
// ---------------------------------------------------------------------------
|
|
10
11
|
// Warn-once tracking for write-path catch blocks
|
|
11
12
|
// ---------------------------------------------------------------------------
|
|
@@ -40,27 +41,60 @@ export function normalizeSql(sql) {
|
|
|
40
41
|
// ---------------------------------------------------------------------------
|
|
41
42
|
/** Max length for a persisted string binding before it is truncated. */
|
|
42
43
|
const MAX_BINDING_LEN = 256;
|
|
44
|
+
/** Placeholder stored in place of a binding that looks like a credential. */
|
|
45
|
+
const REDACTED_BINDING = '[redacted]';
|
|
43
46
|
/**
|
|
44
|
-
* Redact
|
|
45
|
-
* (long tokens, hashes, keys) are not stored in cleartext.
|
|
47
|
+
* Redact and truncate SQL bindings before persistence.
|
|
46
48
|
*
|
|
47
|
-
*
|
|
48
|
-
*
|
|
49
|
+
* Two independent rules, because neither catches everything on its own:
|
|
50
|
+
*
|
|
51
|
+
* 1. **By statement.** If the SQL mentions a credential-shaped identifier
|
|
52
|
+
* (`password`, `remember_token`, `otp`, ...), every binding for that
|
|
53
|
+
* statement is redacted. Positional bindings cannot be mapped back to
|
|
54
|
+
* columns reliably, so this is all-or-nothing per statement — coarse, but
|
|
55
|
+
* it is the only thing that catches a short secret like a 6-digit OTP.
|
|
56
|
+
* 2. **By value shape.** Hashes, JWTs, provider key prefixes, long hex
|
|
57
|
+
* digests, and URLs with embedded credentials are redacted wherever they
|
|
58
|
+
* appear, regardless of the statement.
|
|
59
|
+
*
|
|
60
|
+
* Anything that survives both is truncated at {@link MAX_BINDING_LEN} so an
|
|
61
|
+
* oversized payload cannot bloat the row.
|
|
62
|
+
*
|
|
63
|
+
* Ordinary parameters — ids, flags, emails, timestamps — still pass through, so
|
|
64
|
+
* the query pane stays useful for debugging.
|
|
49
65
|
*/
|
|
50
|
-
export function sanitizeBindings(bindings) {
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
66
|
+
export function sanitizeBindings(bindings, sqlText) {
|
|
67
|
+
const redactAll = sqlText !== undefined && sqlMentionsSecret(sqlText);
|
|
68
|
+
return sanitizeBindingValue(bindings, redactAll);
|
|
69
|
+
}
|
|
70
|
+
function sanitizeBindingValue(value, redactAll) {
|
|
71
|
+
if (Array.isArray(value))
|
|
72
|
+
return value.map((v) => sanitizeBindingValue(v, redactAll));
|
|
73
|
+
if (value !== null && typeof value === 'object')
|
|
74
|
+
return sanitizeNamedBindings(value, redactAll);
|
|
75
|
+
if (typeof value === 'string')
|
|
76
|
+
return sanitizeStringBinding(value, redactAll);
|
|
77
|
+
// A statement touching a secret column may bind it as a non-string — a numeric
|
|
78
|
+
// OTP, for instance — so redact those too rather than only masking strings.
|
|
79
|
+
// Booleans and null carry nothing worth hiding.
|
|
80
|
+
const redactable = value !== null && value !== undefined && typeof value !== 'boolean';
|
|
81
|
+
return redactAll && redactable ? REDACTED_BINDING : value;
|
|
82
|
+
}
|
|
83
|
+
/** Named bindings carry their own key, so use it when it is telling. */
|
|
84
|
+
function sanitizeNamedBindings(value, redactAll) {
|
|
85
|
+
const out = {};
|
|
86
|
+
for (const [key, nested] of Object.entries(value)) {
|
|
87
|
+
out[key] = sanitizeBindingValue(nested, redactAll || isSecretName(key));
|
|
55
88
|
}
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
89
|
+
return out;
|
|
90
|
+
}
|
|
91
|
+
function sanitizeStringBinding(value, redactAll) {
|
|
92
|
+
if (redactAll || looksLikeCredentialValue(value))
|
|
93
|
+
return REDACTED_BINDING;
|
|
94
|
+
if (value.length > MAX_BINDING_LEN) {
|
|
95
|
+
return value.slice(0, MAX_BINDING_LEN) + `…[truncated ${value.length} chars]`;
|
|
62
96
|
}
|
|
63
|
-
return
|
|
97
|
+
return value;
|
|
64
98
|
}
|
|
65
99
|
// ---------------------------------------------------------------------------
|
|
66
100
|
// Pure data-prep functions
|
|
@@ -78,13 +112,14 @@ export function prepareRequestRows(requests) {
|
|
|
78
112
|
.map((q) => ({
|
|
79
113
|
sql_text: q.sql,
|
|
80
114
|
sql_normalized: normalizeSql(q.sql),
|
|
81
|
-
bindings: q.bindings ? JSON.stringify(sanitizeBindings(q.bindings)) : null,
|
|
115
|
+
bindings: q.bindings ? JSON.stringify(sanitizeBindings(q.bindings, q.sql)) : null,
|
|
82
116
|
duration: round(q.duration),
|
|
83
117
|
method: q.method,
|
|
84
118
|
model: q.model,
|
|
85
119
|
connection: q.connection,
|
|
86
120
|
in_transaction: q.inTransaction ? 1 : 0,
|
|
87
121
|
})),
|
|
122
|
+
eventRows: buildEventRows(input.events ?? []),
|
|
88
123
|
traceRow: input.trace
|
|
89
124
|
? {
|
|
90
125
|
method: input.trace.method,
|
|
@@ -135,9 +170,14 @@ export function buildEmailRow(record) {
|
|
|
135
170
|
/**
|
|
136
171
|
* Transform EventRecords into SQLite-ready row objects.
|
|
137
172
|
*/
|
|
173
|
+
/**
|
|
174
|
+
* Build event rows. `request_id` is attached at insert time, once the owning
|
|
175
|
+
* request row has an id — leaving it null here (as this did previously) meant
|
|
176
|
+
* retention never reclaimed them, since events are only pruned via the
|
|
177
|
+
* `server_stats_requests` foreign-key cascade.
|
|
178
|
+
*/
|
|
138
179
|
export function buildEventRows(events) {
|
|
139
180
|
return events.map((e) => ({
|
|
140
|
-
request_id: null,
|
|
141
181
|
event_name: e.event,
|
|
142
182
|
data: e.data,
|
|
143
183
|
}));
|
|
@@ -176,7 +216,7 @@ function buildRequestRow(input) {
|
|
|
176
216
|
}
|
|
177
217
|
/** Insert a single prepared request with its queries and trace. */
|
|
178
218
|
async function insertOneRequest(trx, prepared) {
|
|
179
|
-
const { input, filteredQueries, traceRow } = prepared;
|
|
219
|
+
const { input, filteredQueries, traceRow, eventRows } = prepared;
|
|
180
220
|
const row = buildRequestRow(input);
|
|
181
221
|
const [requestId] = await trx('server_stats_requests').insert(row);
|
|
182
222
|
const hasId = requestId !== null && requestId !== undefined;
|
|
@@ -184,6 +224,10 @@ async function insertOneRequest(trx, prepared) {
|
|
|
184
224
|
const rows = filteredQueries.map((q) => ({ ...q, request_id: requestId }));
|
|
185
225
|
await batchInsert(trx, 'server_stats_queries', rows);
|
|
186
226
|
}
|
|
227
|
+
if (hasId && eventRows.length > 0) {
|
|
228
|
+
const rows = eventRows.map((e) => ({ ...e, request_id: requestId }));
|
|
229
|
+
await batchInsert(trx, 'server_stats_events', rows);
|
|
230
|
+
}
|
|
187
231
|
if (hasId && traceRow) {
|
|
188
232
|
await trx('server_stats_traces').insert({ ...traceRow, request_id: requestId });
|
|
189
233
|
}
|
|
@@ -202,24 +246,6 @@ export async function flushRequests(trx, preparedRequests) {
|
|
|
202
246
|
}
|
|
203
247
|
}
|
|
204
248
|
}
|
|
205
|
-
/**
|
|
206
|
-
* Flush pending events into the database.
|
|
207
|
-
*/
|
|
208
|
-
export async function flushEvents(trx, events) {
|
|
209
|
-
for (const { events: evts } of events) {
|
|
210
|
-
try {
|
|
211
|
-
const rows = buildEventRows(evts);
|
|
212
|
-
await batchInsert(trx, 'server_stats_events', rows);
|
|
213
|
-
}
|
|
214
|
-
catch (err) {
|
|
215
|
-
if (!hasWarned('recordEvents')) {
|
|
216
|
-
markWarned('recordEvents');
|
|
217
|
-
const { log } = await import('../utils/logger.js');
|
|
218
|
-
log.warn(`dashboard: recordEvents failed — ${err?.message}`);
|
|
219
|
-
}
|
|
220
|
-
}
|
|
221
|
-
}
|
|
222
|
-
}
|
|
223
249
|
/**
|
|
224
250
|
* Flush pending emails into the database.
|
|
225
251
|
*/
|
|
@@ -12,6 +12,14 @@ export declare class EventCollector {
|
|
|
12
12
|
private circulars;
|
|
13
13
|
private summarizeData;
|
|
14
14
|
getEvents(): EventRecord[];
|
|
15
|
+
/**
|
|
16
|
+
* Events recorded after `lastId`, oldest first.
|
|
17
|
+
*
|
|
18
|
+
* Mirrors `QueryCollector.getQueriesSince` and drives the dashboard's
|
|
19
|
+
* request pipe, which persists whatever accumulated since the previous
|
|
20
|
+
* request completed.
|
|
21
|
+
*/
|
|
22
|
+
getEventsSince(lastId: number): EventRecord[];
|
|
15
23
|
getLatest(n?: number): EventRecord[];
|
|
16
24
|
getTotalCount(): number;
|
|
17
25
|
getBufferInfo(): {
|
|
@@ -135,6 +135,18 @@ export class EventCollector {
|
|
|
135
135
|
getEvents() {
|
|
136
136
|
return this.buffer.toArray().reverse();
|
|
137
137
|
}
|
|
138
|
+
/**
|
|
139
|
+
* Events recorded after `lastId`, oldest first.
|
|
140
|
+
*
|
|
141
|
+
* Mirrors `QueryCollector.getQueriesSince` and drives the dashboard's
|
|
142
|
+
* request pipe, which persists whatever accumulated since the previous
|
|
143
|
+
* request completed.
|
|
144
|
+
*/
|
|
145
|
+
getEventsSince(lastId) {
|
|
146
|
+
if (lastId <= 0)
|
|
147
|
+
return this.buffer.toArray();
|
|
148
|
+
return this.buffer.collectFromEnd((e) => e.id > lastId);
|
|
149
|
+
}
|
|
138
150
|
getLatest(n = 100) {
|
|
139
151
|
return this.buffer.latest(n);
|
|
140
152
|
}
|
|
@@ -118,18 +118,27 @@ function pipeDashRequests(debugStore, dashboardStore) {
|
|
|
118
118
|
}
|
|
119
119
|
dashRequestPipeInstalled = true;
|
|
120
120
|
let lastQueryId = 0;
|
|
121
|
+
let lastEventId = 0;
|
|
121
122
|
setOnRequestComplete(({ method, url, statusCode, duration, trace, httpRequestId }) => {
|
|
122
123
|
if (!dashboardStore.isReady())
|
|
123
124
|
return;
|
|
124
125
|
const q = debugStore.queries.getQueriesSince(lastQueryId);
|
|
125
126
|
if (q.length > 0)
|
|
126
127
|
lastQueryId = q[q.length - 1].id;
|
|
128
|
+
// Events are collected globally rather than per-request, so — exactly like
|
|
129
|
+
// queries above — anything emitted outside a request is attributed to
|
|
130
|
+
// whichever request finishes next. Imprecise, but it keeps events linked to
|
|
131
|
+
// a request row, which is what retention prunes on.
|
|
132
|
+
const events = debugStore.capture.events ? debugStore.events.getEventsSince(lastEventId) : [];
|
|
133
|
+
if (events.length > 0)
|
|
134
|
+
lastEventId = events[events.length - 1].id;
|
|
127
135
|
dashboardStore.persistRequest({
|
|
128
136
|
method,
|
|
129
137
|
url,
|
|
130
138
|
statusCode,
|
|
131
139
|
duration,
|
|
132
140
|
queries: q,
|
|
141
|
+
events,
|
|
133
142
|
trace: trace ?? null,
|
|
134
143
|
httpRequestId: httpRequestId ?? null,
|
|
135
144
|
});
|