@zintrust/trace 0.4.75 → 0.4.77
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +101 -15
- package/dist/build-manifest.json +210 -162
- package/dist/config.d.ts +1 -0
- package/dist/config.js +123 -4
- package/dist/dashboard/routes.js +4 -4
- package/dist/dashboard/ui.js +80 -23
- package/dist/index.d.ts +2 -0
- package/dist/index.js +30 -25
- package/dist/migrations/{20260331000001_create_zin_debugger_entries_table.d.ts → 20260331000001_create_zin_trace_entries_table.d.ts} +2 -2
- package/dist/migrations/{20260331000001_create_zin_debugger_entries_table.js → 20260331000001_create_zin_trace_entries_table.js} +5 -5
- package/dist/migrations/{20260331000002_create_zin_debugger_entries_tags_table.d.ts → 20260331000002_create_zin_trace_entries_tags_table.d.ts} +2 -2
- package/dist/migrations/{20260331000002_create_zin_debugger_entries_tags_table.js → 20260331000002_create_zin_trace_entries_tags_table.js} +5 -5
- package/dist/migrations/{20260331000003_create_zin_debugger_monitoring_table.d.ts → 20260331000003_create_zin_trace_monitoring_table.d.ts} +2 -2
- package/dist/migrations/{20260331000003_create_zin_debugger_monitoring_table.js → 20260331000003_create_zin_trace_monitoring_table.js} +4 -4
- package/dist/migrations/20260407193000_widen_trace_created_at_for_sql.d.ts +10 -0
- package/dist/migrations/20260407193000_widen_trace_created_at_for_sql.js +34 -0
- package/dist/migrations/index.d.ts +3 -3
- package/dist/migrations/index.js +5 -4
- package/dist/register.js +130 -32
- package/dist/storage/DebuggerStorage.js +1 -1
- package/dist/storage/TraceContentRedaction.d.ts +4 -0
- package/dist/storage/TraceContentRedaction.js +33 -0
- package/dist/storage/TraceEntryFiltering.d.ts +4 -0
- package/dist/storage/TraceEntryFiltering.js +13 -0
- package/dist/storage/TraceStorage.js +36 -6
- package/dist/storage/TraceWriteDiagnostics.d.ts +19 -0
- package/dist/storage/TraceWriteDiagnostics.js +98 -0
- package/dist/storage/index.js +1 -1
- package/dist/types.d.ts +37 -20
- package/dist/ui.js +1 -1
- package/dist/utils/authTag.js +1 -1
- package/dist/utils/entryFilter.d.ts +4 -0
- package/dist/utils/entryFilter.js +95 -0
- package/dist/utils/redact.d.ts +1 -0
- package/dist/utils/redact.js +43 -9
- package/dist/utils/requestFilter.js +1 -1
- package/dist/watchers/AuthWatcher.js +3 -3
- package/dist/watchers/BatchWatcher.js +3 -3
- package/dist/watchers/CacheWatcher.js +5 -5
- package/dist/watchers/CommandWatcher.js +5 -5
- package/dist/watchers/DumpWatcher.js +3 -3
- package/dist/watchers/EventWatcher.js +4 -4
- package/dist/watchers/ExceptionWatcher.js +6 -6
- package/dist/watchers/GateWatcher.js +3 -3
- package/dist/watchers/HttpClientWatcher.js +6 -6
- package/dist/watchers/HttpWatcher.js +108 -24
- package/dist/watchers/JobWatcher.js +4 -4
- package/dist/watchers/LogWatcher.js +5 -4
- package/dist/watchers/MailWatcher.js +3 -3
- package/dist/watchers/MiddlewareWatcher.js +3 -3
- package/dist/watchers/ModelWatcher.js +3 -3
- package/dist/watchers/NotificationWatcher.js +4 -4
- package/dist/watchers/QueryWatcher.js +5 -5
- package/dist/watchers/RedisWatcher.js +4 -4
- package/dist/watchers/ScheduleWatcher.js +3 -3
- package/dist/watchers/ViewWatcher.js +3 -3
- package/package.json +4 -4
- package/src/config.ts +152 -5
- package/src/dashboard/routes.ts +6 -2
- package/src/dashboard/ui.ts +80 -23
- package/src/index.ts +7 -0
- package/src/register.ts +137 -10
- package/src/storage/TraceContentRedaction.ts +44 -0
- package/src/storage/TraceEntryFiltering.ts +14 -0
- package/src/storage/TraceStorage.ts +52 -5
- package/src/storage/TraceWriteDiagnostics.ts +174 -0
- package/src/types.ts +40 -20
- package/src/utils/entryFilter.ts +108 -0
- package/src/utils/redact.ts +57 -9
- package/src/watchers/CommandWatcher.ts +1 -1
- package/src/watchers/HttpClientWatcher.ts +1 -1
- package/src/watchers/HttpWatcher.ts +132 -21
- package/src/watchers/LogWatcher.ts +27 -27
package/src/register.ts
CHANGED
|
@@ -22,7 +22,10 @@
|
|
|
22
22
|
import { TraceConfig } from './config';
|
|
23
23
|
import { TraceContext } from './context';
|
|
24
24
|
import { TraceStorage } from './storage';
|
|
25
|
-
import
|
|
25
|
+
import { TraceContentRedaction } from './storage/TraceContentRedaction';
|
|
26
|
+
import { TraceEntryFiltering } from './storage/TraceEntryFiltering';
|
|
27
|
+
import { TraceWriteDiagnostics } from './storage/TraceWriteDiagnostics';
|
|
28
|
+
import type { ITraceWatcherConfig, TraceConfigOverrides } from './types';
|
|
26
29
|
|
|
27
30
|
export type {}; // side-effect ESM module
|
|
28
31
|
|
|
@@ -58,6 +61,15 @@ type CoreApi = {
|
|
|
58
61
|
RequestContext?: {
|
|
59
62
|
current(): unknown;
|
|
60
63
|
};
|
|
64
|
+
Logger?: {
|
|
65
|
+
warn(message: string, context?: Record<string, unknown>): void;
|
|
66
|
+
};
|
|
67
|
+
StartupConfigFile?: {
|
|
68
|
+
Trace?: string;
|
|
69
|
+
};
|
|
70
|
+
StartupConfigFileRegistry?: {
|
|
71
|
+
get<T>(file: string): T | undefined;
|
|
72
|
+
};
|
|
61
73
|
};
|
|
62
74
|
|
|
63
75
|
type GlobalMiddlewareRegistrarState = {
|
|
@@ -101,35 +113,150 @@ const resolveTraceConnectionName = (
|
|
|
101
113
|
return resolveDefaultConnection();
|
|
102
114
|
};
|
|
103
115
|
|
|
116
|
+
const isObjectValue = (value: unknown): value is Record<string, unknown> => {
|
|
117
|
+
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
118
|
+
};
|
|
119
|
+
|
|
120
|
+
const parseEnvList = (rawValue: string): string[] | undefined => {
|
|
121
|
+
const value = rawValue.trim();
|
|
122
|
+
if (value === '') return undefined;
|
|
123
|
+
|
|
124
|
+
if (value.startsWith('[')) {
|
|
125
|
+
try {
|
|
126
|
+
const parsed = JSON.parse(value) as unknown;
|
|
127
|
+
if (Array.isArray(parsed)) {
|
|
128
|
+
return parsed
|
|
129
|
+
.filter((entry): entry is string => typeof entry === 'string')
|
|
130
|
+
.map((entry) => entry.trim())
|
|
131
|
+
.filter((entry) => entry !== '');
|
|
132
|
+
}
|
|
133
|
+
} catch {
|
|
134
|
+
// fall through to CSV parsing
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
return value
|
|
139
|
+
.split(',')
|
|
140
|
+
.map((entry) => entry.trim())
|
|
141
|
+
.filter((entry) => entry !== '');
|
|
142
|
+
};
|
|
143
|
+
|
|
144
|
+
const resolveTraceStartupOverrides = (core: CoreApi): TraceConfigOverrides | undefined => {
|
|
145
|
+
const traceConfigFile = core.StartupConfigFile?.Trace;
|
|
146
|
+
if (typeof traceConfigFile !== 'string' || traceConfigFile.trim() === '') return undefined;
|
|
147
|
+
|
|
148
|
+
const overrides = core.StartupConfigFileRegistry?.get<unknown>(traceConfigFile);
|
|
149
|
+
return isObjectValue(overrides) ? (overrides as TraceConfigOverrides) : undefined;
|
|
150
|
+
};
|
|
151
|
+
|
|
152
|
+
const buildTraceRedactionOverrides = (input: {
|
|
153
|
+
startupOverrides?: TraceConfigOverrides;
|
|
154
|
+
redactionBody?: string[];
|
|
155
|
+
redactionHeaders?: string[];
|
|
156
|
+
redactionKeys?: string[];
|
|
157
|
+
redactionQuery?: string[];
|
|
158
|
+
}): TraceConfigOverrides['redaction'] | undefined => {
|
|
159
|
+
const redaction: Partial<NonNullable<TraceConfigOverrides['redaction']>> = {
|
|
160
|
+
...(isObjectValue(input.startupOverrides?.redaction) ? input.startupOverrides?.redaction : {}),
|
|
161
|
+
};
|
|
162
|
+
|
|
163
|
+
if (input.redactionKeys === undefined) {
|
|
164
|
+
// no-op
|
|
165
|
+
} else {
|
|
166
|
+
redaction.keys = input.redactionKeys;
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
if (input.redactionHeaders === undefined) {
|
|
170
|
+
// no-op
|
|
171
|
+
} else {
|
|
172
|
+
redaction.headers = input.redactionHeaders;
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
if (input.redactionBody === undefined) {
|
|
176
|
+
// no-op
|
|
177
|
+
} else {
|
|
178
|
+
redaction.body = input.redactionBody;
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
if (input.redactionQuery === undefined) {
|
|
182
|
+
// no-op
|
|
183
|
+
} else {
|
|
184
|
+
redaction.query = input.redactionQuery;
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
return Object.keys(redaction).length > 0
|
|
188
|
+
? (redaction as NonNullable<TraceConfigOverrides['redaction']>)
|
|
189
|
+
: undefined;
|
|
190
|
+
};
|
|
191
|
+
|
|
104
192
|
const core = (await importCore()) as CoreApi;
|
|
105
193
|
const Env = core.Env;
|
|
194
|
+
const startupOverrides = resolveTraceStartupOverrides(core);
|
|
106
195
|
|
|
107
196
|
if (!traceAlreadyInitialized && Env) {
|
|
108
|
-
const enabled = Env.getBool('TRACE_ENABLED', false);
|
|
197
|
+
const enabled = startupOverrides?.enabled === true || Env.getBool('TRACE_ENABLED', false);
|
|
109
198
|
|
|
110
199
|
if (enabled) {
|
|
111
|
-
const
|
|
112
|
-
const
|
|
113
|
-
const
|
|
114
|
-
const
|
|
200
|
+
const connectionRaw = Env.get('TRACE_DB_CONNECTION', '').trim();
|
|
201
|
+
const pruneAfterHoursRaw = Env.get('TRACE_PRUNE_HOURS', '').trim();
|
|
202
|
+
const slowQueryThresholdRaw = Env.get('TRACE_SLOW_QUERY_MS', '').trim();
|
|
203
|
+
const logMinLevelRaw = Env.get('TRACE_LOG_LEVEL', '').trim();
|
|
204
|
+
const redactionKeys = parseEnvList(Env.get('TRACE_REDACT_KEYS', ''));
|
|
205
|
+
const redactionHeaders = parseEnvList(Env.get('TRACE_REDACT_HEADERS', ''));
|
|
206
|
+
const redactionBody = parseEnvList(Env.get('TRACE_REDACT_BODY', ''));
|
|
207
|
+
const redactionQuery = parseEnvList(Env.get('TRACE_REDACT_QUERY', ''));
|
|
208
|
+
|
|
209
|
+
const connection = connectionRaw === '' ? startupOverrides?.connection : connectionRaw;
|
|
210
|
+
const pruneAfterHours =
|
|
211
|
+
pruneAfterHoursRaw === ''
|
|
212
|
+
? startupOverrides?.pruneAfterHours
|
|
213
|
+
: Number.parseInt(pruneAfterHoursRaw, 10);
|
|
214
|
+
const slowQueryThreshold =
|
|
215
|
+
slowQueryThresholdRaw === ''
|
|
216
|
+
? startupOverrides?.slowQueryThreshold
|
|
217
|
+
: Number.parseInt(slowQueryThresholdRaw, 10);
|
|
218
|
+
const logMinLevel = (logMinLevelRaw === '' ? startupOverrides?.logMinLevel : logMinLevelRaw) as
|
|
115
219
|
| 'debug'
|
|
116
220
|
| 'info'
|
|
117
221
|
| 'warn'
|
|
118
222
|
| 'error'
|
|
119
223
|
| 'fatal';
|
|
224
|
+
const redaction = buildTraceRedactionOverrides({
|
|
225
|
+
startupOverrides,
|
|
226
|
+
redactionBody,
|
|
227
|
+
redactionHeaders,
|
|
228
|
+
redactionKeys,
|
|
229
|
+
redactionQuery,
|
|
230
|
+
});
|
|
120
231
|
|
|
121
232
|
const config = TraceConfig.merge({
|
|
233
|
+
...startupOverrides,
|
|
122
234
|
enabled,
|
|
123
235
|
connection,
|
|
124
|
-
pruneAfterHours
|
|
125
|
-
|
|
236
|
+
...(typeof pruneAfterHours === 'number' && Number.isFinite(pruneAfterHours)
|
|
237
|
+
? { pruneAfterHours }
|
|
238
|
+
: {}),
|
|
239
|
+
...(typeof slowQueryThreshold === 'number' && Number.isFinite(slowQueryThreshold)
|
|
240
|
+
? { slowQueryThreshold }
|
|
241
|
+
: {}),
|
|
126
242
|
logMinLevel,
|
|
243
|
+
...(redaction === undefined ? {} : { redaction }),
|
|
127
244
|
});
|
|
128
245
|
|
|
129
|
-
const
|
|
246
|
+
const resolvedConnectionName = resolveTraceConnectionName(Env, config.connection);
|
|
247
|
+
const db = core.useDatabase?.(undefined, resolvedConnectionName);
|
|
130
248
|
|
|
131
249
|
if (db) {
|
|
132
|
-
const storage =
|
|
250
|
+
const storage = TraceWriteDiagnostics.wrapStorage(
|
|
251
|
+
TraceContentRedaction.wrapStorage(
|
|
252
|
+
TraceEntryFiltering.wrapStorage(TraceStorage.resolveStorage(db), config),
|
|
253
|
+
config.redaction
|
|
254
|
+
),
|
|
255
|
+
{
|
|
256
|
+
connectionName: resolvedConnectionName,
|
|
257
|
+
logger: core.Logger,
|
|
258
|
+
}
|
|
259
|
+
);
|
|
133
260
|
|
|
134
261
|
if (core.RequestContext) {
|
|
135
262
|
TraceContext.setRequestContextImpl(
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import type { ITraceEntry, ITraceStorage, RedactionConfig } from '../types';
|
|
2
|
+
import { redactUnknown } from '../utils/redact';
|
|
3
|
+
|
|
4
|
+
const collectRedactionFields = (redaction: RedactionConfig): string[] => {
|
|
5
|
+
return [
|
|
6
|
+
...new Set([...redaction.keys, ...redaction.headers, ...redaction.body, ...redaction.query]),
|
|
7
|
+
];
|
|
8
|
+
};
|
|
9
|
+
|
|
10
|
+
const redactTraceEntry = (entry: ITraceEntry, redaction: RedactionConfig): ITraceEntry => {
|
|
11
|
+
return {
|
|
12
|
+
...entry,
|
|
13
|
+
content: redactUnknown(entry.content, collectRedactionFields(redaction)),
|
|
14
|
+
};
|
|
15
|
+
};
|
|
16
|
+
|
|
17
|
+
const redactTracePatch = (
|
|
18
|
+
patch: Partial<Pick<ITraceEntry, 'content' | 'isLatest'>>,
|
|
19
|
+
redaction: RedactionConfig
|
|
20
|
+
): Partial<Pick<ITraceEntry, 'content' | 'isLatest'>> => {
|
|
21
|
+
if (patch.content === undefined) return patch;
|
|
22
|
+
|
|
23
|
+
return {
|
|
24
|
+
...patch,
|
|
25
|
+
content: redactUnknown(patch.content, collectRedactionFields(redaction)),
|
|
26
|
+
};
|
|
27
|
+
};
|
|
28
|
+
|
|
29
|
+
export const TraceContentRedaction = Object.freeze({
|
|
30
|
+
wrapStorage(storage: ITraceStorage, redaction: RedactionConfig): ITraceStorage {
|
|
31
|
+
return Object.freeze({
|
|
32
|
+
...storage,
|
|
33
|
+
writeEntry: async (entry: ITraceEntry): Promise<void> => {
|
|
34
|
+
await storage.writeEntry(redactTraceEntry(entry, redaction));
|
|
35
|
+
},
|
|
36
|
+
updateEntry: async (
|
|
37
|
+
uuid: string,
|
|
38
|
+
patch: Partial<Pick<ITraceEntry, 'content' | 'isLatest'>>
|
|
39
|
+
): Promise<void> => {
|
|
40
|
+
await storage.updateEntry(uuid, redactTracePatch(patch, redaction));
|
|
41
|
+
},
|
|
42
|
+
});
|
|
43
|
+
},
|
|
44
|
+
});
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import type { ITraceConfig, ITraceEntry, ITraceStorage } from '../types';
|
|
2
|
+
import { TraceEntryFilter } from '../utils/entryFilter';
|
|
3
|
+
|
|
4
|
+
export const TraceEntryFiltering = Object.freeze({
|
|
5
|
+
wrapStorage(storage: ITraceStorage, config: ITraceConfig): ITraceStorage {
|
|
6
|
+
return Object.freeze({
|
|
7
|
+
...storage,
|
|
8
|
+
async writeEntry(entry: ITraceEntry) {
|
|
9
|
+
if (!TraceEntryFilter.shouldCapture(entry, config)) return;
|
|
10
|
+
await storage.writeEntry(entry);
|
|
11
|
+
},
|
|
12
|
+
});
|
|
13
|
+
},
|
|
14
|
+
});
|
|
@@ -26,6 +26,54 @@ type EntryRow = {
|
|
|
26
26
|
|
|
27
27
|
type TagRow = { entry_uuid: string; tag: string };
|
|
28
28
|
|
|
29
|
+
type DatabaseWithDriver = IDatabase & {
|
|
30
|
+
getType?: () => string;
|
|
31
|
+
};
|
|
32
|
+
|
|
33
|
+
const buildIgnoreInsert = (
|
|
34
|
+
db: IDatabase,
|
|
35
|
+
table: string,
|
|
36
|
+
columns: string[],
|
|
37
|
+
conflictColumns: string[]
|
|
38
|
+
): string => {
|
|
39
|
+
const columnList = columns.join(', ');
|
|
40
|
+
const placeholders = columns.map(() => '?').join(', ');
|
|
41
|
+
const driver = (db as DatabaseWithDriver).getType?.() ?? 'sqlite';
|
|
42
|
+
|
|
43
|
+
if (driver === 'sqlite' || driver === 'd1' || driver === 'd1-remote') {
|
|
44
|
+
return `INSERT OR IGNORE INTO ${table} (${columnList}) VALUES (${placeholders})`;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
if (driver === 'mysql') {
|
|
48
|
+
return `INSERT IGNORE INTO ${table} (${columnList}) VALUES (${placeholders})`;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
if (driver === 'postgresql') {
|
|
52
|
+
return `INSERT INTO ${table} (${columnList}) VALUES (${placeholders}) ON CONFLICT (${conflictColumns.join(', ')}) DO NOTHING`;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
if (driver === 'sqlserver') {
|
|
56
|
+
const sourceColumns = columns.map((_, index) => `v${index + 1}`);
|
|
57
|
+
const selectClause = sourceColumns.map((name) => `? AS ${name}`).join(', ');
|
|
58
|
+
const conflictClause = conflictColumns
|
|
59
|
+
.map((column) => `target.${column} = source.${column}`)
|
|
60
|
+
.join(' AND ');
|
|
61
|
+
const insertValues = columns.map((column) => `source.${column}`).join(', ');
|
|
62
|
+
const sourceProjection = columns
|
|
63
|
+
.map((column, index) => `${sourceColumns[index]} AS ${column}`)
|
|
64
|
+
.join(', ');
|
|
65
|
+
|
|
66
|
+
return [
|
|
67
|
+
`MERGE INTO ${table} WITH (HOLDLOCK) AS target`,
|
|
68
|
+
`USING (SELECT ${sourceProjection} FROM (SELECT ${selectClause}) seed) AS source`,
|
|
69
|
+
`ON ${conflictClause}`,
|
|
70
|
+
`WHEN NOT MATCHED THEN INSERT (${columnList}) VALUES (${insertValues});`,
|
|
71
|
+
].join(' ');
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
return `INSERT INTO ${table} (${columnList}) VALUES (${placeholders})`;
|
|
75
|
+
};
|
|
76
|
+
|
|
29
77
|
const rowToEntry = (row: EntryRow, tags: string[]): ITraceEntry => ({
|
|
30
78
|
uuid: row.uuid,
|
|
31
79
|
batchId: row.batch_id,
|
|
@@ -40,12 +88,11 @@ const rowToEntry = (row: EntryRow, tags: string[]): ITraceEntry => ({
|
|
|
40
88
|
const insertTags = async (db: IDatabase, uuid: string, tags: string[]): Promise<void> => {
|
|
41
89
|
if (tags.length === 0) return;
|
|
42
90
|
|
|
91
|
+
const sql = buildIgnoreInsert(db, TABLE_TAGS, ['entry_uuid', 'tag'], ['entry_uuid', 'tag']);
|
|
92
|
+
|
|
43
93
|
await Promise.all(
|
|
44
94
|
tags.map(async (tag) => {
|
|
45
|
-
await db.execute(
|
|
46
|
-
uuid,
|
|
47
|
-
tag,
|
|
48
|
-
]);
|
|
95
|
+
await db.execute(sql, [uuid, tag]);
|
|
49
96
|
})
|
|
50
97
|
);
|
|
51
98
|
};
|
|
@@ -258,7 +305,7 @@ const createStorage = (db: IDatabase): ITraceStorage => {
|
|
|
258
305
|
};
|
|
259
306
|
|
|
260
307
|
const addMonitoring = async (tag: string): Promise<void> => {
|
|
261
|
-
await db.execute(
|
|
308
|
+
await db.execute(buildIgnoreInsert(db, TABLE_MONITORING, ['tag'], ['tag']), [tag]);
|
|
262
309
|
};
|
|
263
310
|
|
|
264
311
|
const removeMonitoring = async (tag: string): Promise<void> => {
|
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
import type { EntryTypeValue, ITraceEntry, ITraceStorage } from '../types';
|
|
2
|
+
|
|
3
|
+
type TraceLogger = {
|
|
4
|
+
warn: (message: string, context?: Record<string, unknown>) => void;
|
|
5
|
+
};
|
|
6
|
+
|
|
7
|
+
type TraceWriteFailureContext = {
|
|
8
|
+
connectionName: string;
|
|
9
|
+
error: unknown;
|
|
10
|
+
operation: string;
|
|
11
|
+
watcherType?: EntryTypeValue;
|
|
12
|
+
};
|
|
13
|
+
|
|
14
|
+
type TraceWriteDiagnosticsSnapshot = {
|
|
15
|
+
degraded: boolean;
|
|
16
|
+
lastErrorMessage: string | null;
|
|
17
|
+
lastFailureAt: number | null;
|
|
18
|
+
totalFailures: number;
|
|
19
|
+
};
|
|
20
|
+
|
|
21
|
+
type TraceWriteDiagnosticsState = TraceWriteDiagnosticsSnapshot & {
|
|
22
|
+
lastLoggedAtByFingerprint: Map<string, number>;
|
|
23
|
+
};
|
|
24
|
+
|
|
25
|
+
const LOG_WINDOW_MS = 30_000;
|
|
26
|
+
|
|
27
|
+
const diagnosticsState: TraceWriteDiagnosticsState = {
|
|
28
|
+
degraded: false,
|
|
29
|
+
lastErrorMessage: null,
|
|
30
|
+
lastFailureAt: null,
|
|
31
|
+
lastLoggedAtByFingerprint: new Map<string, number>(),
|
|
32
|
+
totalFailures: 0,
|
|
33
|
+
};
|
|
34
|
+
|
|
35
|
+
const getErrorMessage = (error: unknown): string => {
|
|
36
|
+
if (error instanceof Error && error.message.trim() !== '') return error.message;
|
|
37
|
+
if (typeof error === 'string' && error.trim() !== '') return error;
|
|
38
|
+
|
|
39
|
+
try {
|
|
40
|
+
const serialized = JSON.stringify(error);
|
|
41
|
+
if (typeof serialized === 'string' && serialized !== '') return serialized;
|
|
42
|
+
} catch {
|
|
43
|
+
// ignore serialization failures
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
return 'Unknown trace storage error';
|
|
47
|
+
};
|
|
48
|
+
|
|
49
|
+
const buildFingerprint = (context: TraceWriteFailureContext): string => {
|
|
50
|
+
return [
|
|
51
|
+
context.connectionName,
|
|
52
|
+
context.operation,
|
|
53
|
+
context.watcherType ?? 'unknown',
|
|
54
|
+
getErrorMessage(context.error),
|
|
55
|
+
].join('|');
|
|
56
|
+
};
|
|
57
|
+
|
|
58
|
+
const reportFailure = (
|
|
59
|
+
logger: TraceLogger | undefined,
|
|
60
|
+
context: TraceWriteFailureContext
|
|
61
|
+
): void => {
|
|
62
|
+
const now = Date.now();
|
|
63
|
+
const errorMessage = getErrorMessage(context.error);
|
|
64
|
+
const fingerprint = buildFingerprint(context);
|
|
65
|
+
const lastLoggedAt = diagnosticsState.lastLoggedAtByFingerprint.get(fingerprint);
|
|
66
|
+
|
|
67
|
+
diagnosticsState.degraded = true;
|
|
68
|
+
diagnosticsState.lastErrorMessage = errorMessage;
|
|
69
|
+
diagnosticsState.lastFailureAt = now;
|
|
70
|
+
diagnosticsState.totalFailures += 1;
|
|
71
|
+
|
|
72
|
+
if (!logger) return;
|
|
73
|
+
if (typeof lastLoggedAt === 'number' && now - lastLoggedAt < LOG_WINDOW_MS) return;
|
|
74
|
+
|
|
75
|
+
diagnosticsState.lastLoggedAtByFingerprint.set(fingerprint, now);
|
|
76
|
+
logger.warn('[trace] Trace storage write degraded', {
|
|
77
|
+
connectionName: context.connectionName,
|
|
78
|
+
error: errorMessage,
|
|
79
|
+
lastFailureAt: now,
|
|
80
|
+
operation: context.operation,
|
|
81
|
+
totalFailures: diagnosticsState.totalFailures,
|
|
82
|
+
watcherType: context.watcherType ?? null,
|
|
83
|
+
});
|
|
84
|
+
};
|
|
85
|
+
|
|
86
|
+
const wrapStorageMethod = <TArgs extends unknown[], TResult>(
|
|
87
|
+
method: (...args: TArgs) => Promise<TResult>,
|
|
88
|
+
describeFailure: (...args: TArgs) => Omit<TraceWriteFailureContext, 'connectionName' | 'error'>,
|
|
89
|
+
connectionName: string,
|
|
90
|
+
logger?: TraceLogger
|
|
91
|
+
): ((...args: TArgs) => Promise<TResult>) => {
|
|
92
|
+
return async (...args: TArgs): Promise<TResult> => {
|
|
93
|
+
try {
|
|
94
|
+
return await method(...args);
|
|
95
|
+
} catch (error) {
|
|
96
|
+
reportFailure(logger, {
|
|
97
|
+
...describeFailure(...args),
|
|
98
|
+
connectionName,
|
|
99
|
+
error,
|
|
100
|
+
});
|
|
101
|
+
throw error;
|
|
102
|
+
}
|
|
103
|
+
};
|
|
104
|
+
};
|
|
105
|
+
|
|
106
|
+
export const TraceWriteDiagnostics = Object.freeze({
|
|
107
|
+
getSnapshot(): TraceWriteDiagnosticsSnapshot {
|
|
108
|
+
return {
|
|
109
|
+
degraded: diagnosticsState.degraded,
|
|
110
|
+
lastErrorMessage: diagnosticsState.lastErrorMessage,
|
|
111
|
+
lastFailureAt: diagnosticsState.lastFailureAt,
|
|
112
|
+
totalFailures: diagnosticsState.totalFailures,
|
|
113
|
+
};
|
|
114
|
+
},
|
|
115
|
+
|
|
116
|
+
reset(): void {
|
|
117
|
+
diagnosticsState.degraded = false;
|
|
118
|
+
diagnosticsState.lastErrorMessage = null;
|
|
119
|
+
diagnosticsState.lastFailureAt = null;
|
|
120
|
+
diagnosticsState.totalFailures = 0;
|
|
121
|
+
diagnosticsState.lastLoggedAtByFingerprint.clear();
|
|
122
|
+
},
|
|
123
|
+
|
|
124
|
+
wrapStorage(
|
|
125
|
+
storage: ITraceStorage,
|
|
126
|
+
options: { connectionName: string; logger?: TraceLogger }
|
|
127
|
+
): ITraceStorage {
|
|
128
|
+
return Object.freeze({
|
|
129
|
+
...storage,
|
|
130
|
+
writeEntry: wrapStorageMethod(
|
|
131
|
+
storage.writeEntry.bind(storage),
|
|
132
|
+
(entry: ITraceEntry) => ({ operation: 'writeEntry', watcherType: entry.type }),
|
|
133
|
+
options.connectionName,
|
|
134
|
+
options.logger
|
|
135
|
+
),
|
|
136
|
+
updateEntry: wrapStorageMethod(
|
|
137
|
+
storage.updateEntry.bind(storage),
|
|
138
|
+
(_uuid: string, _patch) => ({ operation: 'updateEntry' }),
|
|
139
|
+
options.connectionName,
|
|
140
|
+
options.logger
|
|
141
|
+
),
|
|
142
|
+
markFamilyStale: wrapStorageMethod(
|
|
143
|
+
storage.markFamilyStale.bind(storage),
|
|
144
|
+
(_familyHash: string, _exceptUuid: string) => ({ operation: 'markFamilyStale' }),
|
|
145
|
+
options.connectionName,
|
|
146
|
+
options.logger
|
|
147
|
+
),
|
|
148
|
+
prune: wrapStorageMethod(
|
|
149
|
+
storage.prune.bind(storage),
|
|
150
|
+
(_olderThanMs: number, _keepExceptions?: boolean) => ({ operation: 'prune' }),
|
|
151
|
+
options.connectionName,
|
|
152
|
+
options.logger
|
|
153
|
+
),
|
|
154
|
+
clear: wrapStorageMethod(
|
|
155
|
+
storage.clear.bind(storage),
|
|
156
|
+
() => ({ operation: 'clear' }),
|
|
157
|
+
options.connectionName,
|
|
158
|
+
options.logger
|
|
159
|
+
),
|
|
160
|
+
addMonitoring: wrapStorageMethod(
|
|
161
|
+
storage.addMonitoring.bind(storage),
|
|
162
|
+
(_tag: string) => ({ operation: 'addMonitoring' }),
|
|
163
|
+
options.connectionName,
|
|
164
|
+
options.logger
|
|
165
|
+
),
|
|
166
|
+
removeMonitoring: wrapStorageMethod(
|
|
167
|
+
storage.removeMonitoring.bind(storage),
|
|
168
|
+
(_tag: string) => ({ operation: 'removeMonitoring' }),
|
|
169
|
+
options.connectionName,
|
|
170
|
+
options.logger
|
|
171
|
+
),
|
|
172
|
+
});
|
|
173
|
+
},
|
|
174
|
+
});
|
package/src/types.ts
CHANGED
|
@@ -44,6 +44,7 @@ export interface RequestContent {
|
|
|
44
44
|
payload: Record<string, unknown>;
|
|
45
45
|
responseStatus: number;
|
|
46
46
|
responseHeaders: Record<string, string>;
|
|
47
|
+
responseBody?: unknown;
|
|
47
48
|
duration: number;
|
|
48
49
|
memory: number | null;
|
|
49
50
|
middleware: string[];
|
|
@@ -275,32 +276,51 @@ export interface ITraceWatcher {
|
|
|
275
276
|
// ---------------------------------------------------------------------------
|
|
276
277
|
|
|
277
278
|
export type RedactionConfig = {
|
|
279
|
+
keys: string[];
|
|
278
280
|
headers: string[];
|
|
279
281
|
body: string[];
|
|
280
282
|
query: string[];
|
|
281
283
|
};
|
|
282
284
|
|
|
285
|
+
export type TraceFilterRule = {
|
|
286
|
+
enabled?: boolean;
|
|
287
|
+
include?: string[];
|
|
288
|
+
exclude?: string[];
|
|
289
|
+
};
|
|
290
|
+
|
|
291
|
+
export type TraceRequestWatcherConfig = TraceFilterRule & {
|
|
292
|
+
all?: TraceFilterRule;
|
|
293
|
+
get?: TraceFilterRule;
|
|
294
|
+
post?: TraceFilterRule;
|
|
295
|
+
put?: TraceFilterRule;
|
|
296
|
+
patch?: TraceFilterRule;
|
|
297
|
+
delete?: TraceFilterRule;
|
|
298
|
+
};
|
|
299
|
+
|
|
300
|
+
export type TraceWatcherToggle = boolean | TraceFilterRule;
|
|
301
|
+
export type TraceRequestWatcherToggle = boolean | TraceRequestWatcherConfig;
|
|
302
|
+
|
|
283
303
|
export type WatcherToggles = {
|
|
284
|
-
request?:
|
|
285
|
-
query?:
|
|
286
|
-
exception?:
|
|
287
|
-
log?:
|
|
288
|
-
job?:
|
|
289
|
-
cache?:
|
|
290
|
-
schedule?:
|
|
291
|
-
mail?:
|
|
292
|
-
auth?:
|
|
293
|
-
event?:
|
|
294
|
-
model?:
|
|
295
|
-
notification?:
|
|
296
|
-
redis?:
|
|
297
|
-
gate?:
|
|
298
|
-
middleware?:
|
|
299
|
-
command?:
|
|
300
|
-
batch?:
|
|
301
|
-
dump?:
|
|
302
|
-
view?:
|
|
303
|
-
clientRequest?:
|
|
304
|
+
request?: TraceRequestWatcherToggle;
|
|
305
|
+
query?: TraceWatcherToggle;
|
|
306
|
+
exception?: TraceWatcherToggle;
|
|
307
|
+
log?: TraceWatcherToggle;
|
|
308
|
+
job?: TraceWatcherToggle;
|
|
309
|
+
cache?: TraceWatcherToggle;
|
|
310
|
+
schedule?: TraceWatcherToggle;
|
|
311
|
+
mail?: TraceWatcherToggle;
|
|
312
|
+
auth?: TraceWatcherToggle;
|
|
313
|
+
event?: TraceWatcherToggle;
|
|
314
|
+
model?: TraceWatcherToggle;
|
|
315
|
+
notification?: TraceWatcherToggle;
|
|
316
|
+
redis?: TraceWatcherToggle;
|
|
317
|
+
gate?: TraceWatcherToggle;
|
|
318
|
+
middleware?: TraceWatcherToggle;
|
|
319
|
+
command?: TraceWatcherToggle;
|
|
320
|
+
batch?: TraceWatcherToggle;
|
|
321
|
+
dump?: TraceWatcherToggle;
|
|
322
|
+
view?: TraceWatcherToggle;
|
|
323
|
+
clientRequest?: TraceWatcherToggle;
|
|
304
324
|
};
|
|
305
325
|
|
|
306
326
|
export interface ITraceConfig {
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
import type {
|
|
2
|
+
ITraceConfig,
|
|
3
|
+
ITraceEntry,
|
|
4
|
+
TraceFilterRule,
|
|
5
|
+
TraceRequestWatcherConfig,
|
|
6
|
+
WatcherToggles,
|
|
7
|
+
} from '../types';
|
|
8
|
+
import { EntryType } from '../types';
|
|
9
|
+
|
|
10
|
+
const isObjectValue = (value: unknown): value is Record<string, unknown> => {
|
|
11
|
+
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
12
|
+
};
|
|
13
|
+
|
|
14
|
+
const normalizeTerms = (terms?: string[]): string[] => {
|
|
15
|
+
if (!Array.isArray(terms)) return [];
|
|
16
|
+
|
|
17
|
+
return terms
|
|
18
|
+
.filter((term): term is string => typeof term === 'string')
|
|
19
|
+
.map((term) => term.trim().toLowerCase())
|
|
20
|
+
.filter((term) => term !== '');
|
|
21
|
+
};
|
|
22
|
+
|
|
23
|
+
const matchesRule = (haystack: string, rule?: TraceFilterRule): boolean => {
|
|
24
|
+
if (!rule) return true;
|
|
25
|
+
|
|
26
|
+
const include = normalizeTerms(rule.include);
|
|
27
|
+
const exclude = normalizeTerms(rule.exclude);
|
|
28
|
+
|
|
29
|
+
if (exclude.some((term) => haystack.includes(term))) return false;
|
|
30
|
+
if (include.length === 0) return true;
|
|
31
|
+
|
|
32
|
+
return include.some((term) => haystack.includes(term));
|
|
33
|
+
};
|
|
34
|
+
|
|
35
|
+
const toSearchableText = (entry: ITraceEntry): string => {
|
|
36
|
+
const sections = [entry.type, entry.batchId, ...(entry.tags ?? [])];
|
|
37
|
+
|
|
38
|
+
try {
|
|
39
|
+
sections.push(JSON.stringify(entry.content) ?? '');
|
|
40
|
+
} catch {
|
|
41
|
+
sections.push(String(entry.content ?? ''));
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
return sections.join(' ').toLowerCase();
|
|
45
|
+
};
|
|
46
|
+
|
|
47
|
+
const watcherKeyByEntryType: Record<ITraceEntry['type'], keyof WatcherToggles> = {
|
|
48
|
+
[EntryType.REQUEST]: 'request',
|
|
49
|
+
[EntryType.QUERY]: 'query',
|
|
50
|
+
[EntryType.EXCEPTION]: 'exception',
|
|
51
|
+
[EntryType.LOG]: 'log',
|
|
52
|
+
[EntryType.JOB]: 'job',
|
|
53
|
+
[EntryType.CACHE]: 'cache',
|
|
54
|
+
[EntryType.SCHEDULE]: 'schedule',
|
|
55
|
+
[EntryType.MAIL]: 'mail',
|
|
56
|
+
[EntryType.AUTH]: 'auth',
|
|
57
|
+
[EntryType.EVENT]: 'event',
|
|
58
|
+
[EntryType.MODEL]: 'model',
|
|
59
|
+
[EntryType.NOTIFICATION]: 'notification',
|
|
60
|
+
[EntryType.REDIS]: 'redis',
|
|
61
|
+
[EntryType.GATE]: 'gate',
|
|
62
|
+
[EntryType.MIDDLEWARE]: 'middleware',
|
|
63
|
+
[EntryType.COMMAND]: 'command',
|
|
64
|
+
[EntryType.BATCH]: 'batch',
|
|
65
|
+
[EntryType.DUMP]: 'dump',
|
|
66
|
+
[EntryType.VIEW]: 'view',
|
|
67
|
+
[EntryType.CLIENT_REQUEST]: 'clientRequest',
|
|
68
|
+
};
|
|
69
|
+
|
|
70
|
+
const getRequestMethodRule = (
|
|
71
|
+
watcher: TraceRequestWatcherConfig,
|
|
72
|
+
entry: ITraceEntry
|
|
73
|
+
): TraceFilterRule | undefined => {
|
|
74
|
+
if (entry.type !== EntryType.REQUEST) return undefined;
|
|
75
|
+
|
|
76
|
+
const content = isObjectValue(entry.content) ? entry.content : undefined;
|
|
77
|
+
const methodValue = content?.['method'];
|
|
78
|
+
const method = typeof methodValue === 'string' ? methodValue.trim().toLowerCase() : '';
|
|
79
|
+
|
|
80
|
+
if (method === 'get') return watcher.get;
|
|
81
|
+
if (method === 'post') return watcher.post;
|
|
82
|
+
if (method === 'put') return watcher.put;
|
|
83
|
+
if (method === 'patch') return watcher.patch;
|
|
84
|
+
if (method === 'delete' || method === 'del') return watcher.delete;
|
|
85
|
+
|
|
86
|
+
return watcher.all;
|
|
87
|
+
};
|
|
88
|
+
|
|
89
|
+
export const TraceEntryFilter = Object.freeze({
|
|
90
|
+
shouldCapture(entry: ITraceEntry, config: ITraceConfig): boolean {
|
|
91
|
+
const watcherKey = watcherKeyByEntryType[entry.type];
|
|
92
|
+
const watcher = config.watchers[watcherKey];
|
|
93
|
+
if (watcher === false) return false;
|
|
94
|
+
if (!isObjectValue(watcher)) return true;
|
|
95
|
+
|
|
96
|
+
const haystack = toSearchableText(entry);
|
|
97
|
+
if (!matchesRule(haystack, watcher)) return false;
|
|
98
|
+
|
|
99
|
+
if (watcherKey === 'request') {
|
|
100
|
+
const requestWatcher = watcher as TraceRequestWatcherConfig;
|
|
101
|
+
const methodRule = getRequestMethodRule(requestWatcher, entry);
|
|
102
|
+
if (!matchesRule(haystack, requestWatcher.all)) return false;
|
|
103
|
+
if (!matchesRule(haystack, methodRule)) return false;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
return true;
|
|
107
|
+
},
|
|
108
|
+
});
|