adonisjs-server-stats 1.16.1 → 1.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +25 -8
- package/dist/core/core/dashboard-api.d.ts +6 -4
- package/dist/core/core/define-config-helpers.d.ts +1 -0
- package/dist/core/core/types-diagnostics.d.ts +2 -0
- package/dist/core/debug/types.d.ts +2 -0
- package/dist/core/index.js +9 -6
- package/dist/core/types.d.ts +55 -4
- package/dist/react/{CacheSection-DkLtukWz.js → CacheSection-B-lSIN8Q.js} +48 -44
- package/dist/react/{CacheTab-DWeXnEBI.js → CacheTab-BN0QgPXz.js} +1 -1
- package/dist/react/{ConfigSection-DaVbFPDg.js → ConfigSection-u_TSfb9f.js} +1 -1
- package/dist/react/{ConfigTab-D4640WJZ.js → ConfigTab-au26s-ON.js} +1 -1
- package/dist/react/{CustomPaneTab-DglmAVMC.js → CustomPaneTab-qvMnh8ss.js} +1 -1
- package/dist/react/{EmailsSection-vYx6ExTb.js → EmailsSection-HfEMeRex.js} +65 -57
- package/dist/react/{EmailsTab-DKDrRmRH.js → EmailsTab-BWwq-CxB.js} +1 -1
- package/dist/react/{EventsSection-CapH5xut.js → EventsSection-CA9b15dJ.js} +1 -1
- package/dist/react/{EventsTab-BNVEySAm.js → EventsTab-CgSdHbKU.js} +1 -1
- package/dist/react/{InternalsContent-C9lIA92C.js → InternalsContent-CI7HvBtH.js} +51 -47
- package/dist/react/{InternalsSection-lOv_UcSV.js → InternalsSection-IEeWXJqP.js} +1 -1
- package/dist/react/{InternalsTab-BygoSorC.js → InternalsTab-CoVLHQbF.js} +1 -1
- package/dist/react/{JobsSection-ZIQsJWid.js → JobsSection-DgsS_VwU.js} +1 -1
- package/dist/react/{JobsTab-DXQzXrDt.js → JobsTab-CzwLSRUC.js} +1 -1
- package/dist/react/{LogEntryRow-BWkHE51-.js → LogEntryRow-VCCzwD1H.js} +1 -1
- package/dist/react/{LogsSection-ConXdBkL.js → LogsSection-BujNn_cO.js} +2 -2
- package/dist/react/{LogsTab-CJM47LPn.js → LogsTab-BtD1h7DM.js} +2 -2
- package/dist/react/{OverviewSection-DOMu2qvl.js → OverviewSection-B28em0gQ.js} +119 -111
- package/dist/react/{QueriesSection-DFsaOSJI.js → QueriesSection-B0tFn_Dc.js} +1 -1
- package/dist/react/{QueriesTab-CY9CG_7L.js → QueriesTab-DJY7aavR.js} +1 -1
- package/dist/react/{RequestsSection-CC-eVHsl.js → RequestsSection-DPmYM7tq.js} +2 -2
- package/dist/react/{RoutesSection-p1DMq41y.js → RoutesSection-Ck-RcFVS.js} +1 -1
- package/dist/react/{RoutesTab-Cnqy8UcK.js → RoutesTab-Ch3PDLzo.js} +1 -1
- package/dist/react/{SplitPaneWrapper-XgkA0QxE.js → SplitPaneWrapper-BZ6VrBr-.js} +1 -1
- package/dist/react/{TimelineTab-1YOERxe5.js → TimelineTab-_byxh2mJ.js} +2 -2
- package/dist/react/components/Dashboard/overview-data-context.d.ts +15 -0
- package/dist/react/components/Dashboard/sections/OverviewSection.d.ts +9 -0
- package/dist/react/{index-DOSlCpZ9.js → index-De7nsWBN.js} +148 -141
- package/dist/react/index.js +1 -1
- package/dist/src/collectors/app_collector.d.ts +11 -0
- package/dist/src/collectors/app_collector.js +11 -0
- package/dist/src/collectors/redis_collector.js +13 -13
- package/dist/src/dashboard/cache_handlers.d.ts +17 -3
- package/dist/src/dashboard/cache_handlers.js +14 -19
- package/dist/src/dashboard/coalesce_cache.d.ts +2 -8
- package/dist/src/dashboard/coalesce_cache.js +25 -0
- package/dist/src/dashboard/dashboard_controller.d.ts +1 -0
- package/dist/src/dashboard/dashboard_controller.js +8 -3
- package/dist/src/dashboard/dashboard_store.d.ts +0 -1
- package/dist/src/dashboard/dashboard_store.js +18 -18
- package/dist/src/dashboard/filter_handlers.js +26 -4
- package/dist/src/dashboard/flush_manager.d.ts +1 -0
- package/dist/src/dashboard/flush_manager.js +6 -1
- package/dist/src/dashboard/format_helpers.d.ts +0 -4
- package/dist/src/dashboard/format_helpers.js +0 -19
- package/dist/src/dashboard/integrations/config_inspector.js +6 -3
- package/dist/src/dashboard/knex_factory.js +8 -0
- package/dist/src/dashboard/migrator.d.ts +7 -2
- package/dist/src/dashboard/migrator.js +140 -6
- package/dist/src/dashboard/migrator_tables.js +8 -0
- package/dist/src/dashboard/paginate_helper.js +21 -11
- package/dist/src/dashboard/sensitive_patterns.d.ts +37 -0
- package/dist/src/dashboard/sensitive_patterns.js +146 -5
- package/dist/src/dashboard/storage_stats.d.ts +5 -0
- package/dist/src/dashboard/storage_stats.js +15 -1
- package/dist/src/dashboard/write_queue.d.ts +20 -21
- package/dist/src/dashboard/write_queue.js +37 -60
- package/dist/src/debug/query_collector.js +6 -1
- package/dist/src/debug/trace_collector.js +26 -13
- package/dist/src/debug/types.d.ts +2 -0
- package/dist/src/define_config.js +2 -0
- package/dist/src/edge/client/dashboard.js +2 -2
- package/dist/src/edge/client/debug-panel-deferred.js +1 -1
- package/dist/src/edge/client-vue/dashboard.js +5 -5
- package/dist/src/edge/client-vue/debug-panel-deferred.js +2 -2
- package/dist/src/edge/views/dashboard.edge +1 -1
- package/dist/src/edge/views/debug-panel.edge +1 -1
- package/dist/src/edge/views/stats-bar.edge +1 -1
- package/dist/src/middleware/request_tracking_middleware.js +7 -2
- package/dist/src/provider/dashboard_init.d.ts +9 -1
- package/dist/src/provider/dashboard_init.js +10 -0
- package/dist/src/provider/dashboard_setup.js +14 -0
- package/dist/src/provider/diagnostics.d.ts +2 -0
- package/dist/src/provider/diagnostics.js +1 -0
- package/dist/src/provider/email_helpers.js +20 -5
- package/dist/src/provider/server_stats_provider.d.ts +10 -0
- package/dist/src/provider/server_stats_provider.js +26 -7
- package/dist/src/provider/shutdown_helpers.js +3 -1
- package/dist/src/provider/toolbar_setup.js +21 -15
- package/dist/src/stubs/config.stub +4 -0
- package/dist/src/types.d.ts +55 -4
- package/dist/vue/CacheSection-z2TQRez9.js +161 -0
- package/dist/vue/{ConfigSection-BiRAiaHj.js → ConfigSection-CFEaig69.js} +1 -1
- package/dist/vue/{EmailsSection-Dl44qyqY.js → EmailsSection-CYohBo62.js} +89 -85
- package/dist/vue/{EventsSection-CWjeitjU.js → EventsSection-B_TuVrGj.js} +1 -1
- package/dist/vue/{InternalsSection-jhTCuK3j.js → InternalsSection-B1P2kNT-.js} +92 -84
- package/dist/vue/{InternalsTab-Z3c82glB.js → InternalsTab-CRI-IFTo.js} +155 -147
- package/dist/vue/{JobsSection-DOBb4LjZ.js → JobsSection-AjkEzyiv.js} +1 -1
- package/dist/vue/{LogsSection-CXx-HOWJ.js → LogsSection-DZD0CCDi.js} +1 -1
- package/dist/vue/{OverviewSection-CyfNQ8uV.js → OverviewSection-BEc4Gktf.js} +1 -1
- package/dist/vue/{QueriesSection-CXBsFp-y.js → QueriesSection-BMImGXTn.js} +1 -1
- package/dist/vue/{RequestsSection-YTIaZGZd.js → RequestsSection-DOI2MUFX.js} +1 -1
- package/dist/vue/{RoutesSection-CDKMey49.js → RoutesSection-C1sxt-AW.js} +1 -1
- package/dist/vue/composables/useDashboardData.d.ts +1 -1
- package/dist/vue/{index-14x39RI_.js → index-BfPAAjL5.js} +95 -95
- package/dist/vue/index.js +1 -1
- package/package.json +12 -3
- package/dist/src/dashboard/explain_query.d.ts +0 -8
- package/dist/src/dashboard/explain_query.js +0 -22
- package/dist/src/provider/boot_initializer.d.ts +0 -28
- package/dist/src/provider/boot_initializer.js +0 -35
- package/dist/vue/CacheSection-CxEBVVkF.js +0 -156
|
@@ -27,15 +27,20 @@ export async function autoMigrate(db) {
|
|
|
27
27
|
await migrateSavedFilters(db);
|
|
28
28
|
}
|
|
29
29
|
/**
|
|
30
|
-
* Delete records older than `retentionDays` from all tables
|
|
30
|
+
* Delete records older than `retentionDays` from all tables, then enforce
|
|
31
|
+
* the database size cap.
|
|
31
32
|
*
|
|
32
33
|
* Foreign-key cascades on `server_stats_requests` handle the child
|
|
33
34
|
* tables (queries, events, traces). Standalone tables (logs, emails,
|
|
34
35
|
* metrics, saved_filters) are pruned individually.
|
|
35
36
|
*
|
|
37
|
+
* `maxDbSizeMb` bounds the *live* data size: when exceeded, the globally
|
|
38
|
+
* oldest rows are deleted — regardless of age — until usage drops below
|
|
39
|
+
* ~90% of the cap. `0` disables the cap.
|
|
40
|
+
*
|
|
36
41
|
* Yields between each DELETE so the event loop stays responsive.
|
|
37
42
|
*/
|
|
38
|
-
export async function runRetentionCleanup(db, retentionDays) {
|
|
43
|
+
export async function runRetentionCleanup(db, retentionDays, maxDbSizeMb = 0) {
|
|
39
44
|
// Use string interpolation instead of parameterized bindings.
|
|
40
45
|
// Knex + better-sqlite3 can hang on parameterized db.raw() calls,
|
|
41
46
|
// while non-parameterized queries (used in migrations) work fine.
|
|
@@ -53,7 +58,12 @@ export async function runRetentionCleanup(db, retentionDays) {
|
|
|
53
58
|
await yieldToEventLoop();
|
|
54
59
|
await batchDelete(db, 'server_stats_metrics', cutoff);
|
|
55
60
|
await yieldToEventLoop();
|
|
56
|
-
|
|
61
|
+
if (maxDbSizeMb > 0)
|
|
62
|
+
await enforceSizeCap(db, maxDbSizeMb);
|
|
63
|
+
// Return deleted pages to the OS — PRAGMA optimize alone only updates
|
|
64
|
+
// planner stats, leaving the file at its high-water mark forever.
|
|
65
|
+
await reclaimFreeSpace(db);
|
|
66
|
+
// Update query planner statistics
|
|
57
67
|
await db.raw('PRAGMA optimize');
|
|
58
68
|
}
|
|
59
69
|
catch (err) {
|
|
@@ -72,10 +82,134 @@ async function batchDelete(db, table, cutoff) {
|
|
|
72
82
|
let hasMore = true;
|
|
73
83
|
while (hasMore) {
|
|
74
84
|
await db.raw(`DELETE FROM ${table} WHERE rowid IN (SELECT rowid FROM ${table} WHERE created_at < ${cutoff} LIMIT 1000)`);
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
85
|
+
// changes() reports the last DELETE's row count on this connection
|
|
86
|
+
// (pool is min:1/max:1). Re-counting the remaining backlog here instead
|
|
87
|
+
// made total work quadratic in backlog size.
|
|
88
|
+
const deleted = await pragmaNumber(db, 'SELECT changes() AS n');
|
|
89
|
+
hasMore = deleted === 1000;
|
|
78
90
|
if (hasMore)
|
|
79
91
|
await yieldToEventLoop();
|
|
80
92
|
}
|
|
81
93
|
}
|
|
94
|
+
/** Root tables pruned by the size cap; requests cascades to queries/events/traces. */
|
|
95
|
+
const SIZE_CAP_TABLES = [
|
|
96
|
+
'server_stats_requests',
|
|
97
|
+
'server_stats_logs',
|
|
98
|
+
'server_stats_emails',
|
|
99
|
+
'server_stats_metrics',
|
|
100
|
+
];
|
|
101
|
+
/**
|
|
102
|
+
* Bytes of live data: total pages minus freelist pages. Freed pages are
|
|
103
|
+
* excluded because reclaimFreeSpace returns them to the OS afterwards.
|
|
104
|
+
*/
|
|
105
|
+
async function liveSizeBytes(db, pageSize) {
|
|
106
|
+
const pages = await pragmaNumber(db, 'PRAGMA page_count');
|
|
107
|
+
const freelist = await pragmaNumber(db, 'PRAGMA freelist_count');
|
|
108
|
+
return (pages - freelist) * pageSize;
|
|
109
|
+
}
|
|
110
|
+
/**
|
|
111
|
+
* Keep live data under `maxSizeMb` by deleting the globally oldest rows,
|
|
112
|
+
* whatever their age — a size cap effectively shortens the retention
|
|
113
|
+
* window when write volume outruns `retentionDays`.
|
|
114
|
+
*
|
|
115
|
+
* Prunes down to ~90% of the cap so the hourly cleanup doesn't re-trigger
|
|
116
|
+
* on every pass. Each round deletes 1000 rows from whichever root table
|
|
117
|
+
* currently holds the oldest data, so mixed workloads (trace-heavy vs
|
|
118
|
+
* log-heavy) shed their actual oldest history first.
|
|
119
|
+
*/
|
|
120
|
+
async function enforceSizeCap(db, maxSizeMb) {
|
|
121
|
+
const budget = maxSizeMb * 1024 * 1024;
|
|
122
|
+
const pageSize = await pragmaNumber(db, 'PRAGMA page_size');
|
|
123
|
+
if ((await liveSizeBytes(db, pageSize)) <= budget)
|
|
124
|
+
return;
|
|
125
|
+
const { log } = await import('../utils/logger.js');
|
|
126
|
+
log.info(`dashboard: database over the ${maxSizeMb} MB size cap — pruning oldest records`);
|
|
127
|
+
const target = budget * 0.9;
|
|
128
|
+
// 10k rounds × 1000 rows bounds one pass; the next hourly pass continues.
|
|
129
|
+
let guard = 10_000;
|
|
130
|
+
while (guard-- > 0 && (await liveSizeBytes(db, pageSize)) > target) {
|
|
131
|
+
const table = await findTableWithOldestRow(db);
|
|
132
|
+
if (!table)
|
|
133
|
+
break;
|
|
134
|
+
await db.raw(`DELETE FROM ${table} WHERE rowid IN (SELECT rowid FROM ${table} ORDER BY created_at LIMIT 1000)`);
|
|
135
|
+
const deleted = await pragmaNumber(db, 'SELECT changes() AS n');
|
|
136
|
+
if (deleted === 0)
|
|
137
|
+
break;
|
|
138
|
+
await yieldToEventLoop();
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
/** The root table whose oldest row is globally oldest, or null when all are empty. */
|
|
142
|
+
async function findTableWithOldestRow(db) {
|
|
143
|
+
let best = null;
|
|
144
|
+
let bestTs = '';
|
|
145
|
+
for (const table of SIZE_CAP_TABLES) {
|
|
146
|
+
const rows = (await db.raw(`SELECT MIN(created_at) AS ts FROM ${table}`));
|
|
147
|
+
const ts = rows?.[0]?.ts;
|
|
148
|
+
if (ts && (best === null || ts < bestTs)) {
|
|
149
|
+
best = table;
|
|
150
|
+
bestTs = ts;
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
return best;
|
|
154
|
+
}
|
|
155
|
+
/**
|
|
156
|
+
* Return freelist pages to the OS after a cleanup pass.
|
|
157
|
+
*
|
|
158
|
+
* Databases created with `auto_vacuum=INCREMENTAL` are trimmed in bounded
|
|
159
|
+
* chunks so each synchronous step stays short. Legacy databases (created
|
|
160
|
+
* before the pragma existed) can't be trimmed incrementally, so when dead
|
|
161
|
+
* pages exceed ~30% of the file a one-time VACUUM rewrites it — which also
|
|
162
|
+
* converts it to incremental mode, since `auto_vacuum=INCREMENTAL` is set
|
|
163
|
+
* on this connection.
|
|
164
|
+
*/
|
|
165
|
+
async function reclaimFreeSpace(db) {
|
|
166
|
+
const mode = await pragmaNumber(db, 'PRAGMA auto_vacuum');
|
|
167
|
+
if (mode === 2) {
|
|
168
|
+
// Chunks of ~2000 pages (~8 MB at the 4 KB default) keep each synchronous
|
|
169
|
+
// call short; guard bounds one pass at ~8 GB in case freelist_count
|
|
170
|
+
// misbehaves.
|
|
171
|
+
let guard = 1000;
|
|
172
|
+
while (guard-- > 0) {
|
|
173
|
+
const freelist = await pragmaNumber(db, 'PRAGMA freelist_count');
|
|
174
|
+
if (freelist <= 0)
|
|
175
|
+
break;
|
|
176
|
+
if (!(await incrementalVacuumChunk(db)))
|
|
177
|
+
break;
|
|
178
|
+
await yieldToEventLoop();
|
|
179
|
+
}
|
|
180
|
+
return;
|
|
181
|
+
}
|
|
182
|
+
const pageCount = await pragmaNumber(db, 'PRAGMA page_count');
|
|
183
|
+
const freelistCount = await pragmaNumber(db, 'PRAGMA freelist_count');
|
|
184
|
+
if (pageCount > 0 && freelistCount / pageCount > 0.3) {
|
|
185
|
+
const { log } = await import('../utils/logger.js');
|
|
186
|
+
log.info(`dashboard: reclaiming ${freelistCount} of ${pageCount} pages via one-time VACUUM (may pause briefly)`);
|
|
187
|
+
await db.raw('VACUUM');
|
|
188
|
+
log.info('dashboard: VACUUM complete — database converted to incremental auto-vacuum');
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
/**
|
|
192
|
+
* Free up to 2000 freelist pages. incremental_vacuum frees ONE page per
|
|
193
|
+
* sqlite3_step, and knex's raw() steps no-result pragmas exactly once — so
|
|
194
|
+
* it must run on the underlying better-sqlite3 handle, whose .pragma()
|
|
195
|
+
* steps to completion. Returns false if the handle has no pragma method.
|
|
196
|
+
*/
|
|
197
|
+
async function incrementalVacuumChunk(db) {
|
|
198
|
+
const client = db.client;
|
|
199
|
+
const conn = (await client.acquireConnection());
|
|
200
|
+
try {
|
|
201
|
+
if (typeof conn.pragma !== 'function')
|
|
202
|
+
return false;
|
|
203
|
+
conn.pragma('incremental_vacuum(2000)');
|
|
204
|
+
return true;
|
|
205
|
+
}
|
|
206
|
+
finally {
|
|
207
|
+
client.releaseConnection(conn);
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
/** Run a single-row/single-column statement and return its value as a number. */
|
|
211
|
+
async function pragmaNumber(db, statement) {
|
|
212
|
+
const rows = (await db.raw(statement));
|
|
213
|
+
const value = Object.values(rows?.[0] ?? {})[0];
|
|
214
|
+
return Number(value ?? 0);
|
|
215
|
+
}
|
|
@@ -28,6 +28,10 @@ export async function migrateRequests(db) {
|
|
|
28
28
|
await db.raw(`CREATE INDEX IF NOT EXISTS idx_ss_requests_created ON server_stats_requests(created_at)`);
|
|
29
29
|
await db.raw(`CREATE INDEX IF NOT EXISTS idx_ss_requests_url ON server_stats_requests(url)`);
|
|
30
30
|
await db.raw(`CREATE INDEX IF NOT EXISTS idx_ss_requests_duration ON server_stats_requests(duration)`);
|
|
31
|
+
// Covers the overview p95 query (WHERE created_at >= ? ORDER BY duration):
|
|
32
|
+
// with only the single-column indexes SQLite sorts the whole filtered range
|
|
33
|
+
// on every 2s cache miss.
|
|
34
|
+
await db.raw(`CREATE INDEX IF NOT EXISTS idx_ss_requests_created_duration ON server_stats_requests(created_at, duration)`);
|
|
31
35
|
await db.raw(`CREATE INDEX IF NOT EXISTS idx_ss_requests_status ON server_stats_requests(status_code)`);
|
|
32
36
|
try {
|
|
33
37
|
await db.raw('ALTER TABLE server_stats_requests ADD COLUMN http_request_id TEXT');
|
|
@@ -68,6 +72,10 @@ export async function migrateEvents(db) {
|
|
|
68
72
|
`);
|
|
69
73
|
await db.raw(`CREATE INDEX IF NOT EXISTS idx_ss_events_created ON server_stats_events(created_at)`);
|
|
70
74
|
await db.raw(`CREATE INDEX IF NOT EXISTS idx_ss_events_name ON server_stats_events(event_name)`);
|
|
75
|
+
// Without this index every cascaded delete from server_stats_requests
|
|
76
|
+
// full-scans the events table (synchronously, on the host event loop) —
|
|
77
|
+
// the other FK children (queries, traces, logs) all have one.
|
|
78
|
+
await db.raw(`CREATE INDEX IF NOT EXISTS idx_ss_events_request ON server_stats_events(request_id)`);
|
|
71
79
|
}
|
|
72
80
|
export async function migrateEmails(db) {
|
|
73
81
|
await db.raw(`
|
|
@@ -15,31 +15,41 @@ export function clampPerPage(value) {
|
|
|
15
15
|
return 25;
|
|
16
16
|
return clamp(Math.trunc(n), 1, 200);
|
|
17
17
|
}
|
|
18
|
+
/**
|
|
19
|
+
* Rows the COUNT is allowed to scan. Filters include leading-wildcard LIKEs
|
|
20
|
+
* that no index can serve, so an uncapped COUNT walks the entire table twice
|
|
21
|
+
* per request (count + data). Past this many matches the exact total stops
|
|
22
|
+
* being interesting — the UI just needs "lots of pages".
|
|
23
|
+
*/
|
|
24
|
+
const COUNT_SCAN_CAP = 10_000;
|
|
18
25
|
/**
|
|
19
26
|
* Execute a paginated query within a transaction.
|
|
20
27
|
*/
|
|
21
28
|
export async function executePaginate(db, opts) {
|
|
22
29
|
const perPage = clampPerPage(opts.perPage);
|
|
23
30
|
return db.transaction(async (trx) => {
|
|
24
|
-
const
|
|
31
|
+
const countSub = trx(opts.table).select(trx.raw('1')).limit(COUNT_SCAN_CAP);
|
|
25
32
|
if (opts.applyFilters)
|
|
26
|
-
opts.applyFilters(
|
|
27
|
-
const
|
|
28
|
-
const total = Number(
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
33
|
+
opts.applyFilters(countSub);
|
|
34
|
+
const countRows = (await trx.count('* as count').from(countSub.as('t')));
|
|
35
|
+
const total = Number(countRows[0]?.count ?? 0);
|
|
36
|
+
// Clamp the page into the real range: an arbitrary ?page= forces a
|
|
37
|
+
// full-scan-sized OFFSET on the single sqlite connection, blocking every
|
|
38
|
+
// other dashboard read for nothing.
|
|
39
|
+
const lastPage = Math.ceil(total / perPage);
|
|
40
|
+
const requested = Number.isFinite(Number(opts.page)) ? Math.trunc(Number(opts.page)) : 1;
|
|
41
|
+
const page = clamp(requested, 1, Math.max(1, lastPage));
|
|
42
|
+
const offset = (page - 1) * perPage;
|
|
43
|
+
const dataQuery = trx(opts.table).orderBy('created_at', 'desc').limit(perPage).offset(offset);
|
|
34
44
|
if (opts.applyFilters)
|
|
35
45
|
opts.applyFilters(dataQuery);
|
|
36
46
|
const data = await dataQuery;
|
|
37
47
|
return {
|
|
38
48
|
data,
|
|
39
49
|
total,
|
|
40
|
-
page
|
|
50
|
+
page,
|
|
41
51
|
perPage,
|
|
42
|
-
lastPage
|
|
52
|
+
lastPage,
|
|
43
53
|
};
|
|
44
54
|
});
|
|
45
55
|
}
|
|
@@ -37,3 +37,40 @@ export declare function isSensitiveConfigName(name: string): boolean;
|
|
|
37
37
|
*/
|
|
38
38
|
export declare function sqlMentionsSecret(sql: string): boolean;
|
|
39
39
|
export declare function looksLikeCredentialValue(value: string): boolean;
|
|
40
|
+
/**
|
|
41
|
+
* Redact and truncate SQL bindings before persistence.
|
|
42
|
+
*
|
|
43
|
+
* Two independent rules, because neither catches everything on its own:
|
|
44
|
+
*
|
|
45
|
+
* 1. **By statement.** If the SQL mentions a credential-shaped identifier
|
|
46
|
+
* (`password`, `remember_token`, `otp`, ...), every binding for that
|
|
47
|
+
* statement is redacted. Positional bindings cannot be mapped back to
|
|
48
|
+
* columns reliably, so this is all-or-nothing per statement — coarse, but
|
|
49
|
+
* it is the only thing that catches a short secret like a 6-digit OTP.
|
|
50
|
+
* 2. **By value shape.** Hashes, JWTs, provider key prefixes, long hex
|
|
51
|
+
* digests, and URLs with embedded credentials are redacted wherever they
|
|
52
|
+
* appear, regardless of the statement.
|
|
53
|
+
*
|
|
54
|
+
* Anything that survives both is truncated at {@link MAX_BINDING_LEN} so an
|
|
55
|
+
* oversized payload cannot bloat the row.
|
|
56
|
+
*
|
|
57
|
+
* Ordinary parameters — ids, flags, emails, timestamps — still pass through, so
|
|
58
|
+
* the query pane stays useful for debugging.
|
|
59
|
+
*/
|
|
60
|
+
export declare function sanitizeBindings(bindings: unknown, sqlText?: string): unknown;
|
|
61
|
+
/**
|
|
62
|
+
* Redact a structured record (a pino log entry, an event payload) with the
|
|
63
|
+
* same key/shape rules as bindings but WITHOUT length truncation — log data
|
|
64
|
+
* carries stack traces and payload context that must stay whole.
|
|
65
|
+
*/
|
|
66
|
+
export declare function sanitizeRecordValues(record: unknown): unknown;
|
|
67
|
+
/**
|
|
68
|
+
* Redact credential-bearing query parameters from a URL before it is stored.
|
|
69
|
+
*
|
|
70
|
+
* Request URLs are persisted with their query strings (useful for debugging
|
|
71
|
+
* pagination and filters), but a `?token=…` password-reset link or a
|
|
72
|
+
* `?signature=…` signed URL must not land in SQLite in the clear. Parameters
|
|
73
|
+
* are redacted by name (see {@link isSecretName}) or when the value itself
|
|
74
|
+
* looks like a credential; everything else passes through untouched.
|
|
75
|
+
*/
|
|
76
|
+
export declare function sanitizeUrlQuery(url: string): string;
|
|
@@ -9,9 +9,22 @@
|
|
|
9
9
|
*/
|
|
10
10
|
// Custom word boundaries: `\b` does not match between `CLIENT` and `SECRET` in
|
|
11
11
|
// `GOOGLE_CLIENT_SECRET`, because `_` is a word character. These treat `_`,
|
|
12
|
-
// `.`, and `-` as separators, and also match a bare token on its own.
|
|
12
|
+
// `.`, and `-` as separators, and also match a bare token on its own. The
|
|
13
|
+
// trailing boundary tolerates a plural `s` so `tokens`/`secrets` match too.
|
|
14
|
+
// camelCase is handled by normalizing the name BEFORE testing (see
|
|
15
|
+
// `splitCamelCase`), not by the boundaries themselves.
|
|
13
16
|
const B = '(?:^|[_.\\-])'; // boundary before
|
|
14
|
-
const A = '(?:$|[_.\\-])'; // boundary after
|
|
17
|
+
const A = 's?(?:$|[_.\\-])'; // boundary after (optional plural)
|
|
18
|
+
/**
|
|
19
|
+
* Insert `_` at lower/digit→UPPER transitions so the separator-based patterns
|
|
20
|
+
* above see camelCase the same way they see snake_case. Without this,
|
|
21
|
+
* `passwordHash` and `clientSecret` — the *normal* shape for AdonisJS config
|
|
22
|
+
* keys and camelCase column strategies — sail past every pattern while their
|
|
23
|
+
* snake_case twins are redacted.
|
|
24
|
+
*/
|
|
25
|
+
function splitCamelCase(name) {
|
|
26
|
+
return name.replace(/([a-z0-9])([A-Z])/g, '$1_$2');
|
|
27
|
+
}
|
|
15
28
|
/**
|
|
16
29
|
* Names that identify a credential — applies to env vars, config keys, and SQL
|
|
17
30
|
* identifiers alike.
|
|
@@ -23,6 +36,9 @@ export const SECRET_NAME_PATTERNS = [
|
|
|
23
36
|
new RegExp(`${B}credential${A}`, 'i'),
|
|
24
37
|
new RegExp(`${B}private${A}`, 'i'),
|
|
25
38
|
new RegExp(`${B}auth${A}`, 'i'),
|
|
39
|
+
// HTTP credential carriers — log records embed request headers
|
|
40
|
+
new RegExp(`${B}authorization${A}`, 'i'),
|
|
41
|
+
new RegExp(`${B}cookie${A}`, 'i'),
|
|
26
42
|
// API keys: `api_key`, `apiKey`, `API_KEY`
|
|
27
43
|
/api[_-]?key/i,
|
|
28
44
|
// `_KEY` at end or `_KEY_` in middle (AWS_ACCESS_KEY_ID, ENCRYPTION_KEY, etc.)
|
|
@@ -41,8 +57,18 @@ export const SECRET_NAME_PATTERNS = [
|
|
|
41
57
|
new RegExp(`${B}encryption${A}`, 'i'),
|
|
42
58
|
// App key / app secret
|
|
43
59
|
/app[_-]key/i,
|
|
44
|
-
// One-time codes
|
|
60
|
+
// One-time codes and second factors
|
|
45
61
|
new RegExp(`${B}otp${A}`, 'i'),
|
|
62
|
+
new RegExp(`${B}totp${A}`, 'i'),
|
|
63
|
+
new RegExp(`${B}mfa${A}`, 'i'),
|
|
64
|
+
// Password abbreviations
|
|
65
|
+
new RegExp(`${B}passwd${A}`, 'i'),
|
|
66
|
+
new RegExp(`${B}pwd${A}`, 'i'),
|
|
67
|
+
// Card / identity numbers
|
|
68
|
+
new RegExp(`${B}cvv${A}`, 'i'),
|
|
69
|
+
new RegExp(`${B}cvc${A}`, 'i'),
|
|
70
|
+
new RegExp(`${B}pin${A}`, 'i'),
|
|
71
|
+
new RegExp(`${B}ssn${A}`, 'i'),
|
|
46
72
|
];
|
|
47
73
|
/**
|
|
48
74
|
* Names that matter for env vars and config keys but NOT for SQL identifiers.
|
|
@@ -60,11 +86,15 @@ export const CONFIG_ONLY_NAME_PATTERNS = [
|
|
|
60
86
|
];
|
|
61
87
|
/** Whether a name identifies a credential. */
|
|
62
88
|
export function isSecretName(name) {
|
|
63
|
-
|
|
89
|
+
const normalized = splitCamelCase(name);
|
|
90
|
+
return SECRET_NAME_PATTERNS.some((pattern) => pattern.test(normalized));
|
|
64
91
|
}
|
|
65
92
|
/** Whether a name is sensitive in a config/env context (credentials plus contact/service names). */
|
|
66
93
|
export function isSensitiveConfigName(name) {
|
|
67
|
-
|
|
94
|
+
if (isSecretName(name))
|
|
95
|
+
return true;
|
|
96
|
+
const normalized = splitCamelCase(name);
|
|
97
|
+
return CONFIG_ONLY_NAME_PATTERNS.some((pattern) => pattern.test(normalized));
|
|
68
98
|
}
|
|
69
99
|
/** Identifier-ish tokens in a SQL statement: table names, column names, aliases. */
|
|
70
100
|
const SQL_IDENTIFIER_RE = /[A-Za-z_][A-Za-z0-9_$]*/g;
|
|
@@ -116,3 +146,114 @@ export function looksLikeCredentialValue(value) {
|
|
|
116
146
|
return true;
|
|
117
147
|
return CREDENTIAL_VALUE_PATTERNS.some((pattern) => pattern.test(value));
|
|
118
148
|
}
|
|
149
|
+
// ---------------------------------------------------------------------------
|
|
150
|
+
// Binding hygiene
|
|
151
|
+
// ---------------------------------------------------------------------------
|
|
152
|
+
/** Max length for a stored string binding before it is truncated. */
|
|
153
|
+
const MAX_BINDING_LEN = 256;
|
|
154
|
+
/** Placeholder stored in place of a binding that looks like a credential. */
|
|
155
|
+
const REDACTED_BINDING = '[redacted]';
|
|
156
|
+
/**
|
|
157
|
+
* Redact and truncate SQL bindings before persistence.
|
|
158
|
+
*
|
|
159
|
+
* Two independent rules, because neither catches everything on its own:
|
|
160
|
+
*
|
|
161
|
+
* 1. **By statement.** If the SQL mentions a credential-shaped identifier
|
|
162
|
+
* (`password`, `remember_token`, `otp`, ...), every binding for that
|
|
163
|
+
* statement is redacted. Positional bindings cannot be mapped back to
|
|
164
|
+
* columns reliably, so this is all-or-nothing per statement — coarse, but
|
|
165
|
+
* it is the only thing that catches a short secret like a 6-digit OTP.
|
|
166
|
+
* 2. **By value shape.** Hashes, JWTs, provider key prefixes, long hex
|
|
167
|
+
* digests, and URLs with embedded credentials are redacted wherever they
|
|
168
|
+
* appear, regardless of the statement.
|
|
169
|
+
*
|
|
170
|
+
* Anything that survives both is truncated at {@link MAX_BINDING_LEN} so an
|
|
171
|
+
* oversized payload cannot bloat the row.
|
|
172
|
+
*
|
|
173
|
+
* Ordinary parameters — ids, flags, emails, timestamps — still pass through, so
|
|
174
|
+
* the query pane stays useful for debugging.
|
|
175
|
+
*/
|
|
176
|
+
export function sanitizeBindings(bindings, sqlText) {
|
|
177
|
+
const redactAll = sqlText !== undefined && sqlMentionsSecret(sqlText);
|
|
178
|
+
return sanitizeBindingValue(bindings, redactAll, true);
|
|
179
|
+
}
|
|
180
|
+
/**
|
|
181
|
+
* Redact a structured record (a pino log entry, an event payload) with the
|
|
182
|
+
* same key/shape rules as bindings but WITHOUT length truncation — log data
|
|
183
|
+
* carries stack traces and payload context that must stay whole.
|
|
184
|
+
*/
|
|
185
|
+
export function sanitizeRecordValues(record) {
|
|
186
|
+
return sanitizeBindingValue(record, false, false);
|
|
187
|
+
}
|
|
188
|
+
function sanitizeBindingValue(value, redactAll, truncate) {
|
|
189
|
+
if (Array.isArray(value))
|
|
190
|
+
return value.map((v) => sanitizeBindingValue(v, redactAll, truncate));
|
|
191
|
+
if (value !== null && typeof value === 'object') {
|
|
192
|
+
return sanitizeNamedBindings(value, redactAll, truncate);
|
|
193
|
+
}
|
|
194
|
+
if (typeof value === 'string')
|
|
195
|
+
return sanitizeStringBinding(value, redactAll, truncate);
|
|
196
|
+
// A statement touching a secret column may bind it as a non-string — a numeric
|
|
197
|
+
// OTP, for instance — so redact those too rather than only masking strings.
|
|
198
|
+
// Booleans and null carry nothing worth hiding.
|
|
199
|
+
const redactable = value !== null && value !== undefined && typeof value !== 'boolean';
|
|
200
|
+
return redactAll && redactable ? REDACTED_BINDING : value;
|
|
201
|
+
}
|
|
202
|
+
/** Named bindings carry their own key, so use it when it is telling. */
|
|
203
|
+
function sanitizeNamedBindings(value, redactAll, truncate) {
|
|
204
|
+
const out = {};
|
|
205
|
+
for (const [key, nested] of Object.entries(value)) {
|
|
206
|
+
out[key] = sanitizeBindingValue(nested, redactAll || isSecretName(key), truncate);
|
|
207
|
+
}
|
|
208
|
+
return out;
|
|
209
|
+
}
|
|
210
|
+
/** Matches a value this module already truncated, so a second pass is a no-op. */
|
|
211
|
+
const TRUNCATION_MARKER_RE = /…\[truncated \d+ chars\]$/;
|
|
212
|
+
function sanitizeStringBinding(value, redactAll, truncate) {
|
|
213
|
+
if (redactAll || looksLikeCredentialValue(value))
|
|
214
|
+
return REDACTED_BINDING;
|
|
215
|
+
if (truncate && value.length > MAX_BINDING_LEN && !TRUNCATION_MARKER_RE.test(value)) {
|
|
216
|
+
return value.slice(0, MAX_BINDING_LEN) + `…[truncated ${value.length} chars]`;
|
|
217
|
+
}
|
|
218
|
+
return value;
|
|
219
|
+
}
|
|
220
|
+
// ---------------------------------------------------------------------------
|
|
221
|
+
// URL hygiene
|
|
222
|
+
// ---------------------------------------------------------------------------
|
|
223
|
+
/**
|
|
224
|
+
* Redact credential-bearing query parameters from a URL before it is stored.
|
|
225
|
+
*
|
|
226
|
+
* Request URLs are persisted with their query strings (useful for debugging
|
|
227
|
+
* pagination and filters), but a `?token=…` password-reset link or a
|
|
228
|
+
* `?signature=…` signed URL must not land in SQLite in the clear. Parameters
|
|
229
|
+
* are redacted by name (see {@link isSecretName}) or when the value itself
|
|
230
|
+
* looks like a credential; everything else passes through untouched.
|
|
231
|
+
*/
|
|
232
|
+
export function sanitizeUrlQuery(url) {
|
|
233
|
+
const qIdx = url.indexOf('?');
|
|
234
|
+
if (qIdx === -1)
|
|
235
|
+
return url;
|
|
236
|
+
const sanitized = url
|
|
237
|
+
.slice(qIdx + 1)
|
|
238
|
+
.split('&')
|
|
239
|
+
.map((pair) => {
|
|
240
|
+
if (pair === '')
|
|
241
|
+
return pair;
|
|
242
|
+
const eq = pair.indexOf('=');
|
|
243
|
+
const name = eq === -1 ? pair : pair.slice(0, eq);
|
|
244
|
+
const rawValue = eq === -1 ? '' : pair.slice(eq + 1);
|
|
245
|
+
let value = rawValue;
|
|
246
|
+
try {
|
|
247
|
+
value = decodeURIComponent(rawValue);
|
|
248
|
+
}
|
|
249
|
+
catch {
|
|
250
|
+
// Malformed escape — judge the raw text instead.
|
|
251
|
+
}
|
|
252
|
+
if (isSecretName(name) || (value !== '' && looksLikeCredentialValue(value))) {
|
|
253
|
+
return `${name}=[redacted]`;
|
|
254
|
+
}
|
|
255
|
+
return pair;
|
|
256
|
+
})
|
|
257
|
+
.join('&');
|
|
258
|
+
return url.slice(0, qIdx) + '?' + sanitized;
|
|
259
|
+
}
|
|
@@ -20,18 +20,23 @@ export interface StorageStatsResult {
|
|
|
20
20
|
fileSizeMb: number;
|
|
21
21
|
walSizeMb: number;
|
|
22
22
|
retentionDays: number;
|
|
23
|
+
/** Configured live-data size cap in MB; `0` means the cap is disabled. */
|
|
24
|
+
maxDbSizeMb: number;
|
|
23
25
|
tables: Array<{
|
|
24
26
|
name: string;
|
|
25
27
|
rowCount: number;
|
|
26
28
|
}>;
|
|
27
29
|
lastCleanupAt: number | null;
|
|
28
30
|
}
|
|
31
|
+
/** The not-ready result returned before the SQLite connection exists. */
|
|
32
|
+
export declare function emptyStorageStats(dbPath: string, retentionDays: number, maxDbSizeMb: number): StorageStatsResult;
|
|
29
33
|
export interface StorageStatsOpts {
|
|
30
34
|
db: Knex;
|
|
31
35
|
cache: CoalesceCache;
|
|
32
36
|
dbFilePath: string;
|
|
33
37
|
dbPath: string;
|
|
34
38
|
retentionDays: number;
|
|
39
|
+
maxDbSizeMb: number;
|
|
35
40
|
lastCleanupAt: number | null;
|
|
36
41
|
onResult: (stats: StorageStatsResult) => void;
|
|
37
42
|
}
|
|
@@ -58,11 +58,24 @@ export async function countAllTables(db) {
|
|
|
58
58
|
return result;
|
|
59
59
|
});
|
|
60
60
|
}
|
|
61
|
+
/** The not-ready result returned before the SQLite connection exists. */
|
|
62
|
+
export function emptyStorageStats(dbPath, retentionDays, maxDbSizeMb) {
|
|
63
|
+
return {
|
|
64
|
+
ready: false,
|
|
65
|
+
dbPath,
|
|
66
|
+
fileSizeMb: 0,
|
|
67
|
+
walSizeMb: 0,
|
|
68
|
+
retentionDays,
|
|
69
|
+
maxDbSizeMb,
|
|
70
|
+
tables: [],
|
|
71
|
+
lastCleanupAt: null,
|
|
72
|
+
};
|
|
73
|
+
}
|
|
61
74
|
/**
|
|
62
75
|
* Build storage stats by querying file sizes and table row counts.
|
|
63
76
|
*/
|
|
64
77
|
export function fetchStorageStats(opts) {
|
|
65
|
-
const { db, cache, dbFilePath, dbPath, retentionDays, lastCleanupAt, onResult } = opts;
|
|
78
|
+
const { db, cache, dbFilePath, dbPath, retentionDays, maxDbSizeMb, lastCleanupAt, onResult } = opts;
|
|
66
79
|
return cache.coalesce('storageStats', async () => {
|
|
67
80
|
const [fileSizeMb, walSizeMb] = await getFileSizes(dbFilePath);
|
|
68
81
|
const tables = await countAllTables(db);
|
|
@@ -72,6 +85,7 @@ export function fetchStorageStats(opts) {
|
|
|
72
85
|
fileSizeMb,
|
|
73
86
|
walSizeMb,
|
|
74
87
|
retentionDays,
|
|
88
|
+
maxDbSizeMb,
|
|
75
89
|
tables,
|
|
76
90
|
lastCleanupAt,
|
|
77
91
|
};
|
|
@@ -5,11 +5,18 @@
|
|
|
5
5
|
* objects. They are pure (no I/O, no Knex dependency) so they can
|
|
6
6
|
* be tested in isolation.
|
|
7
7
|
*/
|
|
8
|
+
import { sanitizeBindings } from './sensitive_patterns.js';
|
|
8
9
|
import type { EventRecord, EmailRecord } from '../debug/types.js';
|
|
9
10
|
import type { PersistRequestInput } from './dashboard_types.js';
|
|
10
11
|
import type { Knex } from 'knex';
|
|
11
12
|
export declare function hasWarned(path: string): boolean;
|
|
12
13
|
export declare function markWarned(path: string): void;
|
|
14
|
+
/**
|
|
15
|
+
* Forget which write paths have warned. Called when a DashboardStore stops so
|
|
16
|
+
* a re-initialized store in the same process warns afresh instead of
|
|
17
|
+
* inheriting a spent latch from its predecessor.
|
|
18
|
+
*/
|
|
19
|
+
export declare function resetWriteWarnings(): void;
|
|
13
20
|
/**
|
|
14
21
|
* Normalize a SQL query by replacing literal values with `?` placeholders.
|
|
15
22
|
* Used for grouping identical query patterns.
|
|
@@ -20,27 +27,7 @@ export declare function markWarned(path: string): void;
|
|
|
20
27
|
* merged together.
|
|
21
28
|
*/
|
|
22
29
|
export declare function normalizeSql(sql: string): string;
|
|
23
|
-
|
|
24
|
-
* Redact and truncate SQL bindings before persistence.
|
|
25
|
-
*
|
|
26
|
-
* Two independent rules, because neither catches everything on its own:
|
|
27
|
-
*
|
|
28
|
-
* 1. **By statement.** If the SQL mentions a credential-shaped identifier
|
|
29
|
-
* (`password`, `remember_token`, `otp`, ...), every binding for that
|
|
30
|
-
* statement is redacted. Positional bindings cannot be mapped back to
|
|
31
|
-
* columns reliably, so this is all-or-nothing per statement — coarse, but
|
|
32
|
-
* it is the only thing that catches a short secret like a 6-digit OTP.
|
|
33
|
-
* 2. **By value shape.** Hashes, JWTs, provider key prefixes, long hex
|
|
34
|
-
* digests, and URLs with embedded credentials are redacted wherever they
|
|
35
|
-
* appear, regardless of the statement.
|
|
36
|
-
*
|
|
37
|
-
* Anything that survives both is truncated at {@link MAX_BINDING_LEN} so an
|
|
38
|
-
* oversized payload cannot bloat the row.
|
|
39
|
-
*
|
|
40
|
-
* Ordinary parameters — ids, flags, emails, timestamps — still pass through, so
|
|
41
|
-
* the query pane stays useful for debugging.
|
|
42
|
-
*/
|
|
43
|
-
export declare function sanitizeBindings(bindings: unknown, sqlText?: string): unknown;
|
|
30
|
+
export { sanitizeBindings };
|
|
44
31
|
export interface PreparedQuery {
|
|
45
32
|
sql_text: string;
|
|
46
33
|
sql_normalized: string;
|
|
@@ -100,6 +87,13 @@ export interface EventRow {
|
|
|
100
87
|
export declare function prepareRequestRows(requests: PersistRequestInput[]): PreparedRequest[];
|
|
101
88
|
/**
|
|
102
89
|
* Transform raw log entries into SQLite-ready row objects.
|
|
90
|
+
*
|
|
91
|
+
* The full pino record lands in `data`, and log lines routinely carry bound
|
|
92
|
+
* request context — authorization headers, cookies, tokens in payloads — so
|
|
93
|
+
* the record is run through the shared redaction rules (secret-named keys and
|
|
94
|
+
* credential-shaped values) before it is stringified. The `message` column is
|
|
95
|
+
* kept verbatim: it is free text, and the shape rules are anchored to whole
|
|
96
|
+
* values, so prose stays readable.
|
|
103
97
|
*/
|
|
104
98
|
export declare function prepareLogRows(logs: Record<string, unknown>[]): PreparedLog[];
|
|
105
99
|
/**
|
|
@@ -118,6 +112,11 @@ export declare function buildEmailRow(record: EmailRecord): EmailRow;
|
|
|
118
112
|
export declare function buildEventRows(events: EventRecord[]): EventRow[];
|
|
119
113
|
/**
|
|
120
114
|
* Insert rows into a table in batches of 50.
|
|
115
|
+
*
|
|
116
|
+
* Yields to the event loop between chunks: better-sqlite3 is fully
|
|
117
|
+
* synchronous, so without a real macrotask hop a large batch would execute
|
|
118
|
+
* back-to-back on microtask continuations and stall the host app — the exact
|
|
119
|
+
* failure mode `migrator_tables.ts` documents and guards against.
|
|
121
120
|
*/
|
|
122
121
|
export declare function batchInsert(trx: Knex.Transaction, table: string, rows: Record<string, unknown>[]): Promise<void>;
|
|
123
122
|
export declare function flushRequests(trx: Knex.Transaction, preparedRequests: PreparedRequest[]): Promise<void>;
|