adonisjs-server-stats 1.16.1 → 1.18.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. package/README.md +25 -8
  2. package/dist/core/core/dashboard-api.d.ts +6 -4
  3. package/dist/core/core/define-config-helpers.d.ts +1 -0
  4. package/dist/core/core/types-diagnostics.d.ts +2 -0
  5. package/dist/core/debug/types.d.ts +2 -0
  6. package/dist/core/index.js +9 -6
  7. package/dist/core/types.d.ts +55 -4
  8. package/dist/react/{CacheSection-DkLtukWz.js → CacheSection-B-lSIN8Q.js} +48 -44
  9. package/dist/react/{CacheTab-DWeXnEBI.js → CacheTab-BN0QgPXz.js} +1 -1
  10. package/dist/react/{ConfigSection-DaVbFPDg.js → ConfigSection-u_TSfb9f.js} +1 -1
  11. package/dist/react/{ConfigTab-D4640WJZ.js → ConfigTab-au26s-ON.js} +1 -1
  12. package/dist/react/{CustomPaneTab-DglmAVMC.js → CustomPaneTab-qvMnh8ss.js} +1 -1
  13. package/dist/react/{EmailsSection-vYx6ExTb.js → EmailsSection-HfEMeRex.js} +65 -57
  14. package/dist/react/{EmailsTab-DKDrRmRH.js → EmailsTab-BWwq-CxB.js} +1 -1
  15. package/dist/react/{EventsSection-CapH5xut.js → EventsSection-CA9b15dJ.js} +1 -1
  16. package/dist/react/{EventsTab-BNVEySAm.js → EventsTab-CgSdHbKU.js} +1 -1
  17. package/dist/react/{InternalsContent-C9lIA92C.js → InternalsContent-CI7HvBtH.js} +51 -47
  18. package/dist/react/{InternalsSection-lOv_UcSV.js → InternalsSection-IEeWXJqP.js} +1 -1
  19. package/dist/react/{InternalsTab-BygoSorC.js → InternalsTab-CoVLHQbF.js} +1 -1
  20. package/dist/react/{JobsSection-ZIQsJWid.js → JobsSection-DgsS_VwU.js} +1 -1
  21. package/dist/react/{JobsTab-DXQzXrDt.js → JobsTab-CzwLSRUC.js} +1 -1
  22. package/dist/react/{LogEntryRow-BWkHE51-.js → LogEntryRow-VCCzwD1H.js} +1 -1
  23. package/dist/react/{LogsSection-ConXdBkL.js → LogsSection-BujNn_cO.js} +2 -2
  24. package/dist/react/{LogsTab-CJM47LPn.js → LogsTab-BtD1h7DM.js} +2 -2
  25. package/dist/react/{OverviewSection-DOMu2qvl.js → OverviewSection-B28em0gQ.js} +119 -111
  26. package/dist/react/{QueriesSection-DFsaOSJI.js → QueriesSection-B0tFn_Dc.js} +1 -1
  27. package/dist/react/{QueriesTab-CY9CG_7L.js → QueriesTab-DJY7aavR.js} +1 -1
  28. package/dist/react/{RequestsSection-CC-eVHsl.js → RequestsSection-DPmYM7tq.js} +2 -2
  29. package/dist/react/{RoutesSection-p1DMq41y.js → RoutesSection-Ck-RcFVS.js} +1 -1
  30. package/dist/react/{RoutesTab-Cnqy8UcK.js → RoutesTab-Ch3PDLzo.js} +1 -1
  31. package/dist/react/{SplitPaneWrapper-XgkA0QxE.js → SplitPaneWrapper-BZ6VrBr-.js} +1 -1
  32. package/dist/react/{TimelineTab-1YOERxe5.js → TimelineTab-_byxh2mJ.js} +2 -2
  33. package/dist/react/components/Dashboard/overview-data-context.d.ts +15 -0
  34. package/dist/react/components/Dashboard/sections/OverviewSection.d.ts +9 -0
  35. package/dist/react/{index-DOSlCpZ9.js → index-De7nsWBN.js} +148 -141
  36. package/dist/react/index.js +1 -1
  37. package/dist/src/collectors/app_collector.d.ts +11 -0
  38. package/dist/src/collectors/app_collector.js +11 -0
  39. package/dist/src/collectors/redis_collector.js +13 -13
  40. package/dist/src/dashboard/cache_handlers.d.ts +17 -3
  41. package/dist/src/dashboard/cache_handlers.js +14 -19
  42. package/dist/src/dashboard/coalesce_cache.d.ts +2 -8
  43. package/dist/src/dashboard/coalesce_cache.js +25 -0
  44. package/dist/src/dashboard/dashboard_controller.d.ts +1 -0
  45. package/dist/src/dashboard/dashboard_controller.js +8 -3
  46. package/dist/src/dashboard/dashboard_store.d.ts +0 -1
  47. package/dist/src/dashboard/dashboard_store.js +18 -18
  48. package/dist/src/dashboard/filter_handlers.js +26 -4
  49. package/dist/src/dashboard/flush_manager.d.ts +1 -0
  50. package/dist/src/dashboard/flush_manager.js +6 -1
  51. package/dist/src/dashboard/format_helpers.d.ts +0 -4
  52. package/dist/src/dashboard/format_helpers.js +0 -19
  53. package/dist/src/dashboard/integrations/config_inspector.js +6 -3
  54. package/dist/src/dashboard/knex_factory.js +8 -0
  55. package/dist/src/dashboard/migrator.d.ts +7 -2
  56. package/dist/src/dashboard/migrator.js +140 -6
  57. package/dist/src/dashboard/migrator_tables.js +8 -0
  58. package/dist/src/dashboard/paginate_helper.js +21 -11
  59. package/dist/src/dashboard/sensitive_patterns.d.ts +37 -0
  60. package/dist/src/dashboard/sensitive_patterns.js +146 -5
  61. package/dist/src/dashboard/storage_stats.d.ts +5 -0
  62. package/dist/src/dashboard/storage_stats.js +15 -1
  63. package/dist/src/dashboard/write_queue.d.ts +20 -21
  64. package/dist/src/dashboard/write_queue.js +37 -60
  65. package/dist/src/debug/query_collector.js +6 -1
  66. package/dist/src/debug/trace_collector.js +26 -13
  67. package/dist/src/debug/types.d.ts +2 -0
  68. package/dist/src/define_config.js +2 -0
  69. package/dist/src/edge/client/dashboard.js +2 -2
  70. package/dist/src/edge/client/debug-panel-deferred.js +1 -1
  71. package/dist/src/edge/client-vue/dashboard.js +5 -5
  72. package/dist/src/edge/client-vue/debug-panel-deferred.js +2 -2
  73. package/dist/src/edge/views/dashboard.edge +1 -1
  74. package/dist/src/edge/views/debug-panel.edge +1 -1
  75. package/dist/src/edge/views/stats-bar.edge +1 -1
  76. package/dist/src/middleware/request_tracking_middleware.js +7 -2
  77. package/dist/src/provider/dashboard_init.d.ts +9 -1
  78. package/dist/src/provider/dashboard_init.js +10 -0
  79. package/dist/src/provider/dashboard_setup.js +14 -0
  80. package/dist/src/provider/diagnostics.d.ts +2 -0
  81. package/dist/src/provider/diagnostics.js +1 -0
  82. package/dist/src/provider/email_helpers.js +20 -5
  83. package/dist/src/provider/server_stats_provider.d.ts +10 -0
  84. package/dist/src/provider/server_stats_provider.js +26 -7
  85. package/dist/src/provider/shutdown_helpers.js +3 -1
  86. package/dist/src/provider/toolbar_setup.js +21 -15
  87. package/dist/src/stubs/config.stub +4 -0
  88. package/dist/src/types.d.ts +55 -4
  89. package/dist/vue/CacheSection-z2TQRez9.js +161 -0
  90. package/dist/vue/{ConfigSection-BiRAiaHj.js → ConfigSection-CFEaig69.js} +1 -1
  91. package/dist/vue/{EmailsSection-Dl44qyqY.js → EmailsSection-CYohBo62.js} +89 -85
  92. package/dist/vue/{EventsSection-CWjeitjU.js → EventsSection-B_TuVrGj.js} +1 -1
  93. package/dist/vue/{InternalsSection-jhTCuK3j.js → InternalsSection-B1P2kNT-.js} +92 -84
  94. package/dist/vue/{InternalsTab-Z3c82glB.js → InternalsTab-CRI-IFTo.js} +155 -147
  95. package/dist/vue/{JobsSection-DOBb4LjZ.js → JobsSection-AjkEzyiv.js} +1 -1
  96. package/dist/vue/{LogsSection-CXx-HOWJ.js → LogsSection-DZD0CCDi.js} +1 -1
  97. package/dist/vue/{OverviewSection-CyfNQ8uV.js → OverviewSection-BEc4Gktf.js} +1 -1
  98. package/dist/vue/{QueriesSection-CXBsFp-y.js → QueriesSection-BMImGXTn.js} +1 -1
  99. package/dist/vue/{RequestsSection-YTIaZGZd.js → RequestsSection-DOI2MUFX.js} +1 -1
  100. package/dist/vue/{RoutesSection-CDKMey49.js → RoutesSection-C1sxt-AW.js} +1 -1
  101. package/dist/vue/composables/useDashboardData.d.ts +1 -1
  102. package/dist/vue/{index-14x39RI_.js → index-BfPAAjL5.js} +95 -95
  103. package/dist/vue/index.js +1 -1
  104. package/package.json +12 -3
  105. package/dist/src/dashboard/explain_query.d.ts +0 -8
  106. package/dist/src/dashboard/explain_query.js +0 -22
  107. package/dist/src/provider/boot_initializer.d.ts +0 -28
  108. package/dist/src/provider/boot_initializer.js +0 -35
  109. package/dist/vue/CacheSection-CxEBVVkF.js +0 -156
@@ -27,15 +27,20 @@ export async function autoMigrate(db) {
27
27
  await migrateSavedFilters(db);
28
28
  }
29
29
  /**
30
- * Delete records older than `retentionDays` from all tables.
30
+ * Delete records older than `retentionDays` from all tables, then enforce
31
+ * the database size cap.
31
32
  *
32
33
  * Foreign-key cascades on `server_stats_requests` handle the child
33
34
  * tables (queries, events, traces). Standalone tables (logs, emails,
34
35
  * metrics, saved_filters) are pruned individually.
35
36
  *
37
+ * `maxDbSizeMb` bounds the *live* data size: when exceeded, the globally
38
+ * oldest rows are deleted — regardless of age — until usage drops below
39
+ * ~90% of the cap. `0` disables the cap.
40
+ *
36
41
  * Yields between each DELETE so the event loop stays responsive.
37
42
  */
38
- export async function runRetentionCleanup(db, retentionDays) {
43
+ export async function runRetentionCleanup(db, retentionDays, maxDbSizeMb = 0) {
39
44
  // Use string interpolation instead of parameterized bindings.
40
45
  // Knex + better-sqlite3 can hang on parameterized db.raw() calls,
41
46
  // while non-parameterized queries (used in migrations) work fine.
@@ -53,7 +58,12 @@ export async function runRetentionCleanup(db, retentionDays) {
53
58
  await yieldToEventLoop();
54
59
  await batchDelete(db, 'server_stats_metrics', cutoff);
55
60
  await yieldToEventLoop();
56
- // Reclaim space and update query planner statistics
61
+ if (maxDbSizeMb > 0)
62
+ await enforceSizeCap(db, maxDbSizeMb);
63
+ // Return deleted pages to the OS — PRAGMA optimize alone only updates
64
+ // planner stats, leaving the file at its high-water mark forever.
65
+ await reclaimFreeSpace(db);
66
+ // Update query planner statistics
57
67
  await db.raw('PRAGMA optimize');
58
68
  }
59
69
  catch (err) {
@@ -72,10 +82,134 @@ async function batchDelete(db, table, cutoff) {
72
82
  let hasMore = true;
73
83
  while (hasMore) {
74
84
  await db.raw(`DELETE FROM ${table} WHERE rowid IN (SELECT rowid FROM ${table} WHERE created_at < ${cutoff} LIMIT 1000)`);
75
- const remaining = await db.raw(`SELECT COUNT(*) as cnt FROM ${table} WHERE created_at < ${cutoff} LIMIT 1`);
76
- const cnt = remaining?.[0]?.cnt ?? 0;
77
- hasMore = cnt > 0;
85
+ // changes() reports the last DELETE's row count on this connection
86
+ // (pool is min:1/max:1). Re-counting the remaining backlog here instead
87
+ // made total work quadratic in backlog size.
88
+ const deleted = await pragmaNumber(db, 'SELECT changes() AS n');
89
+ hasMore = deleted === 1000;
78
90
  if (hasMore)
79
91
  await yieldToEventLoop();
80
92
  }
81
93
  }
94
+ /** Root tables pruned by the size cap; requests cascades to queries/events/traces. */
95
+ const SIZE_CAP_TABLES = [
96
+ 'server_stats_requests',
97
+ 'server_stats_logs',
98
+ 'server_stats_emails',
99
+ 'server_stats_metrics',
100
+ ];
101
+ /**
102
+ * Bytes of live data: total pages minus freelist pages. Freed pages are
103
+ * excluded because reclaimFreeSpace returns them to the OS afterwards.
104
+ */
105
+ async function liveSizeBytes(db, pageSize) {
106
+ const pages = await pragmaNumber(db, 'PRAGMA page_count');
107
+ const freelist = await pragmaNumber(db, 'PRAGMA freelist_count');
108
+ return (pages - freelist) * pageSize;
109
+ }
110
+ /**
111
+ * Keep live data under `maxSizeMb` by deleting the globally oldest rows,
112
+ * whatever their age — a size cap effectively shortens the retention
113
+ * window when write volume outruns `retentionDays`.
114
+ *
115
+ * Prunes down to ~90% of the cap so the hourly cleanup doesn't re-trigger
116
+ * on every pass. Each round deletes 1000 rows from whichever root table
117
+ * currently holds the oldest data, so mixed workloads (trace-heavy vs
118
+ * log-heavy) shed their actual oldest history first.
119
+ */
120
+ async function enforceSizeCap(db, maxSizeMb) {
121
+ const budget = maxSizeMb * 1024 * 1024;
122
+ const pageSize = await pragmaNumber(db, 'PRAGMA page_size');
123
+ if ((await liveSizeBytes(db, pageSize)) <= budget)
124
+ return;
125
+ const { log } = await import('../utils/logger.js');
126
+ log.info(`dashboard: database over the ${maxSizeMb} MB size cap — pruning oldest records`);
127
+ const target = budget * 0.9;
128
+ // 10k rounds × 1000 rows bounds one pass; the next hourly pass continues.
129
+ let guard = 10_000;
130
+ while (guard-- > 0 && (await liveSizeBytes(db, pageSize)) > target) {
131
+ const table = await findTableWithOldestRow(db);
132
+ if (!table)
133
+ break;
134
+ await db.raw(`DELETE FROM ${table} WHERE rowid IN (SELECT rowid FROM ${table} ORDER BY created_at LIMIT 1000)`);
135
+ const deleted = await pragmaNumber(db, 'SELECT changes() AS n');
136
+ if (deleted === 0)
137
+ break;
138
+ await yieldToEventLoop();
139
+ }
140
+ }
141
+ /** The root table whose oldest row is globally oldest, or null when all are empty. */
142
+ async function findTableWithOldestRow(db) {
143
+ let best = null;
144
+ let bestTs = '';
145
+ for (const table of SIZE_CAP_TABLES) {
146
+ const rows = (await db.raw(`SELECT MIN(created_at) AS ts FROM ${table}`));
147
+ const ts = rows?.[0]?.ts;
148
+ if (ts && (best === null || ts < bestTs)) {
149
+ best = table;
150
+ bestTs = ts;
151
+ }
152
+ }
153
+ return best;
154
+ }
155
+ /**
156
+ * Return freelist pages to the OS after a cleanup pass.
157
+ *
158
+ * Databases created with `auto_vacuum=INCREMENTAL` are trimmed in bounded
159
+ * chunks so each synchronous step stays short. Legacy databases (created
160
+ * before the pragma existed) can't be trimmed incrementally, so when dead
161
+ * pages exceed ~30% of the file a one-time VACUUM rewrites it — which also
162
+ * converts it to incremental mode, since `auto_vacuum=INCREMENTAL` is set
163
+ * on this connection.
164
+ */
165
+ async function reclaimFreeSpace(db) {
166
+ const mode = await pragmaNumber(db, 'PRAGMA auto_vacuum');
167
+ if (mode === 2) {
168
+ // Chunks of ~2000 pages (~8 MB at the 4 KB default) keep each synchronous
169
+ // call short; guard bounds one pass at ~8 GB in case freelist_count
170
+ // misbehaves.
171
+ let guard = 1000;
172
+ while (guard-- > 0) {
173
+ const freelist = await pragmaNumber(db, 'PRAGMA freelist_count');
174
+ if (freelist <= 0)
175
+ break;
176
+ if (!(await incrementalVacuumChunk(db)))
177
+ break;
178
+ await yieldToEventLoop();
179
+ }
180
+ return;
181
+ }
182
+ const pageCount = await pragmaNumber(db, 'PRAGMA page_count');
183
+ const freelistCount = await pragmaNumber(db, 'PRAGMA freelist_count');
184
+ if (pageCount > 0 && freelistCount / pageCount > 0.3) {
185
+ const { log } = await import('../utils/logger.js');
186
+ log.info(`dashboard: reclaiming ${freelistCount} of ${pageCount} pages via one-time VACUUM (may pause briefly)`);
187
+ await db.raw('VACUUM');
188
+ log.info('dashboard: VACUUM complete — database converted to incremental auto-vacuum');
189
+ }
190
+ }
191
+ /**
192
+ * Free up to 2000 freelist pages. incremental_vacuum frees ONE page per
193
+ * sqlite3_step, and knex's raw() steps no-result pragmas exactly once — so
194
+ * it must run on the underlying better-sqlite3 handle, whose .pragma()
195
+ * steps to completion. Returns false if the handle has no pragma method.
196
+ */
197
+ async function incrementalVacuumChunk(db) {
198
+ const client = db.client;
199
+ const conn = (await client.acquireConnection());
200
+ try {
201
+ if (typeof conn.pragma !== 'function')
202
+ return false;
203
+ conn.pragma('incremental_vacuum(2000)');
204
+ return true;
205
+ }
206
+ finally {
207
+ client.releaseConnection(conn);
208
+ }
209
+ }
210
+ /** Run a single-row/single-column statement and return its value as a number. */
211
+ async function pragmaNumber(db, statement) {
212
+ const rows = (await db.raw(statement));
213
+ const value = Object.values(rows?.[0] ?? {})[0];
214
+ return Number(value ?? 0);
215
+ }
@@ -28,6 +28,10 @@ export async function migrateRequests(db) {
28
28
  await db.raw(`CREATE INDEX IF NOT EXISTS idx_ss_requests_created ON server_stats_requests(created_at)`);
29
29
  await db.raw(`CREATE INDEX IF NOT EXISTS idx_ss_requests_url ON server_stats_requests(url)`);
30
30
  await db.raw(`CREATE INDEX IF NOT EXISTS idx_ss_requests_duration ON server_stats_requests(duration)`);
31
+ // Covers the overview p95 query (WHERE created_at >= ? ORDER BY duration):
32
+ // with only the single-column indexes SQLite sorts the whole filtered range
33
+ // on every 2s cache miss.
34
+ await db.raw(`CREATE INDEX IF NOT EXISTS idx_ss_requests_created_duration ON server_stats_requests(created_at, duration)`);
31
35
  await db.raw(`CREATE INDEX IF NOT EXISTS idx_ss_requests_status ON server_stats_requests(status_code)`);
32
36
  try {
33
37
  await db.raw('ALTER TABLE server_stats_requests ADD COLUMN http_request_id TEXT');
@@ -68,6 +72,10 @@ export async function migrateEvents(db) {
68
72
  `);
69
73
  await db.raw(`CREATE INDEX IF NOT EXISTS idx_ss_events_created ON server_stats_events(created_at)`);
70
74
  await db.raw(`CREATE INDEX IF NOT EXISTS idx_ss_events_name ON server_stats_events(event_name)`);
75
+ // Without this index every cascaded delete from server_stats_requests
76
+ // full-scans the events table (synchronously, on the host event loop) —
77
+ // the other FK children (queries, traces, logs) all have one.
78
+ await db.raw(`CREATE INDEX IF NOT EXISTS idx_ss_events_request ON server_stats_events(request_id)`);
71
79
  }
72
80
  export async function migrateEmails(db) {
73
81
  await db.raw(`
@@ -15,31 +15,41 @@ export function clampPerPage(value) {
15
15
  return 25;
16
16
  return clamp(Math.trunc(n), 1, 200);
17
17
  }
18
+ /**
19
+ * Rows the COUNT is allowed to scan. Filters include leading-wildcard LIKEs
20
+ * that no index can serve, so an uncapped COUNT walks the entire table twice
21
+ * per request (count + data). Past this many matches the exact total stops
22
+ * being interesting — the UI just needs "lots of pages".
23
+ */
24
+ const COUNT_SCAN_CAP = 10_000;
18
25
  /**
19
26
  * Execute a paginated query within a transaction.
20
27
  */
21
28
  export async function executePaginate(db, opts) {
22
29
  const perPage = clampPerPage(opts.perPage);
23
30
  return db.transaction(async (trx) => {
24
- const countQuery = trx(opts.table);
31
+ const countSub = trx(opts.table).select(trx.raw('1')).limit(COUNT_SCAN_CAP);
25
32
  if (opts.applyFilters)
26
- opts.applyFilters(countQuery);
27
- const [{ count: totalRaw }] = await countQuery.count('* as count');
28
- const total = Number(totalRaw);
29
- const offset = (opts.page - 1) * perPage;
30
- const dataQuery = trx(opts.table)
31
- .orderBy('created_at', 'desc')
32
- .limit(perPage)
33
- .offset(offset);
33
+ opts.applyFilters(countSub);
34
+ const countRows = (await trx.count('* as count').from(countSub.as('t')));
35
+ const total = Number(countRows[0]?.count ?? 0);
36
+ // Clamp the page into the real range: an arbitrary ?page= forces a
37
+ // full-scan-sized OFFSET on the single sqlite connection, blocking every
38
+ // other dashboard read for nothing.
39
+ const lastPage = Math.ceil(total / perPage);
40
+ const requested = Number.isFinite(Number(opts.page)) ? Math.trunc(Number(opts.page)) : 1;
41
+ const page = clamp(requested, 1, Math.max(1, lastPage));
42
+ const offset = (page - 1) * perPage;
43
+ const dataQuery = trx(opts.table).orderBy('created_at', 'desc').limit(perPage).offset(offset);
34
44
  if (opts.applyFilters)
35
45
  opts.applyFilters(dataQuery);
36
46
  const data = await dataQuery;
37
47
  return {
38
48
  data,
39
49
  total,
40
- page: opts.page,
50
+ page,
41
51
  perPage,
42
- lastPage: Math.ceil(total / perPage),
52
+ lastPage,
43
53
  };
44
54
  });
45
55
  }
@@ -37,3 +37,40 @@ export declare function isSensitiveConfigName(name: string): boolean;
37
37
  */
38
38
  export declare function sqlMentionsSecret(sql: string): boolean;
39
39
  export declare function looksLikeCredentialValue(value: string): boolean;
40
+ /**
41
+ * Redact and truncate SQL bindings before persistence.
42
+ *
43
+ * Two independent rules, because neither catches everything on its own:
44
+ *
45
+ * 1. **By statement.** If the SQL mentions a credential-shaped identifier
46
+ * (`password`, `remember_token`, `otp`, ...), every binding for that
47
+ * statement is redacted. Positional bindings cannot be mapped back to
48
+ * columns reliably, so this is all-or-nothing per statement — coarse, but
49
+ * it is the only thing that catches a short secret like a 6-digit OTP.
50
+ * 2. **By value shape.** Hashes, JWTs, provider key prefixes, long hex
51
+ * digests, and URLs with embedded credentials are redacted wherever they
52
+ * appear, regardless of the statement.
53
+ *
54
+ * Anything that survives both is truncated at {@link MAX_BINDING_LEN} so an
55
+ * oversized payload cannot bloat the row.
56
+ *
57
+ * Ordinary parameters — ids, flags, emails, timestamps — still pass through, so
58
+ * the query pane stays useful for debugging.
59
+ */
60
+ export declare function sanitizeBindings(bindings: unknown, sqlText?: string): unknown;
61
+ /**
62
+ * Redact a structured record (a pino log entry, an event payload) with the
63
+ * same key/shape rules as bindings but WITHOUT length truncation — log data
64
+ * carries stack traces and payload context that must stay whole.
65
+ */
66
+ export declare function sanitizeRecordValues(record: unknown): unknown;
67
+ /**
68
+ * Redact credential-bearing query parameters from a URL before it is stored.
69
+ *
70
+ * Request URLs are persisted with their query strings (useful for debugging
71
+ * pagination and filters), but a `?token=…` password-reset link or a
72
+ * `?signature=…` signed URL must not land in SQLite in the clear. Parameters
73
+ * are redacted by name (see {@link isSecretName}) or when the value itself
74
+ * looks like a credential; everything else passes through untouched.
75
+ */
76
+ export declare function sanitizeUrlQuery(url: string): string;
@@ -9,9 +9,22 @@
9
9
  */
10
10
  // Custom word boundaries: `\b` does not match between `CLIENT` and `SECRET` in
11
11
  // `GOOGLE_CLIENT_SECRET`, because `_` is a word character. These treat `_`,
12
- // `.`, and `-` as separators, and also match a bare token on its own.
12
+ // `.`, and `-` as separators, and also match a bare token on its own. The
13
+ // trailing boundary tolerates a plural `s` so `tokens`/`secrets` match too.
14
+ // camelCase is handled by normalizing the name BEFORE testing (see
15
+ // `splitCamelCase`), not by the boundaries themselves.
13
16
  const B = '(?:^|[_.\\-])'; // boundary before
14
- const A = '(?:$|[_.\\-])'; // boundary after
17
+ const A = 's?(?:$|[_.\\-])'; // boundary after (optional plural)
18
+ /**
19
+ * Insert `_` at lower/digit→UPPER transitions so the separator-based patterns
20
+ * above see camelCase the same way they see snake_case. Without this,
21
+ * `passwordHash` and `clientSecret` — the *normal* shape for AdonisJS config
22
+ * keys and camelCase column strategies — sail past every pattern while their
23
+ * snake_case twins are redacted.
24
+ */
25
+ function splitCamelCase(name) {
26
+ return name.replace(/([a-z0-9])([A-Z])/g, '$1_$2');
27
+ }
15
28
  /**
16
29
  * Names that identify a credential — applies to env vars, config keys, and SQL
17
30
  * identifiers alike.
@@ -23,6 +36,9 @@ export const SECRET_NAME_PATTERNS = [
23
36
  new RegExp(`${B}credential${A}`, 'i'),
24
37
  new RegExp(`${B}private${A}`, 'i'),
25
38
  new RegExp(`${B}auth${A}`, 'i'),
39
+ // HTTP credential carriers — log records embed request headers
40
+ new RegExp(`${B}authorization${A}`, 'i'),
41
+ new RegExp(`${B}cookie${A}`, 'i'),
26
42
  // API keys: `api_key`, `apiKey`, `API_KEY`
27
43
  /api[_-]?key/i,
28
44
  // `_KEY` at end or `_KEY_` in middle (AWS_ACCESS_KEY_ID, ENCRYPTION_KEY, etc.)
@@ -41,8 +57,18 @@ export const SECRET_NAME_PATTERNS = [
41
57
  new RegExp(`${B}encryption${A}`, 'i'),
42
58
  // App key / app secret
43
59
  /app[_-]key/i,
44
- // One-time codes
60
+ // One-time codes and second factors
45
61
  new RegExp(`${B}otp${A}`, 'i'),
62
+ new RegExp(`${B}totp${A}`, 'i'),
63
+ new RegExp(`${B}mfa${A}`, 'i'),
64
+ // Password abbreviations
65
+ new RegExp(`${B}passwd${A}`, 'i'),
66
+ new RegExp(`${B}pwd${A}`, 'i'),
67
+ // Card / identity numbers
68
+ new RegExp(`${B}cvv${A}`, 'i'),
69
+ new RegExp(`${B}cvc${A}`, 'i'),
70
+ new RegExp(`${B}pin${A}`, 'i'),
71
+ new RegExp(`${B}ssn${A}`, 'i'),
46
72
  ];
47
73
  /**
48
74
  * Names that matter for env vars and config keys but NOT for SQL identifiers.
@@ -60,11 +86,15 @@ export const CONFIG_ONLY_NAME_PATTERNS = [
60
86
  ];
61
87
  /** Whether a name identifies a credential. */
62
88
  export function isSecretName(name) {
63
- return SECRET_NAME_PATTERNS.some((pattern) => pattern.test(name));
89
+ const normalized = splitCamelCase(name);
90
+ return SECRET_NAME_PATTERNS.some((pattern) => pattern.test(normalized));
64
91
  }
65
92
  /** Whether a name is sensitive in a config/env context (credentials plus contact/service names). */
66
93
  export function isSensitiveConfigName(name) {
67
- return isSecretName(name) || CONFIG_ONLY_NAME_PATTERNS.some((pattern) => pattern.test(name));
94
+ if (isSecretName(name))
95
+ return true;
96
+ const normalized = splitCamelCase(name);
97
+ return CONFIG_ONLY_NAME_PATTERNS.some((pattern) => pattern.test(normalized));
68
98
  }
69
99
  /** Identifier-ish tokens in a SQL statement: table names, column names, aliases. */
70
100
  const SQL_IDENTIFIER_RE = /[A-Za-z_][A-Za-z0-9_$]*/g;
@@ -116,3 +146,114 @@ export function looksLikeCredentialValue(value) {
116
146
  return true;
117
147
  return CREDENTIAL_VALUE_PATTERNS.some((pattern) => pattern.test(value));
118
148
  }
149
+ // ---------------------------------------------------------------------------
150
+ // Binding hygiene
151
+ // ---------------------------------------------------------------------------
152
+ /** Max length for a stored string binding before it is truncated. */
153
+ const MAX_BINDING_LEN = 256;
154
+ /** Placeholder stored in place of a binding that looks like a credential. */
155
+ const REDACTED_BINDING = '[redacted]';
156
+ /**
157
+ * Redact and truncate SQL bindings before persistence.
158
+ *
159
+ * Two independent rules, because neither catches everything on its own:
160
+ *
161
+ * 1. **By statement.** If the SQL mentions a credential-shaped identifier
162
+ * (`password`, `remember_token`, `otp`, ...), every binding for that
163
+ * statement is redacted. Positional bindings cannot be mapped back to
164
+ * columns reliably, so this is all-or-nothing per statement — coarse, but
165
+ * it is the only thing that catches a short secret like a 6-digit OTP.
166
+ * 2. **By value shape.** Hashes, JWTs, provider key prefixes, long hex
167
+ * digests, and URLs with embedded credentials are redacted wherever they
168
+ * appear, regardless of the statement.
169
+ *
170
+ * Anything that survives both is truncated at {@link MAX_BINDING_LEN} so an
171
+ * oversized payload cannot bloat the row.
172
+ *
173
+ * Ordinary parameters — ids, flags, emails, timestamps — still pass through, so
174
+ * the query pane stays useful for debugging.
175
+ */
176
+ export function sanitizeBindings(bindings, sqlText) {
177
+ const redactAll = sqlText !== undefined && sqlMentionsSecret(sqlText);
178
+ return sanitizeBindingValue(bindings, redactAll, true);
179
+ }
180
+ /**
181
+ * Redact a structured record (a pino log entry, an event payload) with the
182
+ * same key/shape rules as bindings but WITHOUT length truncation — log data
183
+ * carries stack traces and payload context that must stay whole.
184
+ */
185
+ export function sanitizeRecordValues(record) {
186
+ return sanitizeBindingValue(record, false, false);
187
+ }
188
+ function sanitizeBindingValue(value, redactAll, truncate) {
189
+ if (Array.isArray(value))
190
+ return value.map((v) => sanitizeBindingValue(v, redactAll, truncate));
191
+ if (value !== null && typeof value === 'object') {
192
+ return sanitizeNamedBindings(value, redactAll, truncate);
193
+ }
194
+ if (typeof value === 'string')
195
+ return sanitizeStringBinding(value, redactAll, truncate);
196
+ // A statement touching a secret column may bind it as a non-string — a numeric
197
+ // OTP, for instance — so redact those too rather than only masking strings.
198
+ // Booleans and null carry nothing worth hiding.
199
+ const redactable = value !== null && value !== undefined && typeof value !== 'boolean';
200
+ return redactAll && redactable ? REDACTED_BINDING : value;
201
+ }
202
+ /** Named bindings carry their own key, so use it when it is telling. */
203
+ function sanitizeNamedBindings(value, redactAll, truncate) {
204
+ const out = {};
205
+ for (const [key, nested] of Object.entries(value)) {
206
+ out[key] = sanitizeBindingValue(nested, redactAll || isSecretName(key), truncate);
207
+ }
208
+ return out;
209
+ }
210
+ /** Matches a value this module already truncated, so a second pass is a no-op. */
211
+ const TRUNCATION_MARKER_RE = /…\[truncated \d+ chars\]$/;
212
+ function sanitizeStringBinding(value, redactAll, truncate) {
213
+ if (redactAll || looksLikeCredentialValue(value))
214
+ return REDACTED_BINDING;
215
+ if (truncate && value.length > MAX_BINDING_LEN && !TRUNCATION_MARKER_RE.test(value)) {
216
+ return value.slice(0, MAX_BINDING_LEN) + `…[truncated ${value.length} chars]`;
217
+ }
218
+ return value;
219
+ }
220
+ // ---------------------------------------------------------------------------
221
+ // URL hygiene
222
+ // ---------------------------------------------------------------------------
223
+ /**
224
+ * Redact credential-bearing query parameters from a URL before it is stored.
225
+ *
226
+ * Request URLs are persisted with their query strings (useful for debugging
227
+ * pagination and filters), but a `?token=…` password-reset link or a
228
+ * `?signature=…` signed URL must not land in SQLite in the clear. Parameters
229
+ * are redacted by name (see {@link isSecretName}) or when the value itself
230
+ * looks like a credential; everything else passes through untouched.
231
+ */
232
+ export function sanitizeUrlQuery(url) {
233
+ const qIdx = url.indexOf('?');
234
+ if (qIdx === -1)
235
+ return url;
236
+ const sanitized = url
237
+ .slice(qIdx + 1)
238
+ .split('&')
239
+ .map((pair) => {
240
+ if (pair === '')
241
+ return pair;
242
+ const eq = pair.indexOf('=');
243
+ const name = eq === -1 ? pair : pair.slice(0, eq);
244
+ const rawValue = eq === -1 ? '' : pair.slice(eq + 1);
245
+ let value = rawValue;
246
+ try {
247
+ value = decodeURIComponent(rawValue);
248
+ }
249
+ catch {
250
+ // Malformed escape — judge the raw text instead.
251
+ }
252
+ if (isSecretName(name) || (value !== '' && looksLikeCredentialValue(value))) {
253
+ return `${name}=[redacted]`;
254
+ }
255
+ return pair;
256
+ })
257
+ .join('&');
258
+ return url.slice(0, qIdx) + '?' + sanitized;
259
+ }
@@ -20,18 +20,23 @@ export interface StorageStatsResult {
20
20
  fileSizeMb: number;
21
21
  walSizeMb: number;
22
22
  retentionDays: number;
23
+ /** Configured live-data size cap in MB; `0` means the cap is disabled. */
24
+ maxDbSizeMb: number;
23
25
  tables: Array<{
24
26
  name: string;
25
27
  rowCount: number;
26
28
  }>;
27
29
  lastCleanupAt: number | null;
28
30
  }
31
+ /** The not-ready result returned before the SQLite connection exists. */
32
+ export declare function emptyStorageStats(dbPath: string, retentionDays: number, maxDbSizeMb: number): StorageStatsResult;
29
33
  export interface StorageStatsOpts {
30
34
  db: Knex;
31
35
  cache: CoalesceCache;
32
36
  dbFilePath: string;
33
37
  dbPath: string;
34
38
  retentionDays: number;
39
+ maxDbSizeMb: number;
35
40
  lastCleanupAt: number | null;
36
41
  onResult: (stats: StorageStatsResult) => void;
37
42
  }
@@ -58,11 +58,24 @@ export async function countAllTables(db) {
58
58
  return result;
59
59
  });
60
60
  }
61
+ /** The not-ready result returned before the SQLite connection exists. */
62
+ export function emptyStorageStats(dbPath, retentionDays, maxDbSizeMb) {
63
+ return {
64
+ ready: false,
65
+ dbPath,
66
+ fileSizeMb: 0,
67
+ walSizeMb: 0,
68
+ retentionDays,
69
+ maxDbSizeMb,
70
+ tables: [],
71
+ lastCleanupAt: null,
72
+ };
73
+ }
61
74
  /**
62
75
  * Build storage stats by querying file sizes and table row counts.
63
76
  */
64
77
  export function fetchStorageStats(opts) {
65
- const { db, cache, dbFilePath, dbPath, retentionDays, lastCleanupAt, onResult } = opts;
78
+ const { db, cache, dbFilePath, dbPath, retentionDays, maxDbSizeMb, lastCleanupAt, onResult } = opts;
66
79
  return cache.coalesce('storageStats', async () => {
67
80
  const [fileSizeMb, walSizeMb] = await getFileSizes(dbFilePath);
68
81
  const tables = await countAllTables(db);
@@ -72,6 +85,7 @@ export function fetchStorageStats(opts) {
72
85
  fileSizeMb,
73
86
  walSizeMb,
74
87
  retentionDays,
88
+ maxDbSizeMb,
75
89
  tables,
76
90
  lastCleanupAt,
77
91
  };
@@ -5,11 +5,18 @@
5
5
  * objects. They are pure (no I/O, no Knex dependency) so they can
6
6
  * be tested in isolation.
7
7
  */
8
+ import { sanitizeBindings } from './sensitive_patterns.js';
8
9
  import type { EventRecord, EmailRecord } from '../debug/types.js';
9
10
  import type { PersistRequestInput } from './dashboard_types.js';
10
11
  import type { Knex } from 'knex';
11
12
  export declare function hasWarned(path: string): boolean;
12
13
  export declare function markWarned(path: string): void;
14
+ /**
15
+ * Forget which write paths have warned. Called when a DashboardStore stops so
16
+ * a re-initialized store in the same process warns afresh instead of
17
+ * inheriting a spent latch from its predecessor.
18
+ */
19
+ export declare function resetWriteWarnings(): void;
13
20
  /**
14
21
  * Normalize a SQL query by replacing literal values with `?` placeholders.
15
22
  * Used for grouping identical query patterns.
@@ -20,27 +27,7 @@ export declare function markWarned(path: string): void;
20
27
  * merged together.
21
28
  */
22
29
  export declare function normalizeSql(sql: string): string;
23
- /**
24
- * Redact and truncate SQL bindings before persistence.
25
- *
26
- * Two independent rules, because neither catches everything on its own:
27
- *
28
- * 1. **By statement.** If the SQL mentions a credential-shaped identifier
29
- * (`password`, `remember_token`, `otp`, ...), every binding for that
30
- * statement is redacted. Positional bindings cannot be mapped back to
31
- * columns reliably, so this is all-or-nothing per statement — coarse, but
32
- * it is the only thing that catches a short secret like a 6-digit OTP.
33
- * 2. **By value shape.** Hashes, JWTs, provider key prefixes, long hex
34
- * digests, and URLs with embedded credentials are redacted wherever they
35
- * appear, regardless of the statement.
36
- *
37
- * Anything that survives both is truncated at {@link MAX_BINDING_LEN} so an
38
- * oversized payload cannot bloat the row.
39
- *
40
- * Ordinary parameters — ids, flags, emails, timestamps — still pass through, so
41
- * the query pane stays useful for debugging.
42
- */
43
- export declare function sanitizeBindings(bindings: unknown, sqlText?: string): unknown;
30
+ export { sanitizeBindings };
44
31
  export interface PreparedQuery {
45
32
  sql_text: string;
46
33
  sql_normalized: string;
@@ -100,6 +87,13 @@ export interface EventRow {
100
87
  export declare function prepareRequestRows(requests: PersistRequestInput[]): PreparedRequest[];
101
88
  /**
102
89
  * Transform raw log entries into SQLite-ready row objects.
90
+ *
91
+ * The full pino record lands in `data`, and log lines routinely carry bound
92
+ * request context — authorization headers, cookies, tokens in payloads — so
93
+ * the record is run through the shared redaction rules (secret-named keys and
94
+ * credential-shaped values) before it is stringified. The `message` column is
95
+ * kept verbatim: it is free text, and the shape rules are anchored to whole
96
+ * values, so prose stays readable.
103
97
  */
104
98
  export declare function prepareLogRows(logs: Record<string, unknown>[]): PreparedLog[];
105
99
  /**
@@ -118,6 +112,11 @@ export declare function buildEmailRow(record: EmailRecord): EmailRow;
118
112
  export declare function buildEventRows(events: EventRecord[]): EventRow[];
119
113
  /**
120
114
  * Insert rows into a table in batches of 50.
115
+ *
116
+ * Yields to the event loop between chunks: better-sqlite3 is fully
117
+ * synchronous, so without a real macrotask hop a large batch would execute
118
+ * back-to-back on microtask continuations and stall the host app — the exact
119
+ * failure mode `migrator_tables.ts` documents and guards against.
121
120
  */
122
121
  export declare function batchInsert(trx: Knex.Transaction, table: string, rows: Record<string, unknown>[]): Promise<void>;
123
122
  export declare function flushRequests(trx: Knex.Transaction, preparedRequests: PreparedRequest[]): Promise<void>;