@bakery-framework/plugin-analytics 2.0.0-alpha.11 → 2.0.0-alpha.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +37 -6
- package/package.json +3 -2
- package/src/core.ts +73 -31
- package/src/endpoints/stats.ts +0 -1
- package/src/index.ts +0 -1
- package/src/loop.ts +56 -6
- package/src/setup.ts +0 -1
- package/src/storage-sqlite.ts +94 -24
- package/src/timescale.ts +88 -14
- package/src/types.ts +0 -1
package/README.md
CHANGED
|
@@ -22,13 +22,44 @@ export default defineConfig({
|
|
|
22
22
|
|
|
23
23
|
Rolling histories are exported for reading directly — `history1m`, `history1h`,
|
|
24
24
|
`history1d`, `history7d`, `history30d` — alongside `pageHitsMap`,
|
|
25
|
-
`pageHitsLog` and the `recordRouteHit` / `
|
|
26
|
-
counters.
|
|
25
|
+
`pageHitsLog` and the `recordRouteHit` / `recordErrorPageHit` counters.
|
|
27
26
|
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
27
|
+
## Authorization
|
|
28
|
+
|
|
29
|
+
The stats endpoint, the reset endpoint and the live socket are closed unless
|
|
30
|
+
you open them, and there are two ways to do it:
|
|
31
|
+
|
|
32
|
+
```ts
|
|
33
|
+
import { defineConfig } from '@bakery-framework/core'
|
|
34
|
+
import analyticsPlugin from '@bakery-framework/plugin-analytics'
|
|
35
|
+
|
|
36
|
+
export default defineConfig({
|
|
37
|
+
plugins: [
|
|
38
|
+
analyticsPlugin({
|
|
39
|
+
// A shared secret, sent as an `x-analytics-key` header or, for the
|
|
40
|
+
// socket, an `analytics-key` query parameter — a browser cannot set a
|
|
41
|
+
// header on a WebSocket handshake.
|
|
42
|
+
credential: process.env.ANALYTICS_KEY,
|
|
43
|
+
|
|
44
|
+
// Or your own predicate, which is what the dashboard plugin hands over
|
|
45
|
+
// when both are registered.
|
|
46
|
+
authorize: (req: Request) => req.headers.get('x-role') === 'admin',
|
|
47
|
+
}),
|
|
48
|
+
],
|
|
49
|
+
})
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
With neither configured the guard allows loopback in development and denies
|
|
53
|
+
everything in production, so the closed state is the default rather than
|
|
54
|
+
something to remember.
|
|
55
|
+
|
|
56
|
+
**`DASHPASS` is not read and has not been since `bfe410c`.** Earlier versions
|
|
57
|
+
of this file said the endpoint was guarded by it. It never granted access —
|
|
58
|
+
it changed a status code — and nothing in the framework consults it now.
|
|
59
|
+
Delete it from any environment that still carries it: a variable that looks
|
|
60
|
+
like a credential and controls nothing is the kind of leftover an operator
|
|
61
|
+
reasons from. See
|
|
62
|
+
[Environment](https://github.com/obillekyle/bakery/blob/main/docs/configuration/environment.md).
|
|
32
63
|
|
|
33
64
|
## License
|
|
34
65
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bakery-framework/plugin-analytics",
|
|
3
|
-
"version": "2.0.0-alpha.
|
|
3
|
+
"version": "2.0.0-alpha.13",
|
|
4
4
|
"description": "Bakery analytics plugin.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"bakery",
|
|
@@ -27,6 +27,7 @@
|
|
|
27
27
|
".": "./src/index.ts",
|
|
28
28
|
"./setup": "./src/setup.ts",
|
|
29
29
|
"./stats": "./src/endpoints/stats.ts",
|
|
30
|
+
"./timescale": "./src/timescale.ts",
|
|
30
31
|
"./package.json": "./package.json"
|
|
31
32
|
},
|
|
32
33
|
"files": [
|
|
@@ -35,7 +36,7 @@
|
|
|
35
36
|
"!src/tests"
|
|
36
37
|
],
|
|
37
38
|
"dependencies": {
|
|
38
|
-
"@bakery-framework/core": "^2.0.0-alpha.
|
|
39
|
+
"@bakery-framework/core": "^2.0.0-alpha.13"
|
|
39
40
|
},
|
|
40
41
|
"engines": {
|
|
41
42
|
"bun": ">=1.4.0"
|
package/src/core.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { TIMESCALES, timescaleFacts } from './timescale'
|
|
1
2
|
import type { AnalyticsSnapshot } from './types'
|
|
2
3
|
|
|
3
4
|
export const RETENTION_MS = 30 * 24 * 3600 * 1000
|
|
@@ -29,7 +30,6 @@ type TempAccumulator = {
|
|
|
29
30
|
apiHits: number
|
|
30
31
|
pageHits: number
|
|
31
32
|
uniqueRequests: number
|
|
32
|
-
dbHits: number
|
|
33
33
|
errorPageHits: number
|
|
34
34
|
ping: number
|
|
35
35
|
}
|
|
@@ -44,7 +44,6 @@ function createAccumulator(): TempAccumulator {
|
|
|
44
44
|
apiHits: 0,
|
|
45
45
|
pageHits: 0,
|
|
46
46
|
uniqueRequests: 0,
|
|
47
|
-
dbHits: 0,
|
|
48
47
|
errorPageHits: 0,
|
|
49
48
|
ping: 0,
|
|
50
49
|
}
|
|
@@ -59,7 +58,6 @@ let routeHitsThisSecond = 0
|
|
|
59
58
|
let apiHitsThisSecond = 0
|
|
60
59
|
let pageHitsThisSecond = 0
|
|
61
60
|
const uniqueRequestsThisSecond = new Set<string>()
|
|
62
|
-
let dbHitsThisSecond = 0
|
|
63
61
|
let errorPageHitsThisSecond = 0
|
|
64
62
|
|
|
65
63
|
/**
|
|
@@ -104,9 +102,17 @@ export function ensurePageHitsLogPruner() {
|
|
|
104
102
|
try {
|
|
105
103
|
prunePageHitsLog(Date.now())
|
|
106
104
|
} catch (_e) {
|
|
107
|
-
//
|
|
105
|
+
// Best-effort: a pruning failure must not take down the telemetry that
|
|
106
|
+
// is only observing the server.
|
|
108
107
|
}
|
|
109
108
|
}, 60_000)
|
|
109
|
+
// Started by the *first page hit*, so any process that serves one ordinary
|
|
110
|
+
// request holds the event loop open for ever without it - a script that
|
|
111
|
+
// imports the plugin and finishes its work never exits. Same class as the
|
|
112
|
+
// three core timers unref'd for A15; this one lives in a plugin and was
|
|
113
|
+
// outside what that pass looked at. Optional-called because a test may
|
|
114
|
+
// install a fake timer that has no `unref`.
|
|
115
|
+
_pageHitsLogPruneTimer.unref?.()
|
|
110
116
|
}
|
|
111
117
|
|
|
112
118
|
export function stopPageHitsLogPruner() {
|
|
@@ -125,7 +131,6 @@ function accumulate(temp: TempAccumulator, s: AnalyticsSnapshot) {
|
|
|
125
131
|
temp.apiHits += s.apiHits || 0
|
|
126
132
|
temp.pageHits += s.pageHits || 0
|
|
127
133
|
temp.uniqueRequests += s.uniqueRequests || 0
|
|
128
|
-
temp.dbHits += s.dbHits || 0
|
|
129
134
|
temp.errorPageHits += s.errorPageHits || 0
|
|
130
135
|
temp.ping += s.ping || 0
|
|
131
136
|
}
|
|
@@ -144,7 +149,6 @@ function finalizeAggregation(
|
|
|
144
149
|
apiHits: temp.apiHits,
|
|
145
150
|
pageHits: temp.pageHits,
|
|
146
151
|
uniqueRequests: temp.uniqueRequests,
|
|
147
|
-
dbHits: temp.dbHits,
|
|
148
152
|
errorPageHits: temp.errorPageHits,
|
|
149
153
|
ping: Math.round(temp.ping / count),
|
|
150
154
|
}
|
|
@@ -165,7 +169,6 @@ function loadAccumulator(target: TempAccumulator, loaded: any) {
|
|
|
165
169
|
target.apiHits += s.apiHits || 0
|
|
166
170
|
target.pageHits += s.pageHits || 0
|
|
167
171
|
target.uniqueRequests += s.uniqueRequests || 0
|
|
168
|
-
target.dbHits += s.dbHits || 0
|
|
169
172
|
target.errorPageHits += s.errorPageHits || 0
|
|
170
173
|
target.ping += s.ping || 0
|
|
171
174
|
}
|
|
@@ -182,7 +185,32 @@ export function isAssetPath(path: string): boolean {
|
|
|
182
185
|
)
|
|
183
186
|
}
|
|
184
187
|
|
|
188
|
+
/**
|
|
189
|
+
* The analytics loop's own request does not count as traffic.
|
|
190
|
+
*
|
|
191
|
+
* `runAnalyticsTick` fetches `/_analytics/ping` through the real server once a
|
|
192
|
+
* second so it can time a round trip, and that request reaches `onRoute` like
|
|
193
|
+
* any other. The result was a permanent floor of one route hit and one unique
|
|
194
|
+
* request per second on an idle server - every chart reading 1 instead of 0,
|
|
195
|
+
* and a day's `uniqueRequests` carrying 86,400 of the loop's own pings.
|
|
196
|
+
*
|
|
197
|
+
* Exact matches, not a prefix, for the same reason `isAnalyticsPath` in
|
|
198
|
+
* `setup.ts` uses exact matches: `/_analytics/pingback` would belong to the
|
|
199
|
+
* application. The two `/api/_analytics/*` endpoints are the console asking
|
|
200
|
+
* for its own data, which is equally not application traffic.
|
|
201
|
+
*/
|
|
202
|
+
const SELF_PATHS = new Set([
|
|
203
|
+
'/_analytics/ping',
|
|
204
|
+
'/api/_analytics/stats',
|
|
205
|
+
'/api/_analytics/reset',
|
|
206
|
+
])
|
|
207
|
+
|
|
208
|
+
export function isSelfPath(path: string): boolean {
|
|
209
|
+
return SELF_PATHS.has(path)
|
|
210
|
+
}
|
|
211
|
+
|
|
185
212
|
export function recordRouteHit(method: string, path: string, search = '') {
|
|
213
|
+
if (SELF_PATHS.has(path)) return
|
|
186
214
|
routeHitsThisSecond += 1
|
|
187
215
|
if (path.startsWith('/api/')) {
|
|
188
216
|
apiHitsThisSecond += 1
|
|
@@ -195,10 +223,6 @@ export function recordRouteHit(method: string, path: string, search = '') {
|
|
|
195
223
|
uniqueRequestsThisSecond.add(`${method} ${path}${search}`)
|
|
196
224
|
}
|
|
197
225
|
|
|
198
|
-
export function recordDbHit() {
|
|
199
|
-
dbHitsThisSecond += 1
|
|
200
|
-
}
|
|
201
|
-
|
|
202
226
|
export function recordErrorPageHit() {
|
|
203
227
|
errorPageHitsThisSecond += 1
|
|
204
228
|
}
|
|
@@ -216,31 +240,34 @@ export function pushAnalyticsSnapshot(snapshot: {
|
|
|
216
240
|
apiHits: apiHitsThisSecond,
|
|
217
241
|
pageHits: pageHitsThisSecond,
|
|
218
242
|
uniqueRequests: uniqueRequestsThisSecond.size,
|
|
219
|
-
dbHits: dbHitsThisSecond,
|
|
220
243
|
errorPageHits: errorPageHitsThisSecond,
|
|
221
244
|
}
|
|
222
245
|
|
|
223
246
|
history1m.push(fullSnapshot)
|
|
224
247
|
if (history1m.length > 60) history1m.shift()
|
|
225
248
|
|
|
249
|
+
// The bucket sizes were four literals here — 60, 1800, 21600, 86400 — and
|
|
250
|
+
// they are the same numbers the point limits and the chart intervals were
|
|
251
|
+
// built from in three other places. `samples` derives from the window and
|
|
252
|
+
// the point count, so a timescale that changes shape changes all of them.
|
|
226
253
|
accumulate(temp1h, fullSnapshot)
|
|
227
254
|
accumulate(temp1d, fullSnapshot)
|
|
228
255
|
accumulate(temp7d, fullSnapshot)
|
|
229
256
|
accumulate(temp30d, fullSnapshot)
|
|
230
257
|
|
|
231
|
-
if (temp1h.count >=
|
|
258
|
+
if (temp1h.count >= TIMESCALES['1h'].samples) {
|
|
232
259
|
history1h.push(finalizeAggregation(temp1h, fullSnapshot.timestamp))
|
|
233
260
|
if (history1h.length > 60) history1h.shift()
|
|
234
261
|
}
|
|
235
|
-
if (temp1d.count >=
|
|
262
|
+
if (temp1d.count >= TIMESCALES['1d'].samples) {
|
|
236
263
|
history1d.push(finalizeAggregation(temp1d, fullSnapshot.timestamp))
|
|
237
264
|
if (history1d.length > 48) history1d.shift()
|
|
238
265
|
}
|
|
239
|
-
if (temp7d.count >=
|
|
266
|
+
if (temp7d.count >= TIMESCALES['7d'].samples) {
|
|
240
267
|
history7d.push(finalizeAggregation(temp7d, fullSnapshot.timestamp))
|
|
241
268
|
if (history7d.length > 28) history7d.shift()
|
|
242
269
|
}
|
|
243
|
-
if (temp30d.count >=
|
|
270
|
+
if (temp30d.count >= TIMESCALES['30d'].samples) {
|
|
244
271
|
history30d.push(finalizeAggregation(temp30d, fullSnapshot.timestamp))
|
|
245
272
|
if (history30d.length > 30) history30d.shift()
|
|
246
273
|
}
|
|
@@ -249,7 +276,6 @@ export function pushAnalyticsSnapshot(snapshot: {
|
|
|
249
276
|
apiHitsThisSecond = 0
|
|
250
277
|
pageHitsThisSecond = 0
|
|
251
278
|
uniqueRequestsThisSecond.clear()
|
|
252
|
-
dbHitsThisSecond = 0
|
|
253
279
|
errorPageHitsThisSecond = 0
|
|
254
280
|
}
|
|
255
281
|
|
|
@@ -260,26 +286,15 @@ export function getLatestAnalyticsSnapshot() {
|
|
|
260
286
|
apiHits: 0,
|
|
261
287
|
pageHits: 0,
|
|
262
288
|
uniqueRequests: 0,
|
|
263
|
-
dbHits: 0,
|
|
264
289
|
errorPageHits: 0,
|
|
265
290
|
ping: 0,
|
|
266
291
|
}
|
|
267
292
|
)
|
|
268
293
|
}
|
|
269
294
|
|
|
295
|
+
/** One of the five copies of the timescale table; see `timescale.ts`. */
|
|
270
296
|
export function getHistoryLimitForTimescale(timescale: string): number {
|
|
271
|
-
|
|
272
|
-
case '30d':
|
|
273
|
-
return 30
|
|
274
|
-
case '7d':
|
|
275
|
-
return 28
|
|
276
|
-
case '1d':
|
|
277
|
-
return 48
|
|
278
|
-
case '1h':
|
|
279
|
-
return 60
|
|
280
|
-
default:
|
|
281
|
-
return 60
|
|
282
|
-
}
|
|
297
|
+
return timescaleFacts(timescale).points
|
|
283
298
|
}
|
|
284
299
|
|
|
285
300
|
export function getHistoryForTimescale(timescale: string): AnalyticsSnapshot[] {
|
|
@@ -354,7 +369,6 @@ export function getFilledHistoryForTimescale(
|
|
|
354
369
|
apiHits: null,
|
|
355
370
|
pageHits: null,
|
|
356
371
|
uniqueRequests: null,
|
|
357
|
-
dbHits: null,
|
|
358
372
|
errorPageHits: null,
|
|
359
373
|
ping: null,
|
|
360
374
|
})
|
|
@@ -368,6 +382,34 @@ export function getFilledHistoryForTimescale(
|
|
|
368
382
|
return filled
|
|
369
383
|
}
|
|
370
384
|
|
|
385
|
+
/**
|
|
386
|
+
* The write half of `loadTemps`, which had no write half.
|
|
387
|
+
*
|
|
388
|
+
* `setup.ts` checked `data.temp1h` on boot and `loadTemps` knew how to restore
|
|
389
|
+
* all four buckets, but nothing ever put them in the persisted snapshot - so
|
|
390
|
+
* the check was always false and every restart began aggregating from zero. A
|
|
391
|
+
* bucket only finalises at its full count (60 samples for 1h, 1800 for 1d), so
|
|
392
|
+
* the visible symptom was the 1h chart staying empty for up to an hour after
|
|
393
|
+
* every boot, and the longer windows correspondingly longer.
|
|
394
|
+
*
|
|
395
|
+
* Plain objects rather than the accumulators themselves: this is serialised to
|
|
396
|
+
* JSON in the `core` row, and handing out the live objects would let a caller
|
|
397
|
+
* mutate the running aggregation.
|
|
398
|
+
*/
|
|
399
|
+
export function snapshotTemps(): {
|
|
400
|
+
temp1h: TempAccumulator
|
|
401
|
+
temp1d: TempAccumulator
|
|
402
|
+
temp7d: TempAccumulator
|
|
403
|
+
temp30d: TempAccumulator
|
|
404
|
+
} {
|
|
405
|
+
return {
|
|
406
|
+
temp1h: { ...temp1h },
|
|
407
|
+
temp1d: { ...temp1d },
|
|
408
|
+
temp7d: { ...temp7d },
|
|
409
|
+
temp30d: { ...temp30d },
|
|
410
|
+
}
|
|
411
|
+
}
|
|
412
|
+
|
|
371
413
|
export function loadTemps(loaded: any) {
|
|
372
414
|
if (!loaded) return
|
|
373
415
|
if (loaded.temp1h) loadAccumulator(temp1h, loaded.temp1h)
|
package/src/endpoints/stats.ts
CHANGED
|
@@ -66,7 +66,6 @@ export function computeStats(
|
|
|
66
66
|
apiHits: latestHistory.apiHits || 0,
|
|
67
67
|
pageHits: latestHistory.pageHits || 0,
|
|
68
68
|
uniqueRequests: latestHistory.uniqueRequests,
|
|
69
|
-
dbHits: latestHistory.dbHits,
|
|
70
69
|
errorPageHits: latestHistory.errorPageHits,
|
|
71
70
|
ping: latestHistory.ping,
|
|
72
71
|
topPages: topPagesFiltered,
|
package/src/index.ts
CHANGED
package/src/loop.ts
CHANGED
|
@@ -6,7 +6,7 @@ import * as core from './core'
|
|
|
6
6
|
import { computeStats } from './endpoints/stats'
|
|
7
7
|
import { connectedAnalyticsClients } from './endpoints/websocket'
|
|
8
8
|
import { analyticsLog } from './log'
|
|
9
|
-
import { saveAnalyticsData } from './storage-sqlite'
|
|
9
|
+
import { flushPageHits, saveAnalyticsData } from './storage-sqlite'
|
|
10
10
|
|
|
11
11
|
const SAVE_THROTTLE_MS = 60000
|
|
12
12
|
|
|
@@ -53,18 +53,68 @@ async function runAnalyticsTick(server: any) {
|
|
|
53
53
|
ping: pingVal,
|
|
54
54
|
})
|
|
55
55
|
|
|
56
|
+
// Hits go to disk every tick; the prune and the history snapshot stay on the
|
|
57
|
+
// throttle. The two were one call, and the insert was blamed for a stall the
|
|
58
|
+
// prune was doing: a minute of hits at 1,000 req/s inserts in 89-115 ms
|
|
59
|
+
// while the prune beside it cost 376 ms. Per tick the insert is 1-4 ms, so
|
|
60
|
+
// the large batch leaves the request thread and the expensive half runs a
|
|
61
|
+
// sixtieth as often.
|
|
62
|
+
//
|
|
56
63
|
// Awaited, not floated: a rejected flush used to escape this tick entirely
|
|
57
64
|
// and land nowhere.
|
|
65
|
+
const [flushErr] = await Try.catch(flushPageHits())
|
|
66
|
+
if (flushErr) analyticsLog.SAVE_ERR({ error: errorMsg(flushErr) })
|
|
67
|
+
|
|
58
68
|
await throttleSave()
|
|
59
69
|
|
|
70
|
+
broadcastStats()
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* One `computeStats` per distinct subscription, not per client.
|
|
75
|
+
*
|
|
76
|
+
* Every connected console got its own call each second, and two consoles
|
|
77
|
+
* watching the same window asked the same question twice. They rarely differ:
|
|
78
|
+
* the timescale and the pages filter both default to the same values and a
|
|
79
|
+
* console only changes them when somebody clicks. Grouping by those two keys
|
|
80
|
+
* makes the common case one call however many consoles are open.
|
|
81
|
+
*
|
|
82
|
+
* Worth the grouping rather than a single global call, because the answer
|
|
83
|
+
* genuinely depends on both: a client watching `1h` must not be sent a `1m`
|
|
84
|
+
* payload. Measured at 1.10 ms a call with a 2,000-path tally, so this is a
|
|
85
|
+
* millisecond a second at five consoles rather than five - real, and smaller
|
|
86
|
+
* than the 10 ms a call the backlog recorded.
|
|
87
|
+
*
|
|
88
|
+
* The serialised frame is reused too. `JSON.stringify` of a stats payload is
|
|
89
|
+
* not free, and it was run once per client over an identical object.
|
|
90
|
+
*
|
|
91
|
+
* `compute` is a test seam (convention 9): counting the calls is the only
|
|
92
|
+
* way to assert the grouping, and two equal strings are indistinguishable
|
|
93
|
+
* from one computed twice.
|
|
94
|
+
*/
|
|
95
|
+
export function broadcastStats(compute = computeStats) {
|
|
96
|
+
if (connectedAnalyticsClients.size === 0) return
|
|
97
|
+
|
|
98
|
+
const byWindow = new Map<string, string>()
|
|
60
99
|
for (const ws of connectedAnalyticsClients) {
|
|
61
100
|
const opts = ws.data?.data
|
|
62
|
-
if (opts)
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
101
|
+
if (!opts) continue
|
|
102
|
+
|
|
103
|
+
const timescale = opts.timescale
|
|
104
|
+
const pagesFilter = opts.pagesFilter
|
|
105
|
+
// Separated by a delimiter neither value can contain. A timescale is one
|
|
106
|
+
// of `1m|1h|1d|7d|30d` and a filter the same shape, so joining them with a
|
|
107
|
+
// space would in fact be safe today; a form that cannot collide costs
|
|
108
|
+
// nothing and does not depend on that staying true.
|
|
109
|
+
const key = JSON.stringify([timescale, pagesFilter])
|
|
110
|
+
|
|
111
|
+
let frame = byWindow.get(key)
|
|
112
|
+
if (frame === undefined) {
|
|
113
|
+
const stats = compute(timescale, true, pagesFilter)
|
|
114
|
+
frame = JSON.stringify({ status: 200, excludeHistory: true, data: stats })
|
|
115
|
+
byWindow.set(key, frame)
|
|
67
116
|
}
|
|
117
|
+
ws.send(frame)
|
|
68
118
|
}
|
|
69
119
|
}
|
|
70
120
|
|
package/src/setup.ts
CHANGED
|
@@ -31,7 +31,6 @@ export const history7d = core.history7d
|
|
|
31
31
|
export const history30d = core.history30d
|
|
32
32
|
|
|
33
33
|
export const recordRouteHit = core.recordRouteHit
|
|
34
|
-
export const recordDbHit = core.recordDbHit
|
|
35
34
|
export const recordErrorPageHit = core.recordErrorPageHit
|
|
36
35
|
export const pushAnalyticsSnapshot = core.pushAnalyticsSnapshot
|
|
37
36
|
export const getLatestAnalyticsSnapshot = core.getLatestAnalyticsSnapshot
|
package/src/storage-sqlite.ts
CHANGED
|
@@ -12,6 +12,7 @@ import {
|
|
|
12
12
|
pageHitsLog,
|
|
13
13
|
pageHitsMap,
|
|
14
14
|
RETENTION_MS,
|
|
15
|
+
snapshotTemps,
|
|
15
16
|
} from './core'
|
|
16
17
|
import { analyticsLog } from './log'
|
|
17
18
|
import { timescaleToMs } from './timescale'
|
|
@@ -106,6 +107,57 @@ export default {
|
|
|
106
107
|
getDb,
|
|
107
108
|
}
|
|
108
109
|
|
|
110
|
+
/**
|
|
111
|
+
* Write the page hits recorded since the last call, and nothing else.
|
|
112
|
+
*
|
|
113
|
+
* Split out of `saveAnalyticsData` so the two halves can run on different
|
|
114
|
+
* schedules, because they cost very different amounts. Inserting a second of
|
|
115
|
+
* hits is 1-4 ms; the full save, with its prune and its `core` upsert, is not.
|
|
116
|
+
*
|
|
117
|
+
* Measured across a minute of traffic at 1,000 req/s: inserting 60,000 rows in
|
|
118
|
+
* transactions of 1,000 costs 89-115 ms, against 376 ms for the prune the
|
|
119
|
+
* other half runs. So the insert was never what made the flush expensive - the
|
|
120
|
+
* per-minute batch was blamed for a stall the prune was doing - and moving the
|
|
121
|
+
* insert to the tick costs almost nothing while removing the one large batch
|
|
122
|
+
* from the request thread entirely.
|
|
123
|
+
*
|
|
124
|
+
* Returns the number of rows written so a caller can tell "nothing to do" from
|
|
125
|
+
* "could not write", which the void-returning original could not.
|
|
126
|
+
*
|
|
127
|
+
* **Throws rather than reporting.** Both callers already have a catch that
|
|
128
|
+
* names the failure, and a catch here as well produced two log lines for one
|
|
129
|
+
* closed database - which is worse than one, because the second looks like a
|
|
130
|
+
* second failure. The rule that telemetry never takes down what it measures
|
|
131
|
+
* is kept at the call sites, where it belongs.
|
|
132
|
+
*/
|
|
133
|
+
export async function flushPageHits(): Promise<number> {
|
|
134
|
+
{
|
|
135
|
+
await initSqliteStorage()
|
|
136
|
+
const d = getDb()
|
|
137
|
+
if (!d) return 0
|
|
138
|
+
if (pageHitsLog.length === 0) return 0
|
|
139
|
+
|
|
140
|
+
if (!stmtInsertPageHit)
|
|
141
|
+
stmtInsertPageHit = d.prepare(
|
|
142
|
+
'INSERT INTO page_hits(timestamp,path) VALUES(?,?)',
|
|
143
|
+
)
|
|
144
|
+
|
|
145
|
+
const newHits = pageHitsLog.filter(p => p.timestamp > lastSavedPageHitTs)
|
|
146
|
+
if (newHits.length === 0) return 0
|
|
147
|
+
|
|
148
|
+
const tx = d.transaction((rows: [number, string][]) => {
|
|
149
|
+
for (const r of rows) stmtInsertPageHit!.run(r[0], r[1])
|
|
150
|
+
})
|
|
151
|
+
const rows: [number, string][] = newHits.map(p => [p.timestamp, p.path])
|
|
152
|
+
const BATCH = 1000
|
|
153
|
+
for (let i = 0; i < rows.length; i += BATCH) {
|
|
154
|
+
tx(rows.slice(i, i + BATCH))
|
|
155
|
+
}
|
|
156
|
+
lastSavedPageHitTs = newHits[newHits.length - 1].timestamp
|
|
157
|
+
return rows.length
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
|
|
109
161
|
export async function saveAnalyticsData(_cacheBase: string) {
|
|
110
162
|
try {
|
|
111
163
|
await initSqliteStorage()
|
|
@@ -129,36 +181,53 @@ export async function saveAnalyticsData(_cacheBase: string) {
|
|
|
129
181
|
const pruneBefore = now - RETENTION_MS
|
|
130
182
|
try {
|
|
131
183
|
stmtDeletePageHits.run(pruneBefore)
|
|
132
|
-
|
|
133
|
-
//
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
184
|
+
|
|
185
|
+
// **Ask before deleting.** Age alone does not bound this table - a
|
|
186
|
+
// crawler hitting distinct URLs can add millions of rows well inside the
|
|
187
|
+
// retention window - so the row count is capped too. But the capping
|
|
188
|
+
// statement is expensive in a way its shape hides: `NOT IN (SELECT rowid
|
|
189
|
+
// ... ORDER BY timestamp DESC LIMIT n)` sorts and materialises n rowids
|
|
190
|
+
// before it can decide that nothing needs deleting, and it ran on every
|
|
191
|
+
// single flush.
|
|
192
|
+
//
|
|
193
|
+
// A `count(*)` is answered from the index. Measured on a 150k-row table,
|
|
194
|
+
// three rounds each against a CPU-bound control that stayed flat:
|
|
195
|
+
//
|
|
196
|
+
// prune as shipped 376 ms
|
|
197
|
+
// prune with this guard 5 ms
|
|
198
|
+
//
|
|
199
|
+
// Over the cap the delete still runs and still costs what it costs
|
|
200
|
+
// (~360 ms at 250k rows), which is correct: that is the case where work
|
|
201
|
+
// is genuinely required, and it is the rare one.
|
|
202
|
+
const counted = d
|
|
203
|
+
.query<{ n: number }, []>('SELECT count(*) AS n FROM page_hits')
|
|
204
|
+
.get()
|
|
205
|
+
if ((counted?.n ?? 0) > MAX_PAGE_HIT_ROWS) {
|
|
206
|
+
d.run(
|
|
207
|
+
`DELETE FROM page_hits WHERE rowid NOT IN (
|
|
208
|
+
SELECT rowid FROM page_hits ORDER BY timestamp DESC LIMIT ${MAX_PAGE_HIT_ROWS}
|
|
209
|
+
)`,
|
|
210
|
+
)
|
|
211
|
+
}
|
|
139
212
|
} catch {
|
|
140
213
|
// Pruning is best-effort. Failing to trim old rows must not abandon the
|
|
141
214
|
// inserts below, which are the point of this call.
|
|
142
215
|
}
|
|
143
216
|
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
if (newHits.length > 0) {
|
|
152
|
-
const rows: [number, string][] = newHits.map(p => [p.timestamp, p.path])
|
|
153
|
-
const BATCH = 1000
|
|
154
|
-
for (let i = 0; i < rows.length; i += BATCH) {
|
|
155
|
-
tx(rows.slice(i, i + BATCH))
|
|
156
|
-
}
|
|
157
|
-
|
|
158
|
-
lastSavedPageHitTs = newHits[newHits.length - 1].timestamp
|
|
159
|
-
}
|
|
160
|
-
}
|
|
217
|
+
// One writer for page hits, and it is `flushPageHits`. This call used to
|
|
218
|
+
// hold a second copy of the same insert loop; with the loop flushing every
|
|
219
|
+
// tick that copy would find nothing to do on almost every call, which is
|
|
220
|
+
// the worst kind of dead code - it still looks like the thing doing the
|
|
221
|
+
// work. Called rather than deleted because this function is also the
|
|
222
|
+
// shutdown save, where it is the last chance to write anything at all.
|
|
223
|
+
await flushPageHits()
|
|
161
224
|
|
|
225
|
+
// `temp*` are the in-progress aggregation buckets, and they were **not**
|
|
226
|
+
// in this object while `setup.ts` read `data.temp1h` on boot and
|
|
227
|
+
// `core.loadTemps` knew how to restore them. Every restart therefore threw
|
|
228
|
+
// away up to 59 seconds of 1h aggregation, up to 29 minutes of 1d, and so
|
|
229
|
+
// on - and because a bucket only finalises at its full count, the 1h chart
|
|
230
|
+
// stayed empty for up to an hour after every boot rather than resuming.
|
|
162
231
|
const coreData: any = {
|
|
163
232
|
history1m: history1m as AnalyticsSnapshot[],
|
|
164
233
|
history1h: history1h as AnalyticsSnapshot[],
|
|
@@ -166,6 +235,7 @@ export async function saveAnalyticsData(_cacheBase: string) {
|
|
|
166
235
|
history7d: history7d as AnalyticsSnapshot[],
|
|
167
236
|
history30d: history30d as AnalyticsSnapshot[],
|
|
168
237
|
pageHits: Array.from(pageHitsMap.entries()),
|
|
238
|
+
...snapshotTemps(),
|
|
169
239
|
}
|
|
170
240
|
try {
|
|
171
241
|
stmtUpsertCore.run('core', JSON.stringify(coreData))
|
package/src/timescale.ts
CHANGED
|
@@ -1,16 +1,90 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Every timescale fact, in one place, derived from two numbers each.
|
|
3
|
+
*
|
|
4
|
+
* There were **five** of these tables across two packages, and they agreed —
|
|
5
|
+
* which is luck rather than design, because nothing made them. `timescaleToMs`
|
|
6
|
+
* here, `getHistoryLimitForTimescale` in `core.ts`, the bucket counts inside
|
|
7
|
+
* `pushAnalyticsSnapshot`, and `getTimescaleIntervalMs` / `getTimescaleLimit`
|
|
8
|
+
* in the dashboard's client, the last pair byte-identical to the second.
|
|
9
|
+
*
|
|
10
|
+
* Collapsing them turned up that the whole family follows from a window length
|
|
11
|
+
* and a point count. Checked against all five originals before this replaced
|
|
12
|
+
* them, and `timescale.test.ts` keeps checking:
|
|
13
|
+
*
|
|
14
|
+
* bucketMs = windowMs / points
|
|
15
|
+
* samples = bucketMs / TICK_MS
|
|
16
|
+
*
|
|
17
|
+
* So `1d` is a day shown as 48 points, which makes each point half an hour and
|
|
18
|
+
* each half hour 1,800 one-second samples — and those are exactly the numbers
|
|
19
|
+
* the five tables carried.
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
/** One sample per second is what every `history1m` window assumes. */
|
|
23
|
+
export const TICK_MS = 1000
|
|
24
|
+
|
|
25
|
+
export type Timescale = '1m' | '1h' | '1d' | '7d' | '30d'
|
|
26
|
+
|
|
27
|
+
const DAY_MS = 86_400_000
|
|
28
|
+
|
|
29
|
+
/** Window length and how many points it is drawn as. Everything else derives. */
|
|
30
|
+
const SHAPE: Record<Timescale, { windowMs: number; points: number }> = {
|
|
31
|
+
'1m': { windowMs: 60_000, points: 60 },
|
|
32
|
+
'1h': { windowMs: 3_600_000, points: 60 },
|
|
33
|
+
'1d': { windowMs: DAY_MS, points: 48 },
|
|
34
|
+
'7d': { windowMs: 7 * DAY_MS, points: 28 },
|
|
35
|
+
'30d': { windowMs: 30 * DAY_MS, points: 30 },
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export interface TimescaleFacts {
|
|
39
|
+
/** How far back the window reaches. */
|
|
40
|
+
windowMs: number
|
|
41
|
+
/** How many points it is drawn as, and how many the history array keeps. */
|
|
42
|
+
points: number
|
|
43
|
+
/** How much time one point covers. */
|
|
44
|
+
bucketMs: number
|
|
45
|
+
/** How many one-second samples fold into one point. */
|
|
46
|
+
samples: number
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
export const TIMESCALES: Record<Timescale, TimescaleFacts> = Object.freeze(
|
|
50
|
+
Object.fromEntries(
|
|
51
|
+
Object.entries(SHAPE).map(([key, { windowMs, points }]) => [
|
|
52
|
+
key,
|
|
53
|
+
{
|
|
54
|
+
windowMs,
|
|
55
|
+
points,
|
|
56
|
+
bucketMs: windowMs / points,
|
|
57
|
+
samples: windowMs / points / TICK_MS,
|
|
58
|
+
},
|
|
59
|
+
]),
|
|
60
|
+
),
|
|
61
|
+
) as Record<Timescale, TimescaleFacts>
|
|
62
|
+
|
|
63
|
+
/** The ordered list, for anything that offers a choice of them. */
|
|
64
|
+
export const TIMESCALE_KEYS = Object.keys(SHAPE) as Timescale[]
|
|
65
|
+
|
|
66
|
+
export function isTimescale(value: string): value is Timescale {
|
|
67
|
+
return value in SHAPE
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* Facts for a timescale, or `1m`'s.
|
|
72
|
+
*
|
|
73
|
+
* A default rather than a throw, because the value reaches here from a query
|
|
74
|
+
* parameter and a socket frame — both of which a client writes, and neither of
|
|
75
|
+
* which should be able to raise a 500. The narrowest window is the safe one to
|
|
76
|
+
* fall back to: it reads the least data and shows the least.
|
|
77
|
+
*/
|
|
78
|
+
export function timescaleFacts(timescale: string): TimescaleFacts {
|
|
79
|
+
return isTimescale(timescale) ? TIMESCALES[timescale] : TIMESCALES['1m']
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* How far back a timescale reaches, in milliseconds.
|
|
84
|
+
*
|
|
85
|
+
* `0` for an unknown one, which is what this answered before and what
|
|
86
|
+
* `storage-sqlite.ts` relies on to mean "no bound".
|
|
87
|
+
*/
|
|
1
88
|
export function timescaleToMs(timescale: string): number {
|
|
2
|
-
|
|
3
|
-
case '1m':
|
|
4
|
-
return 60_000
|
|
5
|
-
case '1h':
|
|
6
|
-
return 3_600_000
|
|
7
|
-
case '1d':
|
|
8
|
-
return 86_400_000
|
|
9
|
-
case '7d':
|
|
10
|
-
return 7 * 86_400_000
|
|
11
|
-
case '30d':
|
|
12
|
-
return 30 * 86_400_000
|
|
13
|
-
default:
|
|
14
|
-
return 0
|
|
15
|
-
}
|
|
89
|
+
return isTimescale(timescale) ? TIMESCALES[timescale].windowMs : 0
|
|
16
90
|
}
|