@tinyshed/tinystore 0.1.0-rc.5 → 0.1.0-rc.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (122) hide show
  1. package/README.md +37 -37
  2. package/bin/tinystore.js +12 -1
  3. package/dist/backup.d.ts +33 -0
  4. package/dist/backup.js +49 -0
  5. package/dist/blobs.d.ts +160 -79
  6. package/dist/blobs.js +510 -217
  7. package/dist/channels.d.ts +90 -0
  8. package/dist/channels.js +291 -0
  9. package/dist/clock.js +3 -3
  10. package/dist/config.d.ts +209 -86
  11. package/dist/config.js +581 -538
  12. package/dist/connection.d.ts +64 -2
  13. package/dist/connection.js +201 -16
  14. package/dist/console.d.ts +27 -3
  15. package/dist/console.js +165 -12
  16. package/dist/dotenv.d.ts +7 -0
  17. package/dist/dotenv.js +79 -0
  18. package/dist/errors.d.ts +61 -1
  19. package/dist/errors.js +81 -1
  20. package/dist/handles.d.ts +44 -2
  21. package/dist/handles.js +81 -2
  22. package/dist/index.d.ts +17 -11
  23. package/dist/index.js +5 -3
  24. package/dist/instruments.d.ts +177 -0
  25. package/dist/instruments.js +819 -0
  26. package/dist/jobs.d.ts +335 -239
  27. package/dist/jobs.js +716 -536
  28. package/dist/json.d.ts +2 -0
  29. package/dist/json.js +122 -0
  30. package/dist/kv.d.ts +130 -243
  31. package/dist/kv.js +273 -640
  32. package/dist/limits.d.ts +80 -0
  33. package/dist/limits.js +139 -0
  34. package/dist/locks.d.ts +62 -0
  35. package/dist/locks.js +237 -0
  36. package/dist/logger.d.ts +45 -9
  37. package/dist/logger.js +150 -38
  38. package/dist/metrics.d.ts +210 -209
  39. package/dist/metrics.js +319 -556
  40. package/dist/once.d.ts +19 -18
  41. package/dist/once.js +74 -63
  42. package/dist/query.d.ts +181 -0
  43. package/dist/query.js +788 -0
  44. package/dist/records.d.ts +241 -150
  45. package/dist/records.js +507 -291
  46. package/dist/runtime/addon.d.ts +14 -0
  47. package/dist/runtime/addon.js +69 -0
  48. package/dist/runtime/bun.js +5 -7
  49. package/dist/runtime/node.js +5 -7
  50. package/dist/runtime/pipe.d.ts +17 -0
  51. package/dist/runtime/pipe.js +133 -0
  52. package/dist/runtime/shared.d.ts +6 -0
  53. package/dist/runtime/shared.js +19 -0
  54. package/dist/runtime.d.ts +5 -2
  55. package/dist/search.d.ts +189 -0
  56. package/dist/search.js +281 -0
  57. package/dist/session.d.ts +31 -3
  58. package/dist/session.js +153 -41
  59. package/dist/sql.d.ts +228 -78
  60. package/dist/sql.js +745 -240
  61. package/dist/store.d.ts +244 -24
  62. package/dist/store.js +270 -67
  63. package/dist/time.d.ts +5 -1
  64. package/dist/time.js +22 -2
  65. package/dist/tx.d.ts +61 -0
  66. package/dist/tx.js +314 -0
  67. package/dist/wire/codec.d.ts +23 -7
  68. package/dist/wire/codec.js +56 -35
  69. package/dist/wire/frame.d.ts +2 -0
  70. package/dist/wire/frame.js +31 -14
  71. package/dist/wire/messages.d.ts +259 -0
  72. package/dist/wire/messages.js +22 -0
  73. package/dist/wire/msgpack.d.ts +49 -1
  74. package/dist/wire/msgpack.js +293 -56
  75. package/dist/wire/protocol.d.ts +2683 -0
  76. package/dist/wire/protocol.js +10846 -0
  77. package/package.json +39 -38
  78. package/src/backup.ts +81 -0
  79. package/src/blobs.ts +678 -301
  80. package/src/channels.ts +377 -0
  81. package/src/clock.ts +4 -4
  82. package/src/config.ts +902 -612
  83. package/src/connection.ts +279 -21
  84. package/src/console.ts +180 -12
  85. package/src/dotenv.ts +82 -0
  86. package/src/errors.ts +105 -0
  87. package/src/handles.ts +157 -8
  88. package/src/index.ts +158 -52
  89. package/src/instruments.ts +1042 -0
  90. package/src/jobs.ts +1017 -733
  91. package/src/json.ts +133 -0
  92. package/src/kv.ts +424 -847
  93. package/src/limits.ts +202 -0
  94. package/src/locks.ts +320 -0
  95. package/src/logger.ts +209 -40
  96. package/src/metrics.ts +557 -752
  97. package/src/once.ts +86 -83
  98. package/src/query.ts +1005 -0
  99. package/src/records.ts +795 -418
  100. package/src/runtime/addon.ts +99 -0
  101. package/src/runtime/bun.ts +5 -7
  102. package/src/runtime/node.ts +5 -7
  103. package/src/runtime/pipe.ts +178 -0
  104. package/src/runtime/shared.ts +20 -0
  105. package/src/runtime.ts +5 -2
  106. package/src/search.ts +480 -0
  107. package/src/session.ts +187 -41
  108. package/src/sql.ts +993 -306
  109. package/src/store.ts +473 -78
  110. package/src/time.ts +48 -3
  111. package/src/tx.ts +426 -0
  112. package/src/wire/codec.ts +78 -37
  113. package/src/wire/frame.ts +44 -14
  114. package/src/wire/messages.ts +25 -0
  115. package/src/wire/msgpack.ts +327 -56
  116. package/src/wire/protocol.ts +11709 -0
  117. package/dist/limiter.d.ts +0 -45
  118. package/dist/limiter.js +0 -68
  119. package/dist/quota.d.ts +0 -44
  120. package/dist/quota.js +0 -98
  121. package/src/limiter.ts +0 -99
  122. package/src/quota.ts +0 -137
@@ -0,0 +1,1042 @@
1
+ // A program's instruments, as docs/guides/metrics/README.md has them: counters, gauges,
2
+ // a gauge read from a function, timers and histograms, each set of labels a
3
+ // series of its own. A call changes a number here and never waits. What
4
+ // changed goes to the store's totals on a stream of its own, a batch a
5
+ // second, each numbered in this run's session, so that the store counts it
6
+ // once: a batch whose answer a lost connection took goes again on the next,
7
+ // and a store that started again is sent nothing it may have counted.
8
+
9
+ import { randomBytes } from 'node:crypto'
10
+
11
+ import { FailureLog, messageOf } from './background.ts'
12
+ import type { Link } from './connection.ts'
13
+ import { InvalidError, OutcomeUnknownError, UnavailableError } from './errors.ts'
14
+ import { sayOwn } from './logger.ts'
15
+ import { ConnectionLostError, type Stream } from './session.ts'
16
+ import { type Duration, ms } from './time.ts'
17
+ import { MetricsAck, MetricsBatch, MetricsSession, methods } from './wire/protocol.ts'
18
+
19
+ /** A label's value as an instrument takes it: a number or a boolean is kept as its text. */
20
+ export type LabelValue = string | number | boolean
21
+
22
+ /** What an instrument is told when it is made. */
23
+ export interface InstrumentOptions {
24
+ /** the unit of its values, 32 bytes at most: 'bytes', '%' */
25
+ unit?: string | undefined
26
+ /** a line on what it means, 1024 bytes at most */
27
+ help?: string | undefined
28
+ /** how long its name's samples stay, as ingest takes it */
29
+ keep?: Duration | undefined
30
+ }
31
+
32
+ /** A timer's or a histogram's upper bounds, ascending, or the store's grid. */
33
+ export type Buckets = readonly number[] | 'exponential'
34
+
35
+ export interface TimerOptions {
36
+ help?: string | undefined
37
+ keep?: Duration | undefined
38
+ /** the upper bounds of its spread, to which the store adds +Inf, or 'exponential' */
39
+ buckets?: Buckets | undefined
40
+ }
41
+
42
+ export interface HistogramOptions extends InstrumentOptions {
43
+ buckets?: Buckets | undefined
44
+ }
45
+
46
+ export interface GaugeFuncOptions extends InstrumentOptions {
47
+ /** the set of labels its function reads; another set of the name takes a function of its own */
48
+ labels?: InstrumentLabels | undefined
49
+ }
50
+
51
+ /**
52
+ * The labels `with` takes: any, or, when the instrument was made with their
53
+ * names, `counter<'route' | 'status'>(…)`, those alone, checked as they are written.
54
+ */
55
+ export type InstrumentLabels<L extends string = string> = { readonly [K in L]?: LabelValue }
56
+
57
+ type Form = 'counter' | 'gauge' | 'timer' | 'histogram'
58
+
59
+ /** The sets of labels an instrument holds; past them it counts into the series of __overflow. */
60
+ const mostSets = 2000
61
+ const sendEvery = 1000
62
+ /** How often a gauge's function is read. */
63
+ const readEvery = 15_000
64
+ /** Batches sent and not yet answered, past which changes gather for the next. */
65
+ const mostUnanswered = 16
66
+ /** The changes a batch carries at most. */
67
+ const mostChanges = 1000
68
+ /** How long a counter's or a timer's set of labels stays without a call. */
69
+ const idle = 3_600_000
70
+ /** The bounds a timer or a histogram takes at most. */
71
+ const mostBounds = 128
72
+
73
+ /** An instrument as the stream declares it, and its sets of labels. */
74
+ interface Declared {
75
+ id: number
76
+ name: string
77
+ form: Form
78
+ unit: string | undefined
79
+ help: string | undefined
80
+ keep: number | undefined
81
+ bounds: number[] | undefined
82
+ exponential: boolean
83
+ sets: Map<string, LabelSet>
84
+ overflow: LabelSet | undefined
85
+ /** the store refused it: its calls change nothing, said once */
86
+ refused: boolean
87
+ /** a gauge functions read, one a set of labels, by the key of its labels */
88
+ reads: Map<string, GaugeRead> | undefined
89
+ }
90
+
91
+ /** A function that reads a gauge's set of labels. */
92
+ interface GaugeRead {
93
+ labels: Record<string, string>
94
+ read: () => unknown
95
+ /** a call of it runs: it is not called again until it ends */
96
+ reading: boolean
97
+ /** the error it last gave, said once while it repeats */
98
+ failed: string | undefined
99
+ }
100
+
101
+ /** A set of labels of an instrument: its id on the stream, and a gauge's value. */
102
+ interface LabelSet {
103
+ id: number
104
+ instrument: Declared
105
+ labels: Record<string, string>
106
+ overflow: boolean
107
+ /** when a call last changed it */
108
+ touched: number
109
+ value: number | undefined
110
+ }
111
+
112
+ /** What a set of labels changed since the batch before. */
113
+ interface Change {
114
+ add: number | undefined
115
+ value: number | undefined
116
+ count: number
117
+ sum: number
118
+ longest: number
119
+ buckets: number[] | undefined
120
+ bins: Map<string, number> | undefined
121
+ }
122
+
123
+ interface Waiter {
124
+ resolve: () => void
125
+ reject: (err: Error) => void
126
+ }
127
+
128
+ /** A batch: its number in the session, its changes, and who waits for its answer. */
129
+ interface Batch {
130
+ number: number
131
+ changes: Map<LabelSet, Change>
132
+ flush: boolean
133
+ waiters: Waiter[]
134
+ /** the stream it went on, since it goes again on the next */
135
+ sentOn: Opened | undefined
136
+ }
137
+
138
+ /** A stream of the instruments, and what it declared. */
139
+ interface Opened {
140
+ stream: Stream
141
+ instruments: Set<number>
142
+ sets: Set<number>
143
+ }
144
+
145
+ /**
146
+ * The instruments of a store's handle, and what sends their changes. A store
147
+ * has one; `store.metrics` makes its instruments through it.
148
+ */
149
+ export class Instruments {
150
+ readonly #link: Link
151
+ readonly #session = randomBytes(16)
152
+ readonly #declared = new Map<string, Declared>()
153
+ readonly #byId = new Map<number, Declared>()
154
+ #pending = new Map<LabelSet, Change>()
155
+ #unanswered: Batch[] = []
156
+ #answered = 0
157
+ #next = 1
158
+ #ids = 0
159
+ #opened: Opened | undefined
160
+ #opening: Promise<Opened> | undefined
161
+ /** the sends in turn, so that batches leave in their order */
162
+ #pumping: Promise<void> = Promise.resolve()
163
+ #sending: ReturnType<typeof setInterval> | undefined
164
+ #reading: ReturnType<typeof setInterval> | undefined
165
+ #stopped = false
166
+ /** What the store refused, or a restart lost, since a flush last failed with it */
167
+ #unreported: Error | undefined
168
+ readonly #failures = new FailureLog('metrics instruments', sayOwn)
169
+
170
+ constructor(link: Link) {
171
+ this.#link = link
172
+ }
173
+
174
+ /**
175
+ * An instrument of name, made the first time, read by functions when
176
+ * `read` says so; another form or other options than it has are refused.
177
+ */
178
+ declare(name: string, form: Form, options: HistogramOptions, read = false): Declared {
179
+ const told = toldOf(name, form, options)
180
+ const held = this.#declared.get(name)
181
+ if (held !== undefined) {
182
+ if (held.form !== form || !sameOptions(held, told) || read !== (held.reads !== undefined)) {
183
+ throw new InvalidError(
184
+ `metrics ${name}: another instrument or other options than it has while the store is open`,
185
+ )
186
+ }
187
+ return held
188
+ }
189
+ const declared: Declared = {
190
+ ...told,
191
+ id: ++this.#ids,
192
+ name,
193
+ form,
194
+ sets: new Map(),
195
+ overflow: undefined,
196
+ refused: false,
197
+ reads: read ? new Map() : undefined,
198
+ }
199
+ this.#declared.set(name, declared)
200
+ this.#byId.set(declared.id, declared)
201
+ this.#start()
202
+ return declared
203
+ }
204
+
205
+ /**
206
+ * A gauge of name whose set of labels `read` gives every 15 seconds; each
207
+ * set has a function of its own, made once while the store is open.
208
+ */
209
+ reader(
210
+ name: string,
211
+ labels: Record<string, string>,
212
+ options: InstrumentOptions,
213
+ read: () => unknown,
214
+ ): void {
215
+ const declared = this.declare(name, 'gauge', options, true)
216
+ const key = keyOf(labels)
217
+ if (declared.reads?.has(key)) {
218
+ throw new InvalidError(
219
+ `metrics ${name}: a function reads its gauge of ${JSON.stringify(labels)} already while the store is open`,
220
+ )
221
+ }
222
+ declared.reads?.set(key, { labels, read, reading: false, failed: undefined })
223
+ }
224
+
225
+ /** The set of labels of an instrument, made the first time; past the sets it holds, its __overflow. */
226
+ setOf(instrument: Declared, labels: Record<string, string>): LabelSet {
227
+ const key = keyOf(labels)
228
+ const held = instrument.sets.get(key)
229
+ if (held !== undefined) {
230
+ return held
231
+ }
232
+ if (instrument.sets.size >= mostSets) {
233
+ instrument.overflow ??= this.#labelled(instrument, {}, true)
234
+ return instrument.overflow
235
+ }
236
+ const set = this.#labelled(instrument, labels, false)
237
+ instrument.sets.set(key, set)
238
+ return set
239
+ }
240
+
241
+ #labelled(instrument: Declared, labels: Record<string, string>, overflow: boolean): LabelSet {
242
+ return { id: ++this.#ids, instrument, labels, overflow, touched: Date.now(), value: undefined }
243
+ }
244
+
245
+ /** What calls changed of a set since the last batch, if any. */
246
+ pending(set: LabelSet): Change | undefined {
247
+ return this.#pending.get(set)
248
+ }
249
+
250
+ /** What a call changes of a set, kept for the next batch. */
251
+ change(set: LabelSet): Change {
252
+ set.touched = Date.now()
253
+ let change = this.#pending.get(set)
254
+ if (change === undefined) {
255
+ change = blank(set.instrument)
256
+ this.#pending.set(set, change)
257
+ }
258
+ return change
259
+ }
260
+
261
+ /**
262
+ * Sends what changed now and has the store write its totals; resolves
263
+ * once it has, and says whether there was anything to send it with. One
264
+ * the store cannot be reached for fails as a call does, and what it
265
+ * carried goes once the store can be. What the store refused since the
266
+ * flush before, and what a store that started again may have lost, fail
267
+ * it, each once.
268
+ */
269
+ async flush(): Promise<boolean> {
270
+ if (this.#declared.size === 0) {
271
+ return false
272
+ }
273
+ await this.#readGauges()
274
+ const done = new Promise<void>((resolve, reject) => {
275
+ const batches = this.#batches(true)
276
+ batches.at(-1)?.waiters.push({ resolve, reject })
277
+ })
278
+ void this.#pump()
279
+ await done
280
+ return true
281
+ }
282
+
283
+ /** Stops the timers, sends what is left, and ends the stream; the store's close calls it. */
284
+ async stop(): Promise<void> {
285
+ if (this.#stopped) {
286
+ return
287
+ }
288
+ this.#stopped = true
289
+ clearInterval(this.#sending)
290
+ clearInterval(this.#reading)
291
+ if (this.#declared.size === 0) {
292
+ return
293
+ }
294
+ try {
295
+ await this.flush()
296
+ } catch (err) {
297
+ this.#failures.failed(err)
298
+ }
299
+ await this.#opened?.stream.send(new Uint8Array(0), true).catch(() => {})
300
+ }
301
+
302
+ #start(): void {
303
+ if (this.#sending !== undefined || this.#stopped) {
304
+ return
305
+ }
306
+ this.#sending = setInterval(() => this.#tick(), sendEvery)
307
+ this.#sending.unref?.()
308
+ this.#reading = setInterval(() => void this.#readGauges(), readEvery)
309
+ this.#reading.unref?.()
310
+ }
311
+
312
+ #tick(): void {
313
+ const now = Date.now()
314
+ for (const instrument of this.#declared.values()) {
315
+ if (instrument.form === 'gauge') {
316
+ continue
317
+ }
318
+ for (const [key, set] of instrument.sets) {
319
+ if (now - set.touched >= idle && !this.#pending.has(set)) {
320
+ instrument.sets.delete(key)
321
+ }
322
+ }
323
+ }
324
+ if (this.#pending.size > 0 && this.#unanswered.length < mostUnanswered) {
325
+ this.#batches(false)
326
+ }
327
+ void this.#pump()
328
+ }
329
+
330
+ /** What waits, as batches of mostChanges changes each; with flush, the last asks the store to write. */
331
+ #batches(flush: boolean): Batch[] {
332
+ const made: Batch[] = []
333
+ const pending = [...this.#pending]
334
+ this.#pending = new Map()
335
+ for (let from = 0; from < pending.length || (flush && made.length === 0); from += mostChanges) {
336
+ const changes = new Map(pending.slice(from, from + mostChanges))
337
+ const last = from + mostChanges >= pending.length
338
+ made.push({
339
+ number: this.#next++,
340
+ changes,
341
+ flush: flush && last,
342
+ waiters: [],
343
+ sentOn: undefined,
344
+ })
345
+ }
346
+ this.#unanswered.push(...made)
347
+ return made
348
+ }
349
+
350
+ /** Sends every unanswered batch that has not gone on the stream open now, in order, a send at a time. */
351
+ #pump(): Promise<void> {
352
+ this.#pumping = this.#pumping.then(() => this.#sendWaiting())
353
+ return this.#pumping
354
+ }
355
+
356
+ // a connection lost on the way is dialled again, as a call's is; a store
357
+ // that cannot be dialled is tried again at the next tick, not at once
358
+ async #sendWaiting(): Promise<void> {
359
+ for (let tries = 0; !this.#sentAll(); tries++) {
360
+ try {
361
+ await this.#sendOn(await this.#stream())
362
+ this.#failures.succeeded()
363
+ return
364
+ } catch (err) {
365
+ if (tries < 2 && err instanceof ConnectionLostError) {
366
+ continue
367
+ }
368
+ this.#failures.failed(err)
369
+ this.#fail(err)
370
+ return
371
+ }
372
+ }
373
+ }
374
+
375
+ #sentAll(): boolean {
376
+ return this.#unanswered.every(
377
+ batch => batch.sentOn !== undefined && batch.sentOn === this.#opened,
378
+ )
379
+ }
380
+
381
+ async #sendOn(opened: Opened): Promise<void> {
382
+ for (const batch of this.#unanswered) {
383
+ if (batch.sentOn !== opened) {
384
+ batch.sentOn = opened
385
+ await opened.stream.send(encode(batch, opened), false)
386
+ }
387
+ }
388
+ }
389
+
390
+ /** Every flush waiting fails with err; the batches wait for the next send. */
391
+ #fail(err: unknown): void {
392
+ for (const batch of this.#unanswered) {
393
+ for (const waiter of batch.waiters.splice(0)) {
394
+ waiter.reject(errorOf(err))
395
+ }
396
+ }
397
+ }
398
+
399
+ /**
400
+ * The store refused the batch it was to apply next and would refuse it
401
+ * again: it is dropped, said once, and those after it are numbered on.
402
+ */
403
+ #drop(err: unknown): void {
404
+ const dropped = this.#unanswered.shift()
405
+ if (dropped === undefined) {
406
+ return
407
+ }
408
+ const error = messageOf(err)
409
+ sayOwn('warn', 'metrics instruments dropped a batch the store refused', {
410
+ changes: dropped.changes.size,
411
+ error,
412
+ })
413
+ this.#failWith(dropped.waiters, errorOf(err))
414
+ this.#renumber()
415
+ }
416
+
417
+ /** Fails the flushes that wait with err, or the next flush when none does. */
418
+ #failWith(waiters: Waiter[], err: Error): void {
419
+ if (waiters.length === 0) {
420
+ this.#unreported ??= err
421
+ }
422
+ for (const waiter of waiters.splice(0)) {
423
+ waiter.reject(err)
424
+ }
425
+ }
426
+
427
+ /** The stream open now, or one opened and resumed: the store says what it applied of the session. */
428
+ #stream(): Promise<Opened> {
429
+ if (this.#opened !== undefined && !this.#opened.stream.finished) {
430
+ return Promise.resolve(this.#opened)
431
+ }
432
+ this.#opening ??= this.#open().finally(() => {
433
+ this.#opening = undefined
434
+ })
435
+ return this.#opening
436
+ }
437
+
438
+ async #open(): Promise<Opened> {
439
+ const connection = await this.#link.connection()
440
+ const body = MetricsSession.encode({ id: this.#session, after: this.#answered })
441
+ const stream = await connection.session.open(methods['metrics.instruments'], body, false)
442
+ const head = await stream.next()
443
+ const opened: Opened = { stream, instruments: new Set(), sets: new Set() }
444
+ this.#resume(MetricsAck.decode(head.body))
445
+ this.#opened = opened
446
+ void this.#listen(opened)
447
+ return opened
448
+ }
449
+
450
+ /**
451
+ * Takes up where the store is: a batch it applied is done; one sent that
452
+ * it may have applied, when it knows no session, is dropped rather than
453
+ * counted twice, and those never sent are numbered on. Every gauge's value
454
+ * goes again, since a gauge left with the last connection.
455
+ */
456
+ #resume(ack: ReturnType<typeof MetricsAck.decode>): void {
457
+ const applied = ack.applied ?? 0
458
+ if (ack.known === true) {
459
+ this.#answer(ack)
460
+ } else {
461
+ const lost = this.#unanswered.filter(batch => batch.sentOn !== undefined)
462
+ this.#unanswered = this.#unanswered.filter(batch => batch.sentOn === undefined)
463
+ if (lost.length > 0) {
464
+ const changes = lost.reduce((n, batch) => n + batch.changes.size, 0)
465
+ sayOwn(
466
+ 'warn',
467
+ 'metrics instruments dropped what a store that started again may have counted',
468
+ {
469
+ batches: lost.length,
470
+ changes,
471
+ },
472
+ )
473
+ }
474
+ if (lost.length > 0) {
475
+ const unknown = new OutcomeUnknownError(
476
+ `the store started again: ${lost.length} batches of instruments it may have counted before it stopped are dropped rather than counted twice`,
477
+ )
478
+ this.#failWith(
479
+ lost.flatMap(batch => batch.waiters),
480
+ unknown,
481
+ )
482
+ }
483
+ this.#answered = applied
484
+ this.#renumber()
485
+ }
486
+ for (const instrument of this.#declared.values()) {
487
+ for (const set of instrument.form === 'gauge' ? instrument.sets.values() : []) {
488
+ if (set.value !== undefined) {
489
+ this.change(set).value = set.value
490
+ }
491
+ }
492
+ }
493
+ }
494
+
495
+ /** Numbers what waits after the last batch the store answered. */
496
+ #renumber(): void {
497
+ this.#next = this.#answered + 1
498
+ for (const batch of this.#unanswered) {
499
+ batch.number = this.#next++
500
+ }
501
+ }
502
+
503
+ /**
504
+ * Reads the store's answers on a stream until it ends. What it had not
505
+ * answered goes again at once on another, but for a batch it refused.
506
+ */
507
+ async #listen(opened: Opened): Promise<void> {
508
+ try {
509
+ for (;;) {
510
+ const event = await opened.stream.next()
511
+ opened.stream.consumed(event.body.length)
512
+ this.#answer(MetricsAck.decode(event.body))
513
+ if (event.end) {
514
+ return
515
+ }
516
+ }
517
+ } catch (err) {
518
+ if (!this.#stopped) {
519
+ this.#failures.failed(err)
520
+ }
521
+ if (err instanceof InvalidError && !this.#stopped) {
522
+ this.#drop(err)
523
+ }
524
+ } finally {
525
+ if (this.#opened === opened) {
526
+ this.#opened = undefined
527
+ }
528
+ if (!this.#stopped && this.#unanswered.length > 0) {
529
+ void this.#pump()
530
+ }
531
+ }
532
+ }
533
+
534
+ /** An answer: the instruments the store refused, then the batches it applied. */
535
+ #answer(ack: ReturnType<typeof MetricsAck.decode>): void {
536
+ for (const refused of ack.refused ?? []) {
537
+ const instrument = this.#byId.get(refused.instrument ?? 0)
538
+ if (instrument !== undefined && !instrument.refused) {
539
+ instrument.refused = true
540
+ const error = refused.error ?? ''
541
+ sayOwn('warn', 'metrics instrument refused', { instrument: instrument.name, error })
542
+ this.#unreported ??= new InvalidError(
543
+ `metrics ${instrument.name}: the store refused it: ${error}`,
544
+ )
545
+ }
546
+ }
547
+ const applied = ack.applied ?? 0
548
+ this.#answered = Math.max(this.#answered, applied)
549
+ this.#settle(applied)
550
+ }
551
+
552
+ /** The batches the store applied, each done: a flush waiting resolves, or fails with what it was not told yet. */
553
+ #settle(applied: number): void {
554
+ while ((this.#unanswered[0]?.number ?? Infinity) <= applied) {
555
+ const done = this.#unanswered.shift()
556
+ for (const waiter of done?.waiters ?? []) {
557
+ const failed = this.#unreported
558
+ this.#unreported = undefined
559
+ if (failed === undefined) {
560
+ waiter.resolve()
561
+ } else {
562
+ waiter.reject(failed)
563
+ }
564
+ }
565
+ }
566
+ }
567
+
568
+ /** Calls each gauge's function whose last call ended; a value it gives is sent with the next batch. */
569
+ async #readGauges(): Promise<void> {
570
+ const reads: Promise<void>[] = []
571
+ for (const instrument of this.#declared.values()) {
572
+ for (const gauge of instrument.refused ? [] : (instrument.reads?.values() ?? [])) {
573
+ if (!gauge.reading) {
574
+ reads.push(this.#readGauge(instrument, gauge))
575
+ }
576
+ }
577
+ }
578
+ await Promise.all(reads)
579
+ }
580
+
581
+ async #readGauge(instrument: Declared, gauge: GaugeRead): Promise<void> {
582
+ gauge.reading = true
583
+ try {
584
+ const value = await gauge.read()
585
+ if (typeof value !== 'number' || !Number.isFinite(value)) {
586
+ throw new InvalidError(`a gauge's function gave ${String(value)}, where a finite number is`)
587
+ }
588
+ const set = this.setOf(instrument, gauge.labels)
589
+ set.value = value
590
+ this.change(set).value = value
591
+ gauge.failed = undefined
592
+ } catch (err) {
593
+ const error = messageOf(err)
594
+ if (error !== gauge.failed) {
595
+ const labels = JSON.stringify(gauge.labels)
596
+ sayOwn('warn', 'metrics gauge read failed', { instrument: instrument.name, labels, error })
597
+ gauge.failed = error
598
+ }
599
+ } finally {
600
+ gauge.reading = false
601
+ }
602
+ }
603
+ }
604
+
605
+ function errorOf(err: unknown): Error {
606
+ return err instanceof Error ? err : new UnavailableError(String(err))
607
+ }
608
+
609
+ /** A change of nothing yet, with room for an instrument's buckets. */
610
+ function blank(instrument: Declared): Change {
611
+ const buckets =
612
+ instrument.bounds === undefined ? undefined : new Array(instrument.bounds.length + 1).fill(0)
613
+ const bins = instrument.exponential ? new Map<string, number>() : undefined
614
+ return { add: undefined, value: undefined, count: 0, sum: 0, longest: 0, buckets, bins }
615
+ }
616
+
617
+ /** A batch's bytes for a stream: the instruments and sets it names that the stream has not declared. */
618
+ function encode(batch: Batch, opened: Opened): Uint8Array {
619
+ const instruments = []
620
+ const sets = []
621
+ const changes = []
622
+ for (const [set, change] of batch.changes) {
623
+ const instrument = set.instrument
624
+ if (instrument.refused) {
625
+ continue
626
+ }
627
+ if (!opened.instruments.has(instrument.id)) {
628
+ opened.instruments.add(instrument.id)
629
+ instruments.push({
630
+ number: instrument.id,
631
+ name: instrument.name,
632
+ form: instrument.form,
633
+ unit: instrument.unit,
634
+ help: instrument.help,
635
+ keep: instrument.keep,
636
+ bounds: instrument.bounds,
637
+ exponential: instrument.exponential,
638
+ })
639
+ }
640
+ if (!opened.sets.has(set.id)) {
641
+ opened.sets.add(set.id)
642
+ sets.push({
643
+ number: set.id,
644
+ instrument: instrument.id,
645
+ labels: set.labels,
646
+ overflow: set.overflow,
647
+ })
648
+ }
649
+ changes.push({
650
+ set: set.id,
651
+ add: change.add,
652
+ value: change.value,
653
+ count: change.count,
654
+ sum: change.sum,
655
+ longest: change.longest,
656
+ buckets: change.buckets,
657
+ bins: change.bins === undefined ? undefined : Object.fromEntries(change.bins),
658
+ })
659
+ }
660
+ return MetricsBatch.encode({
661
+ number: batch.number,
662
+ instruments,
663
+ sets,
664
+ changes,
665
+ flush: batch.flush,
666
+ })
667
+ }
668
+
669
+ /** What an instrument was told, checked. */
670
+ function toldOf(
671
+ name: string,
672
+ form: Form,
673
+ options: HistogramOptions,
674
+ ): Pick<Declared, 'unit' | 'help' | 'keep' | 'bounds' | 'exponential'> {
675
+ checkName(name)
676
+ if (options.unit !== undefined && bytes(options.unit) > 32) {
677
+ throw new InvalidError(`metrics ${name}: a unit of 32 bytes at most`)
678
+ }
679
+ if (options.help !== undefined && bytes(options.help) > 1024) {
680
+ throw new InvalidError(`metrics ${name}: a help of 1024 bytes at most`)
681
+ }
682
+ const keep = options.keep === undefined ? undefined : ms(options.keep)
683
+ if (keep === 0) {
684
+ throw new InvalidError(`metrics ${name}: keep 0, where a name keeps its samples 1ms at least`)
685
+ }
686
+ const buckets = options.buckets
687
+ if (buckets !== undefined && form !== 'timer' && form !== 'histogram') {
688
+ throw new InvalidError(`metrics ${name}: buckets are a timer's or a histogram's`)
689
+ }
690
+ if (buckets === 'exponential') {
691
+ return { unit: options.unit, help: options.help, keep, bounds: undefined, exponential: true }
692
+ }
693
+ return {
694
+ unit: options.unit,
695
+ help: options.help,
696
+ keep,
697
+ bounds: buckets === undefined ? undefined : checkedBounds(name, buckets),
698
+ exponential: false,
699
+ }
700
+ }
701
+
702
+ function checkedBounds(name: string, bounds: readonly number[]): number[] {
703
+ if (bounds.length === 0 || bounds.length > mostBounds) {
704
+ throw new InvalidError(`metrics ${name}: 1 to ${mostBounds} bounds`)
705
+ }
706
+ bounds.forEach((bound, i) => {
707
+ if (!(Number.isFinite(bound) && bound >= 0) || (i > 0 && !(bound > bounds[i - 1]!))) {
708
+ throw new InvalidError(`metrics ${name}: bounds of 0 or more, each above the one before`)
709
+ }
710
+ })
711
+ return [...bounds]
712
+ }
713
+
714
+ function sameOptions(held: Declared, told: ReturnType<typeof toldOf>): boolean {
715
+ return (
716
+ held.unit === told.unit &&
717
+ held.help === told.help &&
718
+ held.keep === told.keep &&
719
+ held.exponential === told.exponential &&
720
+ JSON.stringify(held.bounds) === JSON.stringify(told.bounds)
721
+ )
722
+ }
723
+
724
+ function checkName(name: string): void {
725
+ if (name === '' || !name.isWellFormed() || bytes(name) > 4096) {
726
+ throw new InvalidError(`metrics name ${JSON.stringify(name)}: 1 to 4096 bytes of UTF-8`)
727
+ }
728
+ }
729
+
730
+ const encoder = new TextEncoder()
731
+
732
+ function bytes(text: string): number {
733
+ return encoder.encode(text).length
734
+ }
735
+
736
+ /** Labels as an instrument keeps them: text, a number or a boolean as its text, none of the store's own. */
737
+ export function labelsOf(given: InstrumentLabels, timed: boolean): Record<string, string> {
738
+ const labels: Record<string, string> = {}
739
+ for (const [label, value] of Object.entries(given)) {
740
+ if (label === '' || bytes(label) > 256) {
741
+ throw new InvalidError(`label ${JSON.stringify(label)}: a name of 1 to 256 bytes`)
742
+ }
743
+ if (label.startsWith('__')) {
744
+ throw new InvalidError(`label ${label}: a name beginning with __ is the store's own`)
745
+ }
746
+ if (timed && label === 'le') {
747
+ throw new InvalidError(`label le: a timer's _bucket series give it`)
748
+ }
749
+ if (typeof value !== 'string' && typeof value !== 'number' && typeof value !== 'boolean') {
750
+ throw new InvalidError(
751
+ `label ${label}: a value of text, a number or a boolean, where ${String(value)} is`,
752
+ )
753
+ }
754
+ const text = String(value)
755
+ if (bytes(text) > 4096) {
756
+ throw new InvalidError(`label ${label}: a value of 4096 bytes at most`)
757
+ }
758
+ labels[label] = text
759
+ }
760
+ return labels
761
+ }
762
+
763
+ function keyOf(labels: Record<string, string>): string {
764
+ return JSON.stringify(Object.entries(labels).sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)))
765
+ }
766
+
767
+ /**
768
+ * What a send would carry of a set, refused at the call that would take it
769
+ * past the largest number, which the store would refuse with nobody told.
770
+ */
771
+ function sums(declared: Declared, sum: number): void {
772
+ if (!Number.isFinite(sum)) {
773
+ throw new InvalidError(`metrics ${declared.name}: a sum past the largest number a send carries`)
774
+ }
775
+ }
776
+
777
+ /** A finite number an instrument takes, or invalid at once where the program made it. */
778
+ function finite(name: string, what: string, value: number): number {
779
+ if (typeof value !== 'number' || !Number.isFinite(value)) {
780
+ throw new InvalidError(`metrics ${name}: ${what} ${String(value)}, where a finite number is`)
781
+ }
782
+ return value
783
+ }
784
+
785
+ /** A total that only goes up, which the store keeps: processes counting into one series count into one total. */
786
+ export class Counter<L extends string = string> {
787
+ readonly #instruments: Instruments
788
+ readonly #declared: Declared
789
+ readonly #labels: Record<string, string>
790
+
791
+ constructor(instruments: Instruments, declared: Declared, labels: Record<string, string>) {
792
+ this.#instruments = instruments
793
+ this.#declared = declared
794
+ this.#labels = labels
795
+ }
796
+
797
+ /** The same counter with more labels, a series of its own; the same labels in any order are one. */
798
+ with(labels: InstrumentLabels<L>): Counter<L> {
799
+ return new Counter(this.#instruments, this.#declared, {
800
+ ...this.#labels,
801
+ ...labelsOf(labels, false),
802
+ })
803
+ }
804
+
805
+ inc(): void {
806
+ this.add(1)
807
+ }
808
+
809
+ /** Adds n, a finite number of 0 or more; add(0) makes the set of labels known with a total of zero. */
810
+ add(n: number): void {
811
+ if (finite(this.#declared.name, 'a counter adds', n) < 0) {
812
+ throw new InvalidError(`metrics ${this.#declared.name}: a counter adds ${n}; it only grows`)
813
+ }
814
+ if (this.#declared.refused) {
815
+ return
816
+ }
817
+ const set = this.#instruments.setOf(this.#declared, this.#labels)
818
+ const added = (this.#instruments.pending(set)?.add ?? 0) + n
819
+ sums(this.#declared, added)
820
+ this.#instruments.change(set).add = added
821
+ }
822
+ }
823
+
824
+ /** A value that goes up and down, written while this program is connected. */
825
+ export class Gauge<L extends string = string> {
826
+ readonly #instruments: Instruments
827
+ readonly #declared: Declared
828
+ readonly #labels: Record<string, string>
829
+
830
+ constructor(instruments: Instruments, declared: Declared, labels: Record<string, string>) {
831
+ this.#instruments = instruments
832
+ this.#declared = declared
833
+ this.#labels = labels
834
+ }
835
+
836
+ with(labels: InstrumentLabels<L>): Gauge<L> {
837
+ return new Gauge(this.#instruments, this.#declared, {
838
+ ...this.#labels,
839
+ ...labelsOf(labels, false),
840
+ })
841
+ }
842
+
843
+ set(value: number): void {
844
+ this.#to(finite(this.#declared.name, 'a gauge set to', value))
845
+ }
846
+
847
+ /** Moves it by n, from zero when it was never set. */
848
+ add(n: number): void {
849
+ finite(this.#declared.name, 'a gauge moved by', n)
850
+ const set = this.#instruments.setOf(this.#declared, this.#labels)
851
+ this.#to(finite(this.#declared.name, 'a gauge moved to', (set.value ?? 0) + n))
852
+ }
853
+
854
+ inc(): void {
855
+ this.add(1)
856
+ }
857
+
858
+ dec(): void {
859
+ this.add(-1)
860
+ }
861
+
862
+ #to(value: number): void {
863
+ if (this.#declared.refused) {
864
+ return
865
+ }
866
+ const set = this.#instruments.setOf(this.#declared, this.#labels)
867
+ set.value = value
868
+ this.#instruments.change(set).value = value
869
+ }
870
+ }
871
+
872
+ /** Counts a duration or a value into a timer's or a histogram's change: its count, sum, longest and bucket. */
873
+ function measured(
874
+ instruments: Instruments,
875
+ declared: Declared,
876
+ labels: Record<string, string>,
877
+ value: number,
878
+ ): void {
879
+ if (declared.refused) {
880
+ return
881
+ }
882
+ const set = instruments.setOf(declared, labels)
883
+ sums(declared, (instruments.pending(set)?.sum ?? 0) + value)
884
+ const change = instruments.change(set)
885
+ change.count++
886
+ change.sum += value
887
+ change.longest = change.count === 1 ? value : Math.max(change.longest, value)
888
+ if (change.buckets !== undefined && declared.bounds !== undefined) {
889
+ const bucket = declared.bounds.findIndex(bound => value <= bound)
890
+ change.buckets[bucket < 0 ? declared.bounds.length : bucket]! += 1
891
+ }
892
+ if (change.bins !== undefined) {
893
+ const bin = binOf(value)
894
+ change.bins.set(bin, (change.bins.get(bin) ?? 0) + 1)
895
+ }
896
+ }
897
+
898
+ /** How many things took how long: their count, sum and longest in milliseconds, and their spread over its buckets. */
899
+ export class Timer<L extends string = string> {
900
+ readonly #instruments: Instruments
901
+ readonly #declared: Declared
902
+ readonly #labels: Record<string, string>
903
+
904
+ constructor(instruments: Instruments, declared: Declared, labels: Record<string, string>) {
905
+ this.#instruments = instruments
906
+ this.#declared = declared
907
+ this.#labels = labels
908
+ }
909
+
910
+ with(labels: InstrumentLabels<L>): Timer<L> {
911
+ return new Timer(this.#instruments, this.#declared, {
912
+ ...this.#labels,
913
+ ...labelsOf(labels, true),
914
+ })
915
+ }
916
+
917
+ /** Records a duration measured elsewhere: milliseconds, as performance.now() counts them, or text such as '250ms'. */
918
+ record(duration: Duration): void {
919
+ const millis =
920
+ typeof duration === 'number'
921
+ ? finite(this.#declared.name, 'a duration of', duration)
922
+ : ms(duration)
923
+ if (millis < 0) {
924
+ throw new InvalidError(
925
+ `metrics ${this.#declared.name}: a duration of ${millis}; a duration is never negative`,
926
+ )
927
+ }
928
+ measured(this.#instruments, this.#declared, this.#labels, millis)
929
+ }
930
+
931
+ /**
932
+ * Runs fn, awaits what it returns, returns or throws what it does, and
933
+ * records how long it took either way.
934
+ *
935
+ * const user = await latency.with({ route: '/users' }).measure(() => users.get(id))
936
+ */
937
+ async measure<R>(fn: () => R | Promise<R>): Promise<R> {
938
+ const started = performance.now()
939
+ try {
940
+ return await fn()
941
+ } finally {
942
+ this.record(performance.now() - started)
943
+ }
944
+ }
945
+
946
+ /** A timing that records when it stops, for a span that is no one function. */
947
+ start(): Timing {
948
+ return new Timing(this)
949
+ }
950
+ }
951
+
952
+ /** A span a timer measures from its start; stop records it, once. */
953
+ export class Timing {
954
+ readonly #timer: Timer
955
+ readonly #started = performance.now()
956
+ #stopped = false
957
+
958
+ constructor(timer: Timer) {
959
+ this.#timer = timer
960
+ }
961
+
962
+ /** Records how long since the timing started, the first time it is called; its milliseconds. */
963
+ stop(): number {
964
+ const took = performance.now() - this.#started
965
+ if (!this.#stopped) {
966
+ this.#stopped = true
967
+ this.#timer.record(took)
968
+ }
969
+ return took
970
+ }
971
+ }
972
+
973
+ /** A timer of numbers that are not durations: a response's size in bytes. */
974
+ export class Histogram<L extends string = string> {
975
+ readonly #instruments: Instruments
976
+ readonly #declared: Declared
977
+ readonly #labels: Record<string, string>
978
+
979
+ constructor(instruments: Instruments, declared: Declared, labels: Record<string, string>) {
980
+ this.#instruments = instruments
981
+ this.#declared = declared
982
+ this.#labels = labels
983
+ }
984
+
985
+ with(labels: InstrumentLabels<L>): Histogram<L> {
986
+ return new Histogram(this.#instruments, this.#declared, {
987
+ ...this.#labels,
988
+ ...labelsOf(labels, true),
989
+ })
990
+ }
991
+
992
+ /** Records a value of 0 or more. */
993
+ record(value: number): void {
994
+ if (finite(this.#declared.name, 'a value of', value) < 0) {
995
+ throw new InvalidError(
996
+ `metrics ${this.#declared.name}: a value of ${value}; the lowest bucket starts at 0`,
997
+ )
998
+ }
999
+ measured(this.#instruments, this.#declared, this.#labels, value)
1000
+ }
1001
+ }
1002
+
1003
+ /** Upper bounds from start, each factor times the one before: exponentialBuckets(1, 2, 14) is 1, 2, 4 … 8192. */
1004
+ export function exponentialBuckets(start: number, factor: number, count: number): number[] {
1005
+ if (!(start > 0 && factor > 1 && Number.isInteger(count) && count >= 1 && count <= mostBounds)) {
1006
+ throw new InvalidError(
1007
+ `exponentialBuckets(${start}, ${factor}, ${count}): a start above 0, a factor above 1, 1 to ${mostBounds} bounds`,
1008
+ )
1009
+ }
1010
+ const bounds = [start]
1011
+ while (bounds.length < count) {
1012
+ bounds.push(bounds.at(-1)! * factor)
1013
+ }
1014
+ return bounds
1015
+ }
1016
+
1017
+ /**
1018
+ * The fractions of Prometheus' native grid for schema 3, 2^(j/8 − 1), as its
1019
+ * table writes them: three a unit in the last place below the nearest float,
1020
+ * so that a value on a bound falls where Prometheus and the store put it.
1021
+ */
1022
+ const fractions = [
1023
+ 0.5, 0.5452538663326288, 0.5946035575013605, 0.6484197773255048, 0.7071067811865475,
1024
+ 0.7711054127039704, 0.8408964152537144, 0.9170040432046711,
1025
+ ]
1026
+
1027
+ const halves = new DataView(new ArrayBuffer(8))
1028
+
1029
+ /** The interval of the store's grid a value of 0 or more falls in, by its label: '3:13', or 'zero'. */
1030
+ export function binOf(value: number): string {
1031
+ if (value === 0) {
1032
+ return 'zero'
1033
+ }
1034
+ // a subnormal falls in the lowest interval, whose bound is the least normal float
1035
+ halves.setFloat64(0, Math.max(value, 2.2250738585072014e-308))
1036
+ const high = halves.getUint32(0)
1037
+ const exponent = ((high >>> 20) & 0x7ff) - 1022
1038
+ halves.setUint32(0, (high & ~(0x7ff << 20)) | (1022 << 20))
1039
+ const fraction = halves.getFloat64(0)
1040
+ const within = fractions.findIndex(bound => bound >= fraction)
1041
+ return `3:${(within < 0 ? 8 : within) + (exponent - 1) * 8}`
1042
+ }