@tinyshed/tinystore 0.1.0-rc.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (100) hide show
  1. package/LICENSE +202 -0
  2. package/README.md +262 -0
  3. package/bin/tinystore.js +50 -0
  4. package/dist/background.d.ts +30 -0
  5. package/dist/background.js +93 -0
  6. package/dist/blobs.d.ts +98 -0
  7. package/dist/blobs.js +299 -0
  8. package/dist/cancel.d.ts +14 -0
  9. package/dist/cancel.js +22 -0
  10. package/dist/clock.d.ts +21 -0
  11. package/dist/clock.js +35 -0
  12. package/dist/config.d.ts +60 -0
  13. package/dist/config.js +417 -0
  14. package/dist/connection.d.ts +78 -0
  15. package/dist/connection.js +405 -0
  16. package/dist/console.d.ts +48 -0
  17. package/dist/console.js +195 -0
  18. package/dist/errors.d.ts +103 -0
  19. package/dist/errors.js +158 -0
  20. package/dist/handles.d.ts +28 -0
  21. package/dist/handles.js +37 -0
  22. package/dist/index.d.ts +21 -0
  23. package/dist/index.js +7 -0
  24. package/dist/jobs.d.ts +304 -0
  25. package/dist/jobs.js +684 -0
  26. package/dist/kv.d.ts +283 -0
  27. package/dist/kv.js +815 -0
  28. package/dist/limiter.d.ts +45 -0
  29. package/dist/limiter.js +68 -0
  30. package/dist/logger.d.ts +96 -0
  31. package/dist/logger.js +243 -0
  32. package/dist/metrics.d.ts +249 -0
  33. package/dist/metrics.js +610 -0
  34. package/dist/once.d.ts +31 -0
  35. package/dist/once.js +116 -0
  36. package/dist/quota.d.ts +44 -0
  37. package/dist/quota.js +98 -0
  38. package/dist/records.d.ts +182 -0
  39. package/dist/records.js +346 -0
  40. package/dist/runtime/bun.d.ts +2 -0
  41. package/dist/runtime/bun.js +164 -0
  42. package/dist/runtime/node.d.ts +2 -0
  43. package/dist/runtime/node.js +157 -0
  44. package/dist/runtime/shared.d.ts +25 -0
  45. package/dist/runtime/shared.js +45 -0
  46. package/dist/runtime.d.ts +48 -0
  47. package/dist/runtime.js +4 -0
  48. package/dist/schema.d.ts +39 -0
  49. package/dist/schema.js +21 -0
  50. package/dist/session.d.ts +120 -0
  51. package/dist/session.js +546 -0
  52. package/dist/sql.d.ts +121 -0
  53. package/dist/sql.js +342 -0
  54. package/dist/store.d.ts +103 -0
  55. package/dist/store.js +153 -0
  56. package/dist/time.d.ts +35 -0
  57. package/dist/time.js +44 -0
  58. package/dist/trace.d.ts +18 -0
  59. package/dist/trace.js +22 -0
  60. package/dist/wire/codec.d.ts +90 -0
  61. package/dist/wire/codec.js +383 -0
  62. package/dist/wire/frame.d.ts +41 -0
  63. package/dist/wire/frame.js +102 -0
  64. package/dist/wire/messages.d.ts +1403 -0
  65. package/dist/wire/messages.js +472 -0
  66. package/dist/wire/msgpack.d.ts +93 -0
  67. package/dist/wire/msgpack.js +680 -0
  68. package/package.json +40 -0
  69. package/src/background.ts +108 -0
  70. package/src/blobs.ts +437 -0
  71. package/src/cancel.ts +26 -0
  72. package/src/clock.ts +45 -0
  73. package/src/config.ts +501 -0
  74. package/src/connection.ts +512 -0
  75. package/src/console.ts +228 -0
  76. package/src/errors.ts +211 -0
  77. package/src/handles.ts +68 -0
  78. package/src/index.ts +92 -0
  79. package/src/jobs.ts +963 -0
  80. package/src/kv.ts +1093 -0
  81. package/src/limiter.ts +99 -0
  82. package/src/logger.ts +326 -0
  83. package/src/metrics.ts +896 -0
  84. package/src/once.ts +142 -0
  85. package/src/quota.ts +137 -0
  86. package/src/records.ts +537 -0
  87. package/src/runtime/bun.ts +190 -0
  88. package/src/runtime/node.ts +177 -0
  89. package/src/runtime/shared.ts +61 -0
  90. package/src/runtime.ts +57 -0
  91. package/src/schema.ts +69 -0
  92. package/src/session.ts +665 -0
  93. package/src/sql.ts +478 -0
  94. package/src/store.ts +219 -0
  95. package/src/time.ts +87 -0
  96. package/src/trace.ts +32 -0
  97. package/src/wire/codec.ts +445 -0
  98. package/src/wire/frame.ts +139 -0
  99. package/src/wire/messages.ts +562 -0
  100. package/src/wire/msgpack.ts +712 -0
package/src/metrics.ts ADDED
@@ -0,0 +1,896 @@
1
+ // Series of samples in metrics.db, as Go's metrics package keeps them: a
2
+ // sample is kept bit for bit, a range read exactly, an aggregate computed
3
+ // exactly and rounded once. Instruments live in the SDK and are ingested
4
+ // every flush.
5
+
6
+ import { FailureLog, messageOf } from './background.ts'
7
+ import { download, type Link } from './connection.ts'
8
+ import { type Code, errorOf, InvalidError, type LimitError, TinystoreError } from './errors.ts'
9
+ import { sayOwn } from './logger.ts'
10
+ import { type Duration, ms, type Time, unixMs } from './time.ts'
11
+ import { byCodePoint } from './wire/codec.ts'
12
+ import {
13
+ MetricsBatch,
14
+ MetricsBuckets,
15
+ MetricsDropped,
16
+ MetricsLabels,
17
+ MetricsPlan,
18
+ MetricsRange,
19
+ MetricsSeries,
20
+ methods,
21
+ } from './wire/messages.ts'
22
+
23
+ export type SeriesKind = 'gauge' | 'counter'
24
+
25
+ /**
26
+ * What tells the series of one name apart: a host, a route, a status. A label
27
+ * whose name begins with __ is the store's own, and refused.
28
+ */
29
+ export type Labels = Record<string, string>
30
+
31
+ export interface SeriesInput {
32
+ /** the metric: 'http_requests_total' */
33
+ name: string
34
+ kind: SeriesKind
35
+ labels?: Labels
36
+ /** samples as [time, value] pairs, or as columns: unix milliseconds and their values */
37
+ samples:
38
+ | readonly (readonly [time: Time, value: number])[]
39
+ | { times: readonly number[] | BigInt64Array; values: readonly number[] | Float64Array }
40
+ }
41
+
42
+ export interface Series {
43
+ name: string
44
+ kind: SeriesKind
45
+ labels: Labels
46
+ /** unix milliseconds, in time order */
47
+ times: number[]
48
+ /** the values, their bits as they were ingested: -0 and a NaN's payload survive */
49
+ values: Float64Array
50
+ }
51
+
52
+ /**
53
+ * The series of a name, of labels, or of both, each matched exactly, and
54
+ * their samples over the last `since` or from `from` to `to`.
55
+ *
56
+ * ```ts
57
+ * { name: 'cpu', since: '1h' } // the last hour of every cpu series
58
+ * { match: { host: 'web-1' }, from: start, to: end } // every series of web-1
59
+ * ```
60
+ */
61
+ /**
62
+ * What a label's value must be beyond equality, made by oneOf, noneOf or
63
+ * prefix: an object of the SDK's own, so that a stored value is never read as
64
+ * a query.
65
+ */
66
+ export class Condition {
67
+ readonly kind: 'one_of' | 'none_of' | 'prefix'
68
+ readonly values: readonly string[]
69
+
70
+ constructor(kind: Condition['kind'], values: readonly string[]) {
71
+ this.kind = kind
72
+ this.values = values
73
+ }
74
+ }
75
+
76
+ /** The label is one of the values. */
77
+ export function oneOf(...values: string[]): Condition {
78
+ return new Condition('one_of', values)
79
+ }
80
+
81
+ /** The label is none of the values, or the series has none. */
82
+ export function noneOf(...values: string[]): Condition {
83
+ return new Condition('none_of', values)
84
+ }
85
+
86
+ /** The label's value begins with the prefix. */
87
+ export function prefix(start: string): Condition {
88
+ return new Condition('prefix', [start])
89
+ }
90
+
91
+ /** What a read or an aggregate would spend, each use beside its limit. */
92
+ export interface Plan {
93
+ series: number
94
+ blocks: number
95
+ /** the blocks an aggregate answers from their summaries, decoding none of their samples */
96
+ summarized: number
97
+ bytes: number
98
+ decoded: number
99
+ limits: { series: number; blocks: number; bytes: number; decoded: number; answered: number }
100
+ /** the limit the call would reach, which it would end with; undefined when it fits */
101
+ stops: LimitError | undefined
102
+ }
103
+
104
+ export interface Range {
105
+ name?: string
106
+ /** labels a series holds, each exactly */
107
+ match?: Labels
108
+ /**
109
+ * labels under a condition: `{ status: oneOf('500', '502'), env: noneOf('dev'), host: prefix('api-') }`;
110
+ * a plain value is equality. A range of noneOf alone is refused, since it would scan every series.
111
+ */
112
+ where?: Record<string, Condition | string>
113
+ /** the span before now the range covers: '1h', '15m', or milliseconds */
114
+ since?: Duration
115
+ /** the range's start, included; the oldest sample kept when absent */
116
+ from?: Time
117
+ /** excluded; the open end when absent */
118
+ to?: Time
119
+ /** each narrows the server's: series matched, blocks decoded, bytes fetched, samples decoded, answered */
120
+ limits?: { series?: number; blocks?: number; bytes?: number; decoded?: number; answered?: number }
121
+ }
122
+
123
+ /**
124
+ * Each computed exactly and rounded once, a group's too: avg is the mean of
125
+ * every sample, rate a counter's increase a second, delta a gauge's last
126
+ * sample less its first.
127
+ */
128
+ export type AggregateOp = 'count' | 'sum' | 'min' | 'max' | 'avg' | 'increase' | 'rate' | 'delta'
129
+
130
+ /**
131
+ * An aggregate's buckets and, to join series, what groups them: by these
132
+ * labels, or without them, each group one result. `by: []` joins every series
133
+ * of a name. A group never joins two names or two kinds.
134
+ */
135
+ export interface AggregateRange extends Range {
136
+ width: Duration
137
+ op: AggregateOp
138
+ by?: string[]
139
+ without?: string[]
140
+ }
141
+
142
+ export interface Bucket {
143
+ from: Date
144
+ to: Date
145
+ /** the samples it counted */
146
+ count: number
147
+ /** the resets among them, a counter's */
148
+ resets: number
149
+ /** computed exactly and rounded once */
150
+ value: number
151
+ /** the value overflowed to an infinity */
152
+ overflow: boolean
153
+ /** retention cut the bucket, which counted only its samples from the cutoff on */
154
+ partial: boolean
155
+ }
156
+
157
+ export interface Aggregate {
158
+ name: string
159
+ kind: SeriesKind
160
+ labels: Labels
161
+ /** only buckets holding samples */
162
+ buckets: Bucket[]
163
+ }
164
+
165
+ const openEnd = 0x7fff_ffff_ffff_ffffn
166
+
167
+ /** The label the wire carries a series' name as, the store's own spelling. */
168
+ const wireName = '__name__'
169
+
170
+ /** A series' name and labels as the wire spells them, its name among its labels. */
171
+ function wireLabels(name: string | undefined, labels: Labels | undefined): Labels {
172
+ const spelled: Labels = {}
173
+ for (const [key, value] of Object.entries(labels ?? {})) {
174
+ if (key.startsWith('__')) {
175
+ throw new InvalidError(`the label ${key}: a name beginning with __ is the store's own`)
176
+ }
177
+ spelled[key] = value
178
+ }
179
+ if (name !== undefined) {
180
+ spelled[wireName] = name
181
+ }
182
+ return spelled
183
+ }
184
+
185
+ /** A series as the wire spelled it: its name apart from its labels. */
186
+ function namedSeries(spelled: Labels | undefined): { name: string; labels: Labels } {
187
+ const { [wireName]: name = '', ...labels } = spelled ?? {}
188
+ return { name, labels }
189
+ }
190
+
191
+ function rangeOf(
192
+ r: Range,
193
+ extra?: {
194
+ width?: number
195
+ op?: AggregateOp
196
+ by?: string[] | undefined
197
+ without?: string[] | undefined
198
+ },
199
+ ): Parameters<typeof MetricsRange.encode>[0] {
200
+ const { match, where } = conditionsOf(r)
201
+ if (r.name === undefined && Object.keys(match).length === 0 && !where.some(finds)) {
202
+ throw new InvalidError('a range names a series, a label to match, or a oneOf or a prefix')
203
+ }
204
+ if (r.since !== undefined && r.from !== undefined) {
205
+ throw new InvalidError('a range starts since a span before now or from a time, not both')
206
+ }
207
+ const from =
208
+ r.since !== undefined ? Date.now() - ms(r.since) : r.from === undefined ? 0 : unixMs(r.from)
209
+ return {
210
+ matchers: wireLabels(r.name, match),
211
+ where: where.length > 0 ? where : undefined,
212
+ from: BigInt(from),
213
+ to: r.to === undefined ? openEnd : BigInt(unixMs(r.to)),
214
+ limitSeries: r.limits?.series,
215
+ limitBlocks: r.limits?.blocks,
216
+ limitBytes: r.limits?.bytes,
217
+ limitDecoded: r.limits?.decoded,
218
+ limitAnswered: r.limits?.answered,
219
+ ...extra,
220
+ }
221
+ }
222
+
223
+ /** A range's equality, plain values of where included, and its conditions in the order of their labels. */
224
+ function conditionsOf(r: Range): {
225
+ match: Labels
226
+ where: { label: string; kind: Condition['kind']; values: string[] }[]
227
+ } {
228
+ const match: Labels = { ...r.match }
229
+ const where = []
230
+ for (const [label, condition] of Object.entries(r.where ?? {}).sort(([a], [b]) =>
231
+ byCodePoint(a, b),
232
+ )) {
233
+ if (label in match) {
234
+ throw new InvalidError(`label ${label} is matched and has a condition`)
235
+ }
236
+ if (typeof condition === 'string') {
237
+ match[label] = condition
238
+ } else if (condition instanceof Condition) {
239
+ where.push({ label, kind: condition.kind, values: [...condition.values] })
240
+ } else {
241
+ throw new InvalidError(`label ${label}: a condition is made by oneOf, noneOf or prefix`)
242
+ }
243
+ }
244
+ return { match, where }
245
+ }
246
+
247
+ function finds(condition: { kind: Condition['kind'] }): boolean {
248
+ return condition.kind !== 'none_of'
249
+ }
250
+
251
+ function columnsOf(samples: SeriesInput['samples']): {
252
+ times: BigInt64Array
253
+ values: Float64Array
254
+ } {
255
+ if (Array.isArray(samples)) {
256
+ const pairs = samples as readonly (readonly [Time, number])[]
257
+ return {
258
+ times: BigInt64Array.from(pairs, ([t]) => BigInt(unixMs(t))),
259
+ values: Float64Array.from(pairs, ([, v]) => v),
260
+ }
261
+ }
262
+ const columns = samples as {
263
+ times: readonly number[] | BigInt64Array
264
+ values: readonly number[] | Float64Array
265
+ }
266
+ return {
267
+ times:
268
+ columns.times instanceof BigInt64Array
269
+ ? columns.times
270
+ : BigInt64Array.from(columns.times, t => BigInt(t)),
271
+ values:
272
+ columns.values instanceof Float64Array ? columns.values : Float64Array.from(columns.values),
273
+ }
274
+ }
275
+
276
+ /** Joins the pieces a series longer than a body came in, one after another with its name and labels. */
277
+ function joined<T extends { name: string; labels: Labels }>(
278
+ pieces: T[],
279
+ join: (into: T, piece: T) => void,
280
+ ): T[] {
281
+ const out: T[] = []
282
+ for (const piece of pieces) {
283
+ const last = out.at(-1)
284
+ if (last !== undefined && last.name === piece.name && sameLabels(last.labels, piece.labels)) {
285
+ join(last, piece)
286
+ } else {
287
+ out.push(piece)
288
+ }
289
+ }
290
+ return out
291
+ }
292
+
293
+ function sameLabels(a: Labels, b: Labels): boolean {
294
+ const keys = Object.keys(a)
295
+ return keys.length === Object.keys(b).length && keys.every(k => a[k] === b[k])
296
+ }
297
+
298
+ function concatFloats(a: Float64Array, b: Float64Array): Float64Array {
299
+ const out = new Float64Array(a.length + b.length)
300
+ out.set(a)
301
+ out.set(b, a.length)
302
+ return out
303
+ }
304
+
305
+ export class Metrics {
306
+ readonly #link: Link
307
+ readonly #instruments = new Map<string, Instrument>()
308
+ readonly #timers = new Map<string, TimerInstrument>()
309
+ #interval: ReturnType<typeof setInterval> | undefined
310
+ readonly #failureLog = new FailureLog('metrics instruments', sayOwn)
311
+ /** instruments' flushes that failed, and why the last did */
312
+ failures = 0
313
+ lastFailure: Error | undefined
314
+
315
+ constructor(link: Link) {
316
+ this.#link = link
317
+ }
318
+
319
+ /** Stores series and their samples, all or none; a refused series is named by its labels. */
320
+ async ingest(series: SeriesInput | readonly SeriesInput[]): Promise<void> {
321
+ const batch = (Array.isArray(series) ? series : [series as SeriesInput]).map(s => ({
322
+ labels: wireLabels(s.name, s.labels),
323
+ kind: s.kind,
324
+ ...columnsOf(s.samples),
325
+ }))
326
+ await this.#link.run('write', connection =>
327
+ connection.session.call(methods['metrics.ingest'], MetricsBatch.encode({ series: batch })),
328
+ )
329
+ }
330
+
331
+ /** Every sample of the series a range matches, exactly, read whole before the first leaves the server. */
332
+ async read(range: Range): Promise<Series[]> {
333
+ const got = await this.#link.run('read', connection =>
334
+ download(connection, methods['metrics.read'], MetricsRange.encode(rangeOf(range))),
335
+ )
336
+ const pieces = got.items.map(item => {
337
+ const s = MetricsSeries.decode(item)
338
+ return {
339
+ ...namedSeries(s.labels as Labels | undefined),
340
+ kind: (s.kind ?? 'gauge') as SeriesKind,
341
+ times: Array.from(s.times ?? [], Number),
342
+ values: s.values ?? new Float64Array(0),
343
+ }
344
+ })
345
+ return joined(pieces, (into, piece) => {
346
+ into.times.push(...piece.times)
347
+ into.values = concatFloats(into.values, piece.values)
348
+ })
349
+ }
350
+
351
+ /** Buckets of a width from the range's start, each computed exactly: a counter's increase counts its resets. */
352
+ async aggregate(range: AggregateRange): Promise<Aggregate[]> {
353
+ if (range.by !== undefined && range.without !== undefined) {
354
+ throw new InvalidError('an aggregate groups by labels or without them, not both')
355
+ }
356
+ const grouping = { by: range.by, without: range.without }
357
+ const got = await this.#link.run('read', connection =>
358
+ download(
359
+ connection,
360
+ methods['metrics.aggregate'],
361
+ MetricsRange.encode(rangeOf(range, { width: ms(range.width), op: range.op, ...grouping })),
362
+ ),
363
+ )
364
+ const pieces = got.items.map(item => {
365
+ const b = MetricsBuckets.decode(item)
366
+ const values = b.values ?? new Float64Array(0)
367
+ const buckets: Bucket[] = Array.from(values, (value, i) => ({
368
+ from: new Date(Number(b.from?.[i] ?? 0n)),
369
+ to: new Date(Number(b.to?.[i] ?? 0n)),
370
+ count: Number(b.count?.[i] ?? 0n),
371
+ resets: Number(b.resets?.[i] ?? 0n),
372
+ value,
373
+ overflow: ((b.flags?.[i] ?? 0) & 1) !== 0,
374
+ partial: ((b.flags?.[i] ?? 0) & 2) !== 0,
375
+ }))
376
+ return {
377
+ ...namedSeries(b.labels as Labels | undefined),
378
+ kind: (b.kind ?? 'gauge') as SeriesKind,
379
+ buckets,
380
+ }
381
+ })
382
+ return joined(pieces, (into, piece) => {
383
+ into.buckets.push(...piece.buckets)
384
+ })
385
+ }
386
+
387
+ /**
388
+ * What read(range), or aggregate(range) when it names an op, would spend
389
+ * of its limits, found without a payload fetched or a sample decoded: the
390
+ * series, the blocks, those an aggregate answers from their summaries, the
391
+ * bytes and the samples, beside the limits; stops is the LimitError the
392
+ * call would end with, undefined when it fits.
393
+ */
394
+ async explain(
395
+ range: Range & Partial<Pick<AggregateRange, 'width' | 'op' | 'by' | 'without'>>,
396
+ ): Promise<Plan> {
397
+ const extra =
398
+ range.op === undefined
399
+ ? {}
400
+ : { width: ms(range.width ?? 0), op: range.op, by: range.by, without: range.without }
401
+ const body = await this.#link.run('read', connection =>
402
+ connection.session.call(
403
+ methods['metrics.explain'],
404
+ MetricsRange.encode(rangeOf(range, extra)),
405
+ ),
406
+ )
407
+ const p = MetricsPlan.decode(body)
408
+ const stops = p.stops === undefined || p.stops === null ? undefined : p.stops
409
+ return {
410
+ series: Number(p.series ?? 0),
411
+ blocks: Number(p.blocks ?? 0),
412
+ summarized: Number(p.summarized ?? 0),
413
+ bytes: Number(p.bytes ?? 0),
414
+ decoded: Number(p.decoded ?? 0),
415
+ limits: {
416
+ series: Number(p.limitSeries ?? 0),
417
+ blocks: Number(p.limitBlocks ?? 0),
418
+ bytes: Number(p.limitBytes ?? 0),
419
+ decoded: Number(p.limitDecoded ?? 0),
420
+ answered: Number(p.limitAnswered ?? 0),
421
+ },
422
+ stops:
423
+ stops === undefined
424
+ ? undefined
425
+ : (errorOf(stops.code ?? 'limit', stops.message ?? '', stops.what ?? {}) as LimitError),
426
+ }
427
+ }
428
+
429
+ /** Removes one series and everything it holds, whether it still reads or not. */
430
+ async drop(series: {
431
+ name: string
432
+ labels?: Labels
433
+ }): Promise<{ found: boolean; unreadableGroups: number }> {
434
+ const labels = wireLabels(series.name, series.labels)
435
+ const body = await this.#link.run('write', connection =>
436
+ connection.session.call(methods['metrics.drop'], MetricsLabels.encode({ labels })),
437
+ )
438
+ const dropped = MetricsDropped.decode(body)
439
+ return { found: dropped.found === true, unreadableGroups: dropped.unreadableGroups ?? 0 }
440
+ }
441
+
442
+ /** A counter: its total since this process started, ingested every flush; a restart is a reset. */
443
+ counter(name: string): Counter {
444
+ return new Counter(this.#instrument(name, 'counter'), {})
445
+ }
446
+
447
+ /** A gauge: its value at each flush. */
448
+ gauge(name: string): Gauge {
449
+ return new Gauge(this.#instrument(name, 'gauge'), {})
450
+ }
451
+
452
+ /** A gauge read by a function at each flush; a function that throws skips that sample. */
453
+ gaugeFunc(name: string, read: () => number | Promise<number>): void {
454
+ this.#instrument(name, 'gauge').read = read
455
+ }
456
+
457
+ /**
458
+ * A timer: how many durations it measured and their sum in milliseconds,
459
+ * ingested every flush as the counters name_count and name_sum, and the
460
+ * longest since the flush before as the gauge name_max, left out when it
461
+ * measured none. A range's mean is its sum's increase over its count's.
462
+ */
463
+ timer(name: string): Timer {
464
+ return new Timer(this.#timer(name), {})
465
+ }
466
+
467
+ #instrument(name: string, kind: SeriesKind): Instrument {
468
+ let instrument = this.#instruments.get(name)
469
+ if (instrument === undefined) {
470
+ const timer = timerWriting(name, this.#timers)
471
+ if (timer !== undefined) {
472
+ throw new InvalidError(`${name} is written by the timer ${timer}`)
473
+ }
474
+ instrument = { name, kind, series: new Map(), refused: new Set(), lastRead: undefined }
475
+ this.#instruments.set(name, instrument)
476
+ this.#flushing()
477
+ } else if (instrument.kind !== kind) {
478
+ throw new InvalidError(`${name} is a ${instrument.kind}, not a ${kind}`)
479
+ }
480
+ return instrument
481
+ }
482
+
483
+ #timer(name: string): TimerInstrument {
484
+ let timer = this.#timers.get(name)
485
+ if (timer === undefined) {
486
+ for (const { suffix } of timerSeries) {
487
+ const writer = this.#instruments.get(name + suffix)
488
+ if (writer !== undefined) {
489
+ throw new InvalidError(
490
+ `${name + suffix} is a ${writer.kind}; the timer ${name} would write it`,
491
+ )
492
+ }
493
+ }
494
+ timer = { name, series: new Map(), refused: new Set() }
495
+ this.#timers.set(name, timer)
496
+ this.#flushing()
497
+ }
498
+ return timer
499
+ }
500
+
501
+ #flushing(): void {
502
+ this.#interval ??= setInterval(() => void this.#flushInBackground(), flushEvery)
503
+ this.#interval.unref?.()
504
+ }
505
+
506
+ // a flush no caller awaits says its failure, as Go's background work does
507
+ async #flushInBackground(): Promise<void> {
508
+ try {
509
+ await this.flush()
510
+ this.#failureLog.succeeded()
511
+ } catch (err) {
512
+ this.#failureLog.failed(err)
513
+ }
514
+ }
515
+
516
+ /**
517
+ * Ingests every instrument's value now, as the timer does every 15 s. A
518
+ * series the store refuses is left out from then on, said once, so that one
519
+ * bad instrument keeps no other out.
520
+ */
521
+ async flush(): Promise<void> {
522
+ const at = BigInt(Date.now())
523
+ const batch = await this.#instrumentSamples(at)
524
+ const taken = this.#takeTimers(at, batch)
525
+ if (batch.length === 0) {
526
+ return
527
+ }
528
+ try {
529
+ await this.#ingestLeavingRefusedOut(batch)
530
+ } catch (err) {
531
+ for (const [timing, longest] of taken) {
532
+ timing.longest = Math.max(timing.longest, longest)
533
+ timing.measured = true
534
+ }
535
+ this.failures++
536
+ this.lastFailure = err as Error
537
+ throw err
538
+ }
539
+ }
540
+
541
+ // every instrument's value now; a gauge whose function throws skips this
542
+ // flush, said once while its error stays the same
543
+ async #instrumentSamples(at: bigint): Promise<Owned[]> {
544
+ const batch: Owned[] = []
545
+ for (const instrument of this.#instruments.values()) {
546
+ if (instrument.read !== undefined && !(await readGauge(instrument, instrument.read))) {
547
+ continue
548
+ }
549
+ for (const [key, s] of instrument.series) {
550
+ batch.push({
551
+ sample: sampleAt(at, instrument.name, s.labels, instrument.kind, s.value),
552
+ owner: {
553
+ series: formatSeries(instrument.name, s.labels),
554
+ refuse: () => refuse(instrument, key),
555
+ },
556
+ })
557
+ }
558
+ }
559
+ return batch
560
+ }
561
+
562
+ // sends the batch until the store takes what is left of it: a refusal that
563
+ // names one of its series leaves that series out, for good, and goes again
564
+ async #ingestLeavingRefusedOut(batch: Owned[]): Promise<void> {
565
+ let sending = batch
566
+ while (sending.length > 0) {
567
+ try {
568
+ const series = sending.map(owned => owned.sample)
569
+ await this.#link.run('write', connection =>
570
+ connection.session.call(methods['metrics.ingest'], MetricsBatch.encode({ series })),
571
+ )
572
+ return
573
+ } catch (err) {
574
+ const refused = refusedIn(sending, err)
575
+ if (refused === undefined) {
576
+ throw err
577
+ }
578
+ refused.refuse()
579
+ sayOwn('warn', 'instrument refused', { series: refused.series, error: messageOf(err) })
580
+ sending = sending.filter(owned => owned.owner !== refused)
581
+ }
582
+ }
583
+ }
584
+
585
+ // adds what the timers ingest to batch and starts their longest again,
586
+ // answering what it took so that a failed flush gives it back; a timer's
587
+ // series of one label set share an owner, so a refusal leaves all three out
588
+ #takeTimers(at: bigint, batch: Owned[]): [Timing, number][] {
589
+ const taken: [Timing, number][] = []
590
+ for (const timer of this.#timers.values()) {
591
+ for (const [key, t] of timer.series) {
592
+ const owner = {
593
+ series: formatSeries(timer.name, t.labels),
594
+ refuse: () => refuse(timer, key),
595
+ }
596
+ const add = (suffix: string, kind: SeriesKind, value: number) =>
597
+ batch.push({ sample: sampleAt(at, timer.name + suffix, t.labels, kind, value), owner })
598
+ add('_count', 'counter', t.count)
599
+ add('_sum', 'counter', t.sum)
600
+ if (t.measured) {
601
+ add('_max', 'gauge', t.longest)
602
+ taken.push([t, t.longest])
603
+ t.longest = 0
604
+ t.measured = false
605
+ }
606
+ }
607
+ }
608
+ return taken
609
+ }
610
+
611
+ /** Stops flushing every 15 s and ingests the instruments' last values; the store's close calls it. */
612
+ async stop(): Promise<void> {
613
+ if (this.#interval !== undefined) {
614
+ clearInterval(this.#interval)
615
+ this.#interval = undefined
616
+ }
617
+ if (this.instruments > 0) {
618
+ await this.#flushInBackground()
619
+ }
620
+ }
621
+
622
+ get instruments(): number {
623
+ return this.#instruments.size + this.#timers.size
624
+ }
625
+ }
626
+
627
+ const flushEvery = 15_000
628
+
629
+ interface Instrument {
630
+ name: string
631
+ kind: SeriesKind
632
+ /** by the canonical spelling of each label set */
633
+ series: Map<string, { labels: Labels; value: number }>
634
+ /** the spellings of the label sets the store refused, whose writes are left out */
635
+ refused: Set<string>
636
+ read?: () => number | Promise<number>
637
+ /** the error a gauge's function last threw, said once while it stays the same */
638
+ lastRead: string | undefined
639
+ }
640
+
641
+ /** what a timer ingests, each series its name and a suffix */
642
+ const timerSeries = [
643
+ { suffix: '_count', kind: 'counter' },
644
+ { suffix: '_sum', kind: 'counter' },
645
+ { suffix: '_max', kind: 'gauge' },
646
+ ] as const
647
+
648
+ interface TimerInstrument {
649
+ name: string
650
+ /** by the canonical spelling of each label set */
651
+ series: Map<string, Timing>
652
+ refused: Set<string>
653
+ }
654
+
655
+ interface Timing {
656
+ labels: Labels
657
+ count: number
658
+ /** milliseconds */
659
+ sum: number
660
+ /** milliseconds, since the flush before */
661
+ longest: number
662
+ /** since the flush before */
663
+ measured: boolean
664
+ }
665
+
666
+ /** the timer that writes the series name, if one does */
667
+ function timerWriting(name: string, timers: Map<string, TimerInstrument>): string | undefined {
668
+ for (const { suffix } of timerSeries) {
669
+ const base = name.slice(0, -suffix.length)
670
+ if (name.endsWith(suffix) && timers.has(base)) {
671
+ return base
672
+ }
673
+ }
674
+ return undefined
675
+ }
676
+
677
+ /** one sample of a series, as metrics.ingest takes it */
678
+ interface Sampled {
679
+ labels: Labels
680
+ kind: SeriesKind
681
+ times: BigInt64Array
682
+ values: Float64Array
683
+ }
684
+
685
+ function sampleAt(
686
+ at: bigint,
687
+ name: string,
688
+ labels: Labels,
689
+ kind: SeriesKind,
690
+ value: number,
691
+ ): Sampled {
692
+ return {
693
+ labels: { ...labels, [wireName]: name },
694
+ kind,
695
+ times: BigInt64Array.of(at),
696
+ values: Float64Array.of(value),
697
+ }
698
+ }
699
+
700
+ function keyOf(labels: Labels): string {
701
+ return JSON.stringify(Object.entries(labels).sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)))
702
+ }
703
+
704
+ function seriesOf(instrument: Instrument, labels: Labels): { labels: Labels; value: number } {
705
+ const key = keyOf(labels)
706
+ let s = instrument.series.get(key)
707
+ if (s === undefined) {
708
+ s = { labels, value: 0 }
709
+ // a refused series' writes go to a value no flush reads
710
+ if (!instrument.refused.has(key)) {
711
+ instrument.series.set(key, s)
712
+ }
713
+ }
714
+ return s
715
+ }
716
+
717
+ /** a sample of an instrument's, and the series a refusal of it leaves out */
718
+ interface Owned {
719
+ sample: Sampled
720
+ owner: Owner
721
+ }
722
+
723
+ /** an instrument's series, or a timer's three of one label set */
724
+ interface Owner {
725
+ /** as Go prints a series: name{route="/a"} */
726
+ series: string
727
+ refuse(): void
728
+ }
729
+
730
+ function refuse(instrument: Instrument | TimerInstrument, key: string): void {
731
+ instrument.series.delete(key)
732
+ instrument.refused.add(key)
733
+ }
734
+
735
+ /** the codes of a refusal that is the series' own doing, as Go's metrics tells them apart */
736
+ const seriesRefusals = new Set<Code>([
737
+ 'invalid',
738
+ 'limit',
739
+ 'too_old',
740
+ 'too_new',
741
+ 'conflict',
742
+ 'corrupt',
743
+ 'suspended',
744
+ ])
745
+
746
+ /** the owner of the series a refusal names by its labels, when it names one of these */
747
+ function refusedIn(sending: Owned[], err: unknown): Owner | undefined {
748
+ if (!(err instanceof TinystoreError) || !seriesRefusals.has(err.code)) {
749
+ return undefined
750
+ }
751
+ const named = Object.entries(err.what)
752
+ return sending.find(
753
+ ({ sample }) =>
754
+ named.length === Object.keys(sample.labels).length &&
755
+ named.every(([key, value]) => sample.labels[key] === value),
756
+ )?.owner
757
+ }
758
+
759
+ /** a gauge's value from its function, or false when it threw: said once while the error stays the same */
760
+ async function readGauge(
761
+ instrument: Instrument,
762
+ read: () => number | Promise<number>,
763
+ ): Promise<boolean> {
764
+ if (instrument.refused.size > 0) {
765
+ return false
766
+ }
767
+ try {
768
+ instrument.series.set('{}', { labels: {}, value: await read() })
769
+ instrument.lastRead = undefined
770
+ return true
771
+ } catch (err) {
772
+ const error = messageOf(err)
773
+ if (error !== instrument.lastRead) {
774
+ sayOwn('warn', 'gauge read failed', { series: instrument.name, error })
775
+ instrument.lastRead = error
776
+ }
777
+ return false
778
+ }
779
+ }
780
+
781
+ /** a series as Go prints one: its name, then its labels sorted and quoted */
782
+ function formatSeries(name: string, labels: Labels): string {
783
+ const keys = Object.keys(labels).sort()
784
+ if (keys.length === 0) {
785
+ return name
786
+ }
787
+ return `${name}{${keys.map(key => `${key}=${JSON.stringify(labels[key])}`).join(',')}}`
788
+ }
789
+
790
+ export class Counter {
791
+ readonly #instrument: Instrument
792
+ readonly #labels: Labels
793
+
794
+ constructor(instrument: Instrument, labels: Labels) {
795
+ this.#instrument = instrument
796
+ this.#labels = labels
797
+ }
798
+
799
+ /** The counter of these labels besides its own: the same labels in any order are one series. */
800
+ with(labels: Labels): Counter {
801
+ return new Counter(this.#instrument, { ...this.#labels, ...labels })
802
+ }
803
+
804
+ inc(): void {
805
+ this.add(1)
806
+ }
807
+
808
+ add(n: number): void {
809
+ if (!(n >= 0)) {
810
+ throw new InvalidError(`a counter adds ${n}; it only grows`)
811
+ }
812
+ seriesOf(this.#instrument, this.#labels).value += n
813
+ }
814
+ }
815
+
816
+ export class Gauge {
817
+ readonly #instrument: Instrument
818
+ readonly #labels: Labels
819
+
820
+ constructor(instrument: Instrument, labels: Labels) {
821
+ this.#instrument = instrument
822
+ this.#labels = labels
823
+ }
824
+
825
+ with(labels: Labels): Gauge {
826
+ return new Gauge(this.#instrument, { ...this.#labels, ...labels })
827
+ }
828
+
829
+ set(value: number): void {
830
+ seriesOf(this.#instrument, this.#labels).value = value
831
+ }
832
+
833
+ add(n: number): void {
834
+ seriesOf(this.#instrument, this.#labels).value += n
835
+ }
836
+
837
+ inc(): void {
838
+ this.add(1)
839
+ }
840
+
841
+ dec(): void {
842
+ this.add(-1)
843
+ }
844
+ }
845
+
846
+ /** A timer of labels; record and measure add a duration to the next flush. */
847
+ export class Timer {
848
+ readonly #timer: TimerInstrument
849
+ readonly #labels: Labels
850
+
851
+ constructor(timer: TimerInstrument, labels: Labels) {
852
+ this.#timer = timer
853
+ this.#labels = labels
854
+ }
855
+
856
+ /** The timer of these labels besides its own: the same labels in any order are one series. */
857
+ with(labels: Labels): Timer {
858
+ return new Timer(this.#timer, { ...this.#labels, ...labels })
859
+ }
860
+
861
+ /** Adds one duration: milliseconds, their fractions kept, or text such as '250ms'. */
862
+ record(d: Duration): void {
863
+ const millis = typeof d === 'number' ? d : ms(d)
864
+ if (!(Number.isFinite(millis) && millis >= 0)) {
865
+ throw new InvalidError(`a timer records ${d}; a duration is never negative`)
866
+ }
867
+ const key = keyOf(this.#labels)
868
+ if (this.#timer.refused.has(key)) {
869
+ return
870
+ }
871
+ let t = this.#timer.series.get(key)
872
+ if (t === undefined) {
873
+ t = { labels: this.#labels, count: 0, sum: 0, longest: 0, measured: false }
874
+ this.#timer.series.set(key, t)
875
+ }
876
+ t.count++
877
+ t.sum += millis
878
+ t.longest = Math.max(t.longest, millis)
879
+ t.measured = true
880
+ }
881
+
882
+ /**
883
+ * Runs fn and records how long it took, whether it returned or threw: its
884
+ * result comes back, and what it threw is thrown again.
885
+ *
886
+ * const user = await latency.with({ route: '/users' }).measure(() => users.get(id))
887
+ */
888
+ async measure<R>(fn: () => R | Promise<R>): Promise<R> {
889
+ const start = performance.now()
890
+ try {
891
+ return await fn()
892
+ } finally {
893
+ this.record(performance.now() - start)
894
+ }
895
+ }
896
+ }