@tinyshed/tinystore 0.1.0-rc.5 → 0.1.0-rc.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (122) hide show
  1. package/README.md +37 -37
  2. package/bin/tinystore.js +12 -1
  3. package/dist/backup.d.ts +33 -0
  4. package/dist/backup.js +49 -0
  5. package/dist/blobs.d.ts +160 -79
  6. package/dist/blobs.js +510 -217
  7. package/dist/channels.d.ts +90 -0
  8. package/dist/channels.js +291 -0
  9. package/dist/clock.js +3 -3
  10. package/dist/config.d.ts +209 -86
  11. package/dist/config.js +581 -538
  12. package/dist/connection.d.ts +64 -2
  13. package/dist/connection.js +201 -16
  14. package/dist/console.d.ts +27 -3
  15. package/dist/console.js +165 -12
  16. package/dist/dotenv.d.ts +7 -0
  17. package/dist/dotenv.js +79 -0
  18. package/dist/errors.d.ts +61 -1
  19. package/dist/errors.js +81 -1
  20. package/dist/handles.d.ts +44 -2
  21. package/dist/handles.js +81 -2
  22. package/dist/index.d.ts +17 -11
  23. package/dist/index.js +5 -3
  24. package/dist/instruments.d.ts +177 -0
  25. package/dist/instruments.js +819 -0
  26. package/dist/jobs.d.ts +335 -239
  27. package/dist/jobs.js +716 -536
  28. package/dist/json.d.ts +2 -0
  29. package/dist/json.js +122 -0
  30. package/dist/kv.d.ts +130 -243
  31. package/dist/kv.js +273 -640
  32. package/dist/limits.d.ts +80 -0
  33. package/dist/limits.js +139 -0
  34. package/dist/locks.d.ts +62 -0
  35. package/dist/locks.js +237 -0
  36. package/dist/logger.d.ts +45 -9
  37. package/dist/logger.js +150 -38
  38. package/dist/metrics.d.ts +210 -209
  39. package/dist/metrics.js +319 -556
  40. package/dist/once.d.ts +19 -18
  41. package/dist/once.js +74 -63
  42. package/dist/query.d.ts +181 -0
  43. package/dist/query.js +788 -0
  44. package/dist/records.d.ts +241 -150
  45. package/dist/records.js +507 -291
  46. package/dist/runtime/addon.d.ts +14 -0
  47. package/dist/runtime/addon.js +69 -0
  48. package/dist/runtime/bun.js +5 -7
  49. package/dist/runtime/node.js +5 -7
  50. package/dist/runtime/pipe.d.ts +17 -0
  51. package/dist/runtime/pipe.js +133 -0
  52. package/dist/runtime/shared.d.ts +6 -0
  53. package/dist/runtime/shared.js +19 -0
  54. package/dist/runtime.d.ts +5 -2
  55. package/dist/search.d.ts +189 -0
  56. package/dist/search.js +281 -0
  57. package/dist/session.d.ts +31 -3
  58. package/dist/session.js +153 -41
  59. package/dist/sql.d.ts +228 -78
  60. package/dist/sql.js +745 -240
  61. package/dist/store.d.ts +244 -24
  62. package/dist/store.js +270 -67
  63. package/dist/time.d.ts +5 -1
  64. package/dist/time.js +22 -2
  65. package/dist/tx.d.ts +61 -0
  66. package/dist/tx.js +314 -0
  67. package/dist/wire/codec.d.ts +23 -7
  68. package/dist/wire/codec.js +56 -35
  69. package/dist/wire/frame.d.ts +2 -0
  70. package/dist/wire/frame.js +31 -14
  71. package/dist/wire/messages.d.ts +259 -0
  72. package/dist/wire/messages.js +22 -0
  73. package/dist/wire/msgpack.d.ts +49 -1
  74. package/dist/wire/msgpack.js +293 -56
  75. package/dist/wire/protocol.d.ts +2683 -0
  76. package/dist/wire/protocol.js +10846 -0
  77. package/package.json +39 -38
  78. package/src/backup.ts +81 -0
  79. package/src/blobs.ts +678 -301
  80. package/src/channels.ts +377 -0
  81. package/src/clock.ts +4 -4
  82. package/src/config.ts +902 -612
  83. package/src/connection.ts +279 -21
  84. package/src/console.ts +180 -12
  85. package/src/dotenv.ts +82 -0
  86. package/src/errors.ts +105 -0
  87. package/src/handles.ts +157 -8
  88. package/src/index.ts +158 -52
  89. package/src/instruments.ts +1042 -0
  90. package/src/jobs.ts +1017 -733
  91. package/src/json.ts +133 -0
  92. package/src/kv.ts +424 -847
  93. package/src/limits.ts +202 -0
  94. package/src/locks.ts +320 -0
  95. package/src/logger.ts +209 -40
  96. package/src/metrics.ts +557 -752
  97. package/src/once.ts +86 -83
  98. package/src/query.ts +1005 -0
  99. package/src/records.ts +795 -418
  100. package/src/runtime/addon.ts +99 -0
  101. package/src/runtime/bun.ts +5 -7
  102. package/src/runtime/node.ts +5 -7
  103. package/src/runtime/pipe.ts +178 -0
  104. package/src/runtime/shared.ts +20 -0
  105. package/src/runtime.ts +5 -2
  106. package/src/search.ts +480 -0
  107. package/src/session.ts +187 -41
  108. package/src/sql.ts +993 -306
  109. package/src/store.ts +473 -78
  110. package/src/time.ts +48 -3
  111. package/src/tx.ts +426 -0
  112. package/src/wire/codec.ts +78 -37
  113. package/src/wire/frame.ts +44 -14
  114. package/src/wire/messages.ts +25 -0
  115. package/src/wire/msgpack.ts +327 -56
  116. package/src/wire/protocol.ts +11709 -0
  117. package/dist/limiter.d.ts +0 -45
  118. package/dist/limiter.js +0 -68
  119. package/dist/quota.d.ts +0 -44
  120. package/dist/quota.js +0 -98
  121. package/src/limiter.ts +0 -99
  122. package/src/quota.ts +0 -137
package/dist/metrics.js CHANGED
@@ -1,610 +1,373 @@
1
- // Series of samples in metrics.db, as Go's metrics package keeps them: a
2
- // sample is kept bit for bit, a range read exactly, an aggregate computed
3
- // exactly and rounded once. Instruments live in the SDK and are ingested
4
- // every flush.
5
- import { FailureLog, messageOf } from './background.js';
1
+ // Numbers that move over time, as docs/guides/metrics/README.md has them, over
2
+ // the protocol: series by their name and labels, written whole by ingest and
3
+ // read back by range, in buckets and as their newest samples; what a store
4
+ // holds of them; and a program's instruments, whose totals the store keeps.
5
+ // A series' samples travel as two columns, an int64 time and a float64 value
6
+ // a sample, so that every value's bits come back as they were written.
6
7
  import { download } from './connection.js';
7
- import { errorOf, InvalidError, TinystoreError } from './errors.js';
8
- import { sayOwn } from './logger.js';
8
+ import { InvalidError } from './errors.js';
9
+ import { Counter, Gauge, Histogram, labelsOf, Timer, } from './instruments.js';
9
10
  import { ms, unixMs } from './time.js';
10
- import { byCodePoint } from './wire/codec.js';
11
- import { MetricsBatch, MetricsBuckets, MetricsDropped, MetricsLabels, MetricsPlan, MetricsRange, MetricsSeries, methods, } from './wire/messages.js';
11
+ import { Empty, MetricsAggregateAsked, MetricsAggregates, MetricsDamages, MetricsDeleted, MetricsDescription, MetricsIngest, MetricsLatest, MetricsNameAsked, MetricsNamePage, MetricsPageAsked, MetricsPlan, MetricsQuery, MetricsRead, MetricsSeriesNamed, MetricsTextPage, methods, } from './wire/protocol.js';
12
+ export { Counter, exponentialBuckets, Gauge, Histogram, Timer, Timing, } from './instruments.js';
12
13
  /**
13
- * The series of a name, of labels, or of both, each matched exactly, and
14
- * their samples over the last `since` or from `from` to `to`.
15
- *
16
- * ```ts
17
- * { name: 'cpu', since: '1h' } // the last hour of every cpu series
18
- * { match: { host: 'web-1' }, from: start, to: end } // every series of web-1
19
- * ```
14
+ * A store's metrics: series by name and labels, their reads and aggregates,
15
+ * what the store holds of them, and the program's instruments.
20
16
  */
21
- /**
22
- * What a label's value must be beyond equality, made by oneOf, noneOf or
23
- * prefix: an object of the SDK's own, so that a stored value is never read as
24
- * a query.
25
- */
26
- export class Condition {
27
- kind;
28
- values;
29
- constructor(kind, values) {
30
- this.kind = kind;
31
- this.values = values;
32
- }
33
- }
34
- /** The label is one of the values. */
35
- export function oneOf(...values) {
36
- return new Condition('one_of', values);
37
- }
38
- /** The label is none of the values, or the series has none. */
39
- export function noneOf(...values) {
40
- return new Condition('none_of', values);
41
- }
42
- /** The label's value begins with the prefix. */
43
- export function prefix(start) {
44
- return new Condition('prefix', [start]);
45
- }
46
- const openEnd = 0x7fffffffffffffffn;
47
- /** The label the wire carries a series' name as, the store's own spelling. */
48
- const wireName = '__name__';
49
- /** A series' name and labels as the wire spells them, its name among its labels. */
50
- function wireLabels(name, labels) {
51
- const spelled = {};
52
- for (const [key, value] of Object.entries(labels ?? {})) {
53
- if (key.startsWith('__')) {
54
- throw new InvalidError(`the label ${key}: a name beginning with __ is the store's own`);
55
- }
56
- spelled[key] = value;
57
- }
58
- if (name !== undefined) {
59
- spelled[wireName] = name;
60
- }
61
- return spelled;
62
- }
63
- /** A series as the wire spelled it: its name apart from its labels. */
64
- function namedSeries(spelled) {
65
- const { [wireName]: name = '', ...labels } = spelled ?? {};
66
- return { name, labels };
67
- }
68
- function rangeOf(r, extra) {
69
- const { match, where } = conditionsOf(r);
70
- if (r.name === undefined && Object.keys(match).length === 0 && !where.some(finds)) {
71
- throw new InvalidError('a range names a series, a label to match, or a oneOf or a prefix');
72
- }
73
- if (r.since !== undefined && r.from !== undefined) {
74
- throw new InvalidError('a range starts since a span before now or from a time, not both');
75
- }
76
- const from = r.since !== undefined ? Date.now() - ms(r.since) : r.from === undefined ? 0 : unixMs(r.from);
77
- return {
78
- matchers: wireLabels(r.name, match),
79
- where: where.length > 0 ? where : undefined,
80
- from: BigInt(from),
81
- to: r.to === undefined ? openEnd : BigInt(unixMs(r.to)),
82
- limitSeries: r.limits?.series,
83
- limitBlocks: r.limits?.blocks,
84
- limitBytes: r.limits?.bytes,
85
- limitDecoded: r.limits?.decoded,
86
- limitAnswered: r.limits?.answered,
87
- ...extra,
88
- };
89
- }
90
- /** A range's equality, plain values of where included, and its conditions in the order of their labels. */
91
- function conditionsOf(r) {
92
- const match = { ...r.match };
93
- const where = [];
94
- for (const [label, condition] of Object.entries(r.where ?? {}).sort(([a], [b]) => byCodePoint(a, b))) {
95
- if (label in match) {
96
- throw new InvalidError(`label ${label} is matched and has a condition`);
97
- }
98
- if (typeof condition === 'string') {
99
- match[label] = condition;
100
- }
101
- else if (condition instanceof Condition) {
102
- where.push({ label, kind: condition.kind, values: [...condition.values] });
103
- }
104
- else {
105
- throw new InvalidError(`label ${label}: a condition is made by oneOf, noneOf or prefix`);
106
- }
107
- }
108
- return { match, where };
109
- }
110
- function finds(condition) {
111
- return condition.kind !== 'none_of';
112
- }
113
- function columnsOf(samples) {
114
- if (Array.isArray(samples)) {
115
- const pairs = samples;
116
- return {
117
- times: BigInt64Array.from(pairs, ([t]) => BigInt(unixMs(t))),
118
- values: Float64Array.from(pairs, ([, v]) => v),
119
- };
120
- }
121
- const columns = samples;
122
- return {
123
- times: columns.times instanceof BigInt64Array
124
- ? columns.times
125
- : BigInt64Array.from(columns.times, t => BigInt(t)),
126
- values: columns.values instanceof Float64Array ? columns.values : Float64Array.from(columns.values),
127
- };
128
- }
129
- /** Joins the pieces a series longer than a body came in, one after another with its name and labels. */
130
- function joined(pieces, join) {
131
- const out = [];
132
- for (const piece of pieces) {
133
- const last = out.at(-1);
134
- if (last !== undefined && last.name === piece.name && sameLabels(last.labels, piece.labels)) {
135
- join(last, piece);
136
- }
137
- else {
138
- out.push(piece);
139
- }
140
- }
141
- return out;
142
- }
143
- function sameLabels(a, b) {
144
- const keys = Object.keys(a);
145
- return keys.length === Object.keys(b).length && keys.every(k => a[k] === b[k]);
146
- }
147
- function concatFloats(a, b) {
148
- const out = new Float64Array(a.length + b.length);
149
- out.set(a);
150
- out.set(b, a.length);
151
- return out;
152
- }
153
17
  export class Metrics {
154
18
  #link;
155
- #instruments = new Map();
156
- #timers = new Map();
157
- #interval;
158
- #failureLog = new FailureLog('metrics instruments', sayOwn);
159
- /** instruments' flushes that failed, and why the last did */
160
- failures = 0;
161
- lastFailure;
162
- constructor(link) {
19
+ #instruments;
20
+ /** The store's close stops instruments, sending what they changed last. */
21
+ constructor(link, instruments) {
163
22
  this.#link = link;
23
+ this.#instruments = instruments;
164
24
  }
165
- /** Stores series and their samples, all or none; a refused series is named by its labels. */
25
+ /** Writes series and their samples, every one or none: a refused series names itself. */
166
26
  async ingest(series) {
167
- const batch = (Array.isArray(series) ? series : [series]).map(s => ({
168
- labels: wireLabels(s.name, s.labels),
169
- kind: s.kind,
170
- ...columnsOf(s.samples),
171
- }));
172
- await this.#link.run('write', connection => connection.session.call(methods['metrics.ingest'], MetricsBatch.encode({ series: batch })));
173
- }
174
- /** Every sample of the series a range matches, exactly, read whole before the first leaves the server. */
175
- async read(range) {
176
- const got = await this.#link.run('read', connection => download(connection, methods['metrics.read'], MetricsRange.encode(rangeOf(range))));
177
- const pieces = got.items.map(item => {
178
- const s = MetricsSeries.decode(item);
179
- return {
180
- ...namedSeries(s.labels),
181
- kind: (s.kind ?? 'gauge'),
182
- times: Array.from(s.times ?? [], Number),
183
- values: s.values ?? new Float64Array(0),
184
- };
185
- });
186
- return joined(pieces, (into, piece) => {
187
- into.times.push(...piece.times);
188
- into.values = concatFloats(into.values, piece.values);
189
- });
190
- }
191
- /** Buckets of a width from the range's start, each computed exactly: a counter's increase counts its resets. */
192
- async aggregate(range) {
193
- if (range.by !== undefined && range.without !== undefined) {
194
- throw new InvalidError('an aggregate groups by labels or without them, not both');
27
+ const given = Array.isArray(series)
28
+ ? series
29
+ : [series];
30
+ const body = MetricsIngest.encode({ series: given.map(newSeries) });
31
+ await this.#link.run('write', connection => connection.session.call(methods['metrics.ingest'], body));
32
+ }
33
+ /** The samples of each series the query finds, in its range: whole, or a limit error naming its bound. */
34
+ async read(query) {
35
+ const parts = await this.#download('metrics.read', MetricsQuery.encode(queryOf(query)));
36
+ const series = [];
37
+ let more = false;
38
+ for (const piece of parts.flatMap(part => MetricsRead.decode(part).series ?? [])) {
39
+ const times = timesOf(piece.times ?? new Uint8Array(0));
40
+ const values = valuesOf(piece.values ?? new Uint8Array(0));
41
+ const last = series.at(-1);
42
+ if (more && last !== undefined) {
43
+ last.times = joined(last.times, times);
44
+ last.values = joined(last.values, values);
45
+ }
46
+ else {
47
+ const kind = (piece.kind ?? 'gauge');
48
+ series.push({ name: piece.name ?? '', kind, labels: piece.labels ?? {}, times, values });
49
+ }
50
+ more = piece.more === true;
195
51
  }
196
- const grouping = { by: range.by, without: range.without };
197
- const got = await this.#link.run('read', connection => download(connection, methods['metrics.aggregate'], MetricsRange.encode(rangeOf(range, { width: ms(range.width), op: range.op, ...grouping }))));
198
- const pieces = got.items.map(item => {
199
- const b = MetricsBuckets.decode(item);
200
- const values = b.values ?? new Float64Array(0);
201
- const buckets = Array.from(values, (value, i) => ({
202
- from: new Date(Number(b.from?.[i] ?? 0n)),
203
- to: new Date(Number(b.to?.[i] ?? 0n)),
204
- count: Number(b.count?.[i] ?? 0n),
205
- resets: Number(b.resets?.[i] ?? 0n),
206
- value,
207
- overflow: ((b.flags?.[i] ?? 0) & 1) !== 0,
208
- partial: ((b.flags?.[i] ?? 0) & 2) !== 0,
209
- }));
210
- return {
211
- ...namedSeries(b.labels),
212
- kind: (b.kind ?? 'gauge'),
213
- buckets,
214
- };
215
- });
216
- return joined(pieces, (into, piece) => {
217
- into.buckets.push(...piece.buckets);
218
- });
52
+ return series;
53
+ }
54
+ /** The newest sample of each series the query finds, when it has one in the range. */
55
+ async latest(query) {
56
+ const parts = await this.#download('metrics.latest', MetricsQuery.encode(queryOf(query)));
57
+ return parts.flatMap(part => (MetricsLatest.decode(part).series ?? []).map(newest => ({
58
+ name: newest.name ?? '',
59
+ labels: newest.labels ?? {},
60
+ at: new Date(newest.at ?? 0),
61
+ value: newest.value ?? 0,
62
+ })));
219
63
  }
220
64
  /**
221
- * What read(range), or aggregate(range) when it names an op, would spend
222
- * of its limits, found without a payload fetched or a sample decoded: the
223
- * series, the blocks, those an aggregate answers from their summaries, the
224
- * bytes and the samples, beside the limits; stops is the LimitError the
225
- * call would end with, undefined when it fits.
65
+ * The buckets of each group of the series the query finds, each one exact
66
+ * value of op, rounded once; a bucket of count … last has its value.
226
67
  */
227
- async explain(range) {
228
- const extra = range.op === undefined
229
- ? {}
230
- : { width: ms(range.width ?? 0), op: range.op, by: range.by, without: range.without };
231
- const body = await this.#link.run('read', connection => connection.session.call(methods['metrics.explain'], MetricsRange.encode(rangeOf(range, extra))));
232
- const p = MetricsPlan.decode(body);
233
- const stops = p.stops === undefined || p.stops === null ? undefined : p.stops;
68
+ async aggregate(query) {
69
+ const parts = await this.#download('metrics.aggregate', MetricsAggregateAsked.encode(aggregateOf(query)));
70
+ const groups = [];
71
+ let more = false;
72
+ for (const piece of parts.flatMap(part => MetricsAggregates.decode(part).groups ?? [])) {
73
+ const buckets = (piece.buckets ?? []).map(bucket => bucketOf(bucket));
74
+ const last = groups.at(-1);
75
+ if (more && last !== undefined) {
76
+ last.buckets.push(...buckets);
77
+ }
78
+ else {
79
+ groups.push({ name: piece.name ?? '', labels: piece.labels ?? {}, buckets });
80
+ }
81
+ more = piece.more === true;
82
+ }
83
+ return groups;
84
+ }
85
+ /** What a read, or an aggregate when given its op, would spend of each bound; it decodes no sample. */
86
+ async explain(query) {
87
+ const asked = query.op === undefined ? { query: queryOf(query) } : aggregateOf(query);
88
+ const answer = await this.#read('metrics.explain', MetricsAggregateAsked.encode(asked));
89
+ const plan = MetricsPlan.decode(answer);
90
+ const exceeds = plan.exceeds;
234
91
  return {
235
- series: Number(p.series ?? 0),
236
- blocks: Number(p.blocks ?? 0),
237
- summarized: Number(p.summarized ?? 0),
238
- bytes: Number(p.bytes ?? 0),
239
- decoded: Number(p.decoded ?? 0),
240
- limits: {
241
- series: Number(p.limitSeries ?? 0),
242
- blocks: Number(p.limitBlocks ?? 0),
243
- bytes: Number(p.limitBytes ?? 0),
244
- decoded: Number(p.limitDecoded ?? 0),
245
- answered: Number(p.limitAnswered ?? 0),
246
- },
247
- stops: stops === undefined
92
+ series: plan.series ?? 0,
93
+ bytes: plan.bytes ?? 0,
94
+ decoded: plan.decoded ?? 0,
95
+ exceeds: exceeds === undefined
248
96
  ? undefined
249
- : errorOf(stops.code ?? 'limit', stops.message ?? '', stops.what ?? {}),
97
+ : { limit: exceeds.limit ?? '', wanted: exceeds.wanted ?? 0, bound: exceeds.bound ?? 0 },
250
98
  };
251
99
  }
252
- /** Removes one series and everything it holds, whether it still reads or not. */
253
- async drop(series) {
254
- const labels = wireLabels(series.name, series.labels);
255
- const body = await this.#link.run('write', connection => connection.session.call(methods['metrics.drop'], MetricsLabels.encode({ labels })));
256
- const dropped = MetricsDropped.decode(body);
257
- return { found: dropped.found === true, unreadableGroups: dropped.unreadableGroups ?? 0 };
258
- }
259
- /** A counter: its total since this process started, ingested every flush; a restart is a reset. */
260
- counter(name) {
261
- return new Counter(this.#instrument(name, 'counter'), {});
262
- }
263
- /** A gauge: its value at each flush. */
264
- gauge(name) {
265
- return new Gauge(this.#instrument(name, 'gauge'), {});
266
- }
267
- /** A gauge read by a function at each flush; a function that throws skips that sample. */
268
- gaugeFunc(name, read) {
269
- this.#instrument(name, 'gauge').read = read;
100
+ /** A page of the names the store holds samples of, in the byte order of their text. */
101
+ async names(options = {}) {
102
+ const answer = await this.#read('metrics.names', MetricsPageAsked.encode(pageOf(options)));
103
+ const page = MetricsNamePage.decode(answer);
104
+ const names = (page.names ?? []).map(name => ({
105
+ name: name.name ?? '',
106
+ kind: (name.kind ?? 'gauge'),
107
+ seriesCount: name.seriesCount ?? 0,
108
+ unit: name.unit ?? '',
109
+ help: name.help ?? '',
110
+ keep: spelled(name.keep ?? 0),
111
+ retainedFrom: new Date(name.retainedFrom ?? 0),
112
+ newest: new Date(name.newest ?? 0),
113
+ }));
114
+ return page.next === undefined ? { names } : { names, next: page.next };
115
+ }
116
+ /** A page of a name's label names, but the store's own. */
117
+ async labelNames(name, options = {}) {
118
+ const page = await this.#texts('metrics.labelNames', { ...pageOf(options), name });
119
+ return page.next === undefined
120
+ ? { labels: page.texts }
121
+ : { labels: page.texts, next: page.next };
122
+ }
123
+ /** A page of the values a label of a name's series holds. */
124
+ async labelValues(name, label, options = {}) {
125
+ const page = await this.#texts('metrics.labelValues', { ...pageOf(options), name, label });
126
+ return page.next === undefined
127
+ ? { values: page.texts }
128
+ : { values: page.texts, next: page.next };
129
+ }
130
+ async #texts(method, asked) {
131
+ const page = MetricsTextPage.decode(await this.#read(method, MetricsPageAsked.encode(asked)));
132
+ return { texts: page.texts ?? [], next: page.next };
270
133
  }
271
134
  /**
272
- * A timer: how many durations it measured and their sum in milliseconds,
273
- * ingested every flush as the counters name_count and name_sum, and the
274
- * longest since the flush before as the gauge name_max, left out when it
275
- * measured none. A range's mean is its sum's increase over its count's.
135
+ * Sets a name's unit and help in place of what it had, as the Rust metrics
136
+ * crate's describe_counter! does: what is not given is removed, and `{}`
137
+ * removes both. They belong to the name, so a delete of its series leaves them.
276
138
  */
277
- timer(name) {
278
- return new Timer(this.#timer(name), {});
279
- }
280
- #instrument(name, kind) {
281
- let instrument = this.#instruments.get(name);
282
- if (instrument === undefined) {
283
- const timer = timerWriting(name, this.#timers);
284
- if (timer !== undefined) {
285
- throw new InvalidError(`${name} is written by the timer ${timer}`);
286
- }
287
- instrument = { name, kind, series: new Map(), refused: new Set(), lastRead: undefined };
288
- this.#instruments.set(name, instrument);
289
- this.#flushing();
290
- }
291
- else if (instrument.kind !== kind) {
292
- throw new InvalidError(`${name} is a ${instrument.kind}, not a ${kind}`);
293
- }
294
- return instrument;
295
- }
296
- #timer(name) {
297
- let timer = this.#timers.get(name);
298
- if (timer === undefined) {
299
- for (const { suffix } of timerSeries) {
300
- const writer = this.#instruments.get(name + suffix);
301
- if (writer !== undefined) {
302
- throw new InvalidError(`${name + suffix} is a ${writer.kind}; the timer ${name} would write it`);
303
- }
304
- }
305
- timer = { name, series: new Map(), refused: new Set() };
306
- this.#timers.set(name, timer);
307
- this.#flushing();
308
- }
309
- return timer;
139
+ async describe(name, description) {
140
+ const body = MetricsDescription.encode({
141
+ name,
142
+ unit: description.unit ?? '',
143
+ help: description.help ?? '',
144
+ });
145
+ await this.#link.run('write', connection => connection.session.call(methods['metrics.describe'], body));
310
146
  }
311
- #flushing() {
312
- this.#interval ??= setInterval(() => void this.#flushInBackground(), flushEvery);
313
- this.#interval.unref?.();
147
+ /** A name's unit and help; both empty when nothing described it. */
148
+ async description(name) {
149
+ const answer = MetricsDescription.decode(await this.#read('metrics.description', MetricsNameAsked.encode({ name })));
150
+ return { unit: answer.unit ?? '', help: answer.help ?? '' };
314
151
  }
315
- // a flush no caller awaits says its failure, as Go's background work does
316
- async #flushInBackground() {
317
- try {
318
- await this.flush();
319
- this.#failureLog.succeeded();
320
- }
321
- catch (err) {
322
- this.#failureLog.failed(err);
152
+ /**
153
+ * Removes the one series of exactly this name and these labels, no more
154
+ * and no fewer, with every sample it holds, whether they still read or
155
+ * not; true when the store held it. `labels: {}` is the series without labels.
156
+ */
157
+ async delete(series) {
158
+ const labels = {};
159
+ for (const [label, value] of Object.entries(series.labels)) {
160
+ labels[label] = String(value);
323
161
  }
162
+ const body = MetricsSeriesNamed.encode({ name: series.name, labels });
163
+ const answer = await this.#link.run('write', connection => connection.session.call(methods['metrics.delete'], body));
164
+ return MetricsDeleted.decode(answer).found === true;
324
165
  }
325
166
  /**
326
- * Ingests every instrument's value now, as the timer does every 15 s. A
327
- * series the store refuses is left out from then on, said once, so that one
328
- * bad instrument keeps no other out.
167
+ * The series whose bytes no longer read, 1000 at most, each with what
168
+ * broke; deleting them shows the next.
169
+ *
170
+ * for (let damaged = await store.metrics.damaged(); damaged.length > 0; damaged = await store.metrics.damaged()) {
171
+ * for (const series of damaged) await store.metrics.delete(series)
172
+ * }
329
173
  */
174
+ async damaged() {
175
+ const answer = MetricsDamages.decode(await this.#read('metrics.damaged', Empty.encode({})));
176
+ return (answer.damages ?? []).map(damage => ({
177
+ name: damage.name ?? '',
178
+ labels: damage.labels ?? {},
179
+ error: damage.error ?? '',
180
+ }));
181
+ }
182
+ /** Sends what every instrument changed and has the store write its totals now, as it does every 15 seconds. */
330
183
  async flush() {
331
- const at = BigInt(Date.now());
332
- const batch = await this.#instrumentSamples(at);
333
- const taken = this.#takeTimers(at, batch);
334
- if (batch.length === 0) {
184
+ if (await this.#instruments.flush()) {
335
185
  return;
336
186
  }
337
- try {
338
- await this.#ingestLeavingRefusedOut(batch);
339
- }
340
- catch (err) {
341
- for (const [timing, longest] of taken) {
342
- timing.longest = Math.max(timing.longest, longest);
343
- timing.measured = true;
344
- }
345
- this.failures++;
346
- this.lastFailure = err;
347
- throw err;
348
- }
187
+ await this.#link.run('write', connection => connection.session.call(methods['metrics.flush'], Empty.encode({})));
349
188
  }
350
- // every instrument's value now; a gauge whose function throws skips this
351
- // flush, said once while its error stays the same
352
- async #instrumentSamples(at) {
353
- const batch = [];
354
- for (const instrument of this.#instruments.values()) {
355
- if (instrument.read !== undefined && !(await readGauge(instrument, instrument.read))) {
356
- continue;
357
- }
358
- for (const [key, s] of instrument.series) {
359
- batch.push({
360
- sample: sampleAt(at, instrument.name, s.labels, instrument.kind, s.value),
361
- owner: {
362
- series: formatSeries(instrument.name, s.labels),
363
- refuse: () => refuse(instrument, key),
364
- },
365
- });
366
- }
367
- }
368
- return batch;
369
- }
370
- // sends the batch until the store takes what is left of it: a refusal that
371
- // names one of its series leaves that series out, for good, and goes again
372
- async #ingestLeavingRefusedOut(batch) {
373
- let sending = batch;
374
- while (sending.length > 0) {
375
- try {
376
- const series = sending.map(owned => owned.sample);
377
- await this.#link.run('write', connection => connection.session.call(methods['metrics.ingest'], MetricsBatch.encode({ series })));
378
- return;
379
- }
380
- catch (err) {
381
- const refused = refusedIn(sending, err);
382
- if (refused === undefined) {
383
- throw err;
384
- }
385
- refused.refuse();
386
- sayOwn('warn', 'instrument refused', { series: refused.series, error: messageOf(err) });
387
- sending = sending.filter(owned => owned.owner !== refused);
388
- }
389
- }
189
+ /**
190
+ * A total that only goes up, which the store keeps: processes counting
191
+ * into one series count into one total. Its label names, when given,
192
+ * `counter<'route' | 'status'>(…)`, are the only ones `with` compiles with.
193
+ */
194
+ counter(name, options = {}) {
195
+ return new Counter(this.#instruments, this.#instruments.declare(name, 'counter', options), {});
390
196
  }
391
- // adds what the timers ingest to batch and starts their longest again,
392
- // answering what it took so that a failed flush gives it back; a timer's
393
- // series of one label set share an owner, so a refusal leaves all three out
394
- #takeTimers(at, batch) {
395
- const taken = [];
396
- for (const timer of this.#timers.values()) {
397
- for (const [key, t] of timer.series) {
398
- const owner = {
399
- series: formatSeries(timer.name, t.labels),
400
- refuse: () => refuse(timer, key),
401
- };
402
- const add = (suffix, kind, value) => batch.push({ sample: sampleAt(at, timer.name + suffix, t.labels, kind, value), owner });
403
- add('_count', 'counter', t.count);
404
- add('_sum', 'counter', t.sum);
405
- if (t.measured) {
406
- add('_max', 'gauge', t.longest);
407
- taken.push([t, t.longest]);
408
- t.longest = 0;
409
- t.measured = false;
410
- }
411
- }
412
- }
413
- return taken;
197
+ /** A value that goes up and down: a queue's length, a temperature. */
198
+ gauge(name, options = {}) {
199
+ return new Gauge(this.#instruments, this.#instruments.declare(name, 'gauge', options), {});
414
200
  }
415
- /** Stops flushing every 15 s and ingests the instruments' last values; the store's close calls it. */
416
- async stop() {
417
- if (this.#interval !== undefined) {
418
- clearInterval(this.#interval);
419
- this.#interval = undefined;
420
- }
421
- if (this.instruments > 0) {
422
- await this.#flushInBackground();
423
- }
201
+ /**
202
+ * A gauge whose value read gives every 15 seconds, of the set of labels
203
+ * given; another set of the name takes a function of its own, and a set
204
+ * is read by one. It may be async, is never called again while a call of
205
+ * it runs, and an error or an answer that is not a number writes nothing
206
+ * that time, said once while it repeats.
207
+ *
208
+ * store.metrics.gaugeFunc('queue_depth', () => emails.depth(), { labels: { queue: 'emails' } })
209
+ */
210
+ gaugeFunc(name, read, options = {}) {
211
+ const { labels, ...told } = options;
212
+ this.#instruments.reader(name, labelsOf(labels ?? {}, false), told, read);
424
213
  }
425
- get instruments() {
426
- return this.#instruments.size + this.#timers.size;
214
+ /** How many things took how long: their count, sum and longest in milliseconds, and with buckets their spread. */
215
+ timer(name, options = {}) {
216
+ return new Timer(this.#instruments, this.#instruments.declare(name, 'timer', options), {});
427
217
  }
428
- }
429
- const flushEvery = 15_000;
430
- /** what a timer ingests, each series its name and a suffix */
431
- const timerSeries = [
432
- { suffix: '_count', kind: 'counter' },
433
- { suffix: '_sum', kind: 'counter' },
434
- { suffix: '_max', kind: 'gauge' },
435
- ];
436
- /** the timer that writes the series name, if one does */
437
- function timerWriting(name, timers) {
438
- for (const { suffix } of timerSeries) {
439
- const base = name.slice(0, -suffix.length);
440
- if (name.endsWith(suffix) && timers.has(base)) {
441
- return base;
442
- }
218
+ /** A timer of numbers that are not durations: a response's size in bytes. */
219
+ histogram(name, options = {}) {
220
+ return new Histogram(this.#instruments, this.#instruments.declare(name, 'histogram', options), {});
221
+ }
222
+ #read(method, body) {
223
+ return this.#link.run('read', connection => connection.session.call(methods[method], body));
224
+ }
225
+ async #download(method, body) {
226
+ const got = await this.#link.run('read', connection => download(connection, methods[method], body));
227
+ return got.items.length === 0 ? [got.trailer] : [...got.items, got.trailer];
443
228
  }
444
- return undefined;
445
229
  }
446
- function sampleAt(at, name, labels, kind, value) {
230
+ /** A query as the wire carries it; a condition given as undefined is refused, never left out. */
231
+ function queryOf(query) {
232
+ const where = {};
233
+ for (const [label, condition] of Object.entries(query.where ?? {})) {
234
+ where[label] = conditionOf(label, condition);
235
+ }
236
+ if (query.since !== undefined && query.from !== undefined) {
237
+ throw new InvalidError('a query starts since a span before now or from a time, not both');
238
+ }
447
239
  return {
448
- labels: { ...labels, [wireName]: name },
449
- kind,
450
- times: BigInt64Array.of(at),
451
- values: Float64Array.of(value),
240
+ name: query.name,
241
+ where,
242
+ since: query.since === undefined ? undefined : ms(query.since),
243
+ from: query.from === undefined ? undefined : unixMs(query.from),
244
+ to: query.to === undefined ? undefined : unixMs(query.to),
452
245
  };
453
246
  }
454
- function keyOf(labels) {
455
- return JSON.stringify(Object.entries(labels).sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)));
456
- }
457
- function seriesOf(instrument, labels) {
458
- const key = keyOf(labels);
459
- let s = instrument.series.get(key);
460
- if (s === undefined) {
461
- s = { labels, value: 0 };
462
- // a refused series' writes go to a value no flush reads
463
- if (!instrument.refused.has(key)) {
464
- instrument.series.set(key, s);
465
- }
247
+ function conditionOf(label, condition) {
248
+ if (condition === undefined || condition === null) {
249
+ throw new InvalidError(`where ${label}: a condition given as ${String(condition)}`);
466
250
  }
467
- return s;
468
- }
469
- function refuse(instrument, key) {
470
- instrument.series.delete(key);
471
- instrument.refused.add(key);
472
- }
473
- /** the codes of a refusal that is the series' own doing, as Go's metrics tells them apart */
474
- const seriesRefusals = new Set([
475
- 'invalid',
476
- 'limit',
477
- 'too_old',
478
- 'too_new',
479
- 'conflict',
480
- 'corrupt',
481
- 'suspended',
482
- ]);
483
- /** the owner of the series a refusal names by its labels, when it names one of these */
484
- function refusedIn(sending, err) {
485
- if (!(err instanceof TinystoreError) || !seriesRefusals.has(err.code)) {
486
- return undefined;
487
- }
488
- const named = Object.entries(err.what);
489
- return sending.find(({ sample }) => named.length === Object.keys(sample.labels).length &&
490
- named.every(([key, value]) => sample.labels[key] === value))?.owner;
491
- }
492
- /** a gauge's value from its function, or false when it threw: said once while the error stays the same */
493
- async function readGauge(instrument, read) {
494
- if (instrument.refused.size > 0) {
495
- return false;
496
- }
497
- try {
498
- instrument.series.set('{}', { labels: {}, value: await read() });
499
- instrument.lastRead = undefined;
500
- return true;
501
- }
502
- catch (err) {
503
- const error = messageOf(err);
504
- if (error !== instrument.lastRead) {
505
- sayOwn('warn', 'gauge read failed', { series: instrument.name, error });
506
- instrument.lastRead = error;
251
+ if (Array.isArray(condition)) {
252
+ return { values: condition.map(String) };
253
+ }
254
+ if (typeof condition === 'object') {
255
+ if ('not' in condition) {
256
+ return { values: [condition.not].flat().map(String), not: true };
257
+ }
258
+ if ('prefix' in condition) {
259
+ return { prefix: condition.prefix };
507
260
  }
508
- return false;
261
+ throw new InvalidError(`where ${label}: a value, a list, { not } or { prefix }`);
509
262
  }
263
+ return { values: [String(condition)] };
510
264
  }
511
- /** a series as Go prints one: its name, then its labels sorted and quoted */
512
- function formatSeries(name, labels) {
513
- const keys = Object.keys(labels).sort();
514
- if (keys.length === 0) {
515
- return name;
516
- }
517
- return `${name}{${keys.map(key => `${key}=${JSON.stringify(labels[key])}`).join(',')}}`;
265
+ function aggregateOf(query) {
266
+ return {
267
+ query: queryOf(query),
268
+ every: query.every === undefined ? undefined : ms(query.every),
269
+ op: query.op,
270
+ by: query.by === undefined ? undefined : [...query.by],
271
+ without: query.without === undefined ? undefined : [...query.without],
272
+ estimate: query.estimate === true,
273
+ };
518
274
  }
519
- export class Counter {
520
- #instrument;
521
- #labels;
522
- constructor(instrument, labels) {
523
- this.#instrument = instrument;
524
- this.#labels = labels;
525
- }
526
- /** The counter of these labels besides its own: the same labels in any order are one series. */
527
- with(labels) {
528
- return new Counter(this.#instrument, { ...this.#labels, ...labels });
529
- }
530
- inc() {
531
- this.add(1);
532
- }
533
- add(n) {
534
- if (!(n >= 0)) {
535
- throw new InvalidError(`a counter adds ${n}; it only grows`);
536
- }
537
- seriesOf(this.#instrument, this.#labels).value += n;
538
- }
275
+ function pageOf(options) {
276
+ return { prefix: options.prefix, limit: options.limit, after: options.after };
539
277
  }
540
- export class Gauge {
541
- #instrument;
542
- #labels;
543
- constructor(instrument, labels) {
544
- this.#instrument = instrument;
545
- this.#labels = labels;
546
- }
547
- with(labels) {
548
- return new Gauge(this.#instrument, { ...this.#labels, ...labels });
549
- }
550
- set(value) {
551
- seriesOf(this.#instrument, this.#labels).value = value;
552
- }
553
- add(n) {
554
- seriesOf(this.#instrument, this.#labels).value += n;
278
+ function bucketOf(bucket) {
279
+ const answer = {
280
+ at: new Date(bucket.at ?? 0),
281
+ samples: bucket.samples ?? 0,
282
+ resets: bucket.resets ?? 0,
283
+ partial: bucket.partial === true,
284
+ gap: bucket.gap === true,
285
+ };
286
+ if (bucket.value !== undefined) {
287
+ answer.value = bucket.value;
555
288
  }
556
- inc() {
557
- this.add(1);
289
+ if (bucket.lower !== undefined) {
290
+ answer.lower = bucket.lower;
291
+ answer.upper = bucket.upper ?? Infinity;
558
292
  }
559
- dec() {
560
- this.add(-1);
293
+ if (bucket.estimate !== undefined) {
294
+ answer.estimate = bucket.estimate;
561
295
  }
296
+ return answer;
562
297
  }
563
- /** A timer of labels; record and measure add a duration to the next flush. */
564
- export class Timer {
565
- #timer;
566
- #labels;
567
- constructor(timer, labels) {
568
- this.#timer = timer;
569
- this.#labels = labels;
570
- }
571
- /** The timer of these labels besides its own: the same labels in any order are one series. */
572
- with(labels) {
573
- return new Timer(this.#timer, { ...this.#labels, ...labels });
574
- }
575
- /** Adds one duration: milliseconds, their fractions kept, or text such as '250ms'. */
576
- record(d) {
577
- const millis = typeof d === 'number' ? d : ms(d);
578
- if (!(Number.isFinite(millis) && millis >= 0)) {
579
- throw new InvalidError(`a timer records ${d}; a duration is never negative`);
580
- }
581
- const key = keyOf(this.#labels);
582
- if (this.#timer.refused.has(key)) {
583
- return;
584
- }
585
- let t = this.#timer.series.get(key);
586
- if (t === undefined) {
587
- t = { labels: this.#labels, count: 0, sum: 0, longest: 0, measured: false };
588
- this.#timer.series.set(key, t);
298
+ /** A series to write, its samples as the two columns the wire carries. */
299
+ function newSeries(given) {
300
+ const labels = {};
301
+ for (const [label, value] of Object.entries(given.labels ?? {})) {
302
+ labels[label] = String(value);
303
+ }
304
+ const [times, values] = columnsOf(given);
305
+ const timeBytes = new BigInt64Array(times.length);
306
+ for (let i = 0; i < times.length; i++) {
307
+ timeBytes[i] = BigInt(unixMs(times[i]));
308
+ }
309
+ // a Float64Array's bytes go as they are, a NaN's payload with them
310
+ const valueBytes = values instanceof Float64Array
311
+ ? new Uint8Array(values.buffer.slice(values.byteOffset, values.byteOffset + values.byteLength))
312
+ : new Uint8Array(Float64Array.from(values).buffer);
313
+ return {
314
+ name: given.name,
315
+ kind: given.kind,
316
+ labels,
317
+ times: new Uint8Array(timeBytes.buffer),
318
+ values: valueBytes,
319
+ keep: given.keep === undefined ? undefined : ms(given.keep),
320
+ };
321
+ }
322
+ function columnsOf(given) {
323
+ if (given.samples !== undefined) {
324
+ if (given.times !== undefined || given.values !== undefined) {
325
+ throw new InvalidError(`metrics ${given.name}: samples, or times and values, not both`);
589
326
  }
590
- t.count++;
591
- t.sum += millis;
592
- t.longest = Math.max(t.longest, millis);
593
- t.measured = true;
327
+ return [given.samples.map(([time]) => time), given.samples.map(([, value]) => value)];
594
328
  }
595
- /**
596
- * Runs fn and records how long it took, whether it returned or threw: its
597
- * result comes back, and what it threw is thrown again.
598
- *
599
- * const user = await latency.with({ route: '/users' }).measure(() => users.get(id))
600
- */
601
- async measure(fn) {
602
- const start = performance.now();
603
- try {
604
- return await fn();
605
- }
606
- finally {
607
- this.record(performance.now() - start);
329
+ if (given.times === undefined ||
330
+ given.values === undefined ||
331
+ given.times.length !== given.values.length) {
332
+ throw new InvalidError(`metrics ${given.name}: samples, or times and values as long as each other`);
333
+ }
334
+ return [given.times, given.values];
335
+ }
336
+ /** A column of int64 times, read as the unix milliseconds a Float64Array holds. */
337
+ function timesOf(bytes) {
338
+ const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
339
+ const times = new Float64Array(bytes.byteLength / 8);
340
+ for (let i = 0; i < times.length; i++) {
341
+ times[i] = Number(view.getBigInt64(i * 8, true));
342
+ }
343
+ return times;
344
+ }
345
+ /**
346
+ * A column of float64 values, its bytes copied as they came, so that each
347
+ * value's bits stay. A socket's bytes are a Buffer, whose slice is a view of
348
+ * a shared pool rather than a copy, so they are copied into a buffer of their own.
349
+ */
350
+ function valuesOf(bytes) {
351
+ const copy = new Uint8Array(bytes.byteLength);
352
+ copy.set(bytes);
353
+ return new Float64Array(copy.buffer);
354
+ }
355
+ function joined(a, b) {
356
+ const out = new Float64Array(a.length + b.length);
357
+ out.set(a);
358
+ out.set(b, a.length);
359
+ return out;
360
+ }
361
+ function spelled(millis) {
362
+ for (const [unit, size] of [
363
+ ['d', 86_400_000],
364
+ ['h', 3_600_000],
365
+ ['m', 60_000],
366
+ ['s', 1000],
367
+ ]) {
368
+ if (millis >= size && millis % size === 0) {
369
+ return `${millis / size}${unit}`;
608
370
  }
609
371
  }
372
+ return `${millis}ms`;
610
373
  }