@tinyshed/tinystore 0.1.0-rc.5 → 0.1.0-rc.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (122) hide show
  1. package/README.md +37 -37
  2. package/bin/tinystore.js +12 -1
  3. package/dist/backup.d.ts +33 -0
  4. package/dist/backup.js +49 -0
  5. package/dist/blobs.d.ts +160 -79
  6. package/dist/blobs.js +510 -217
  7. package/dist/channels.d.ts +90 -0
  8. package/dist/channels.js +291 -0
  9. package/dist/clock.js +3 -3
  10. package/dist/config.d.ts +209 -86
  11. package/dist/config.js +581 -538
  12. package/dist/connection.d.ts +64 -2
  13. package/dist/connection.js +201 -16
  14. package/dist/console.d.ts +27 -3
  15. package/dist/console.js +165 -12
  16. package/dist/dotenv.d.ts +7 -0
  17. package/dist/dotenv.js +79 -0
  18. package/dist/errors.d.ts +61 -1
  19. package/dist/errors.js +81 -1
  20. package/dist/handles.d.ts +44 -2
  21. package/dist/handles.js +81 -2
  22. package/dist/index.d.ts +17 -11
  23. package/dist/index.js +5 -3
  24. package/dist/instruments.d.ts +177 -0
  25. package/dist/instruments.js +819 -0
  26. package/dist/jobs.d.ts +335 -239
  27. package/dist/jobs.js +716 -536
  28. package/dist/json.d.ts +2 -0
  29. package/dist/json.js +122 -0
  30. package/dist/kv.d.ts +130 -243
  31. package/dist/kv.js +273 -640
  32. package/dist/limits.d.ts +80 -0
  33. package/dist/limits.js +139 -0
  34. package/dist/locks.d.ts +62 -0
  35. package/dist/locks.js +237 -0
  36. package/dist/logger.d.ts +45 -9
  37. package/dist/logger.js +150 -38
  38. package/dist/metrics.d.ts +210 -209
  39. package/dist/metrics.js +319 -556
  40. package/dist/once.d.ts +19 -18
  41. package/dist/once.js +74 -63
  42. package/dist/query.d.ts +181 -0
  43. package/dist/query.js +788 -0
  44. package/dist/records.d.ts +241 -150
  45. package/dist/records.js +507 -291
  46. package/dist/runtime/addon.d.ts +14 -0
  47. package/dist/runtime/addon.js +69 -0
  48. package/dist/runtime/bun.js +5 -7
  49. package/dist/runtime/node.js +5 -7
  50. package/dist/runtime/pipe.d.ts +17 -0
  51. package/dist/runtime/pipe.js +133 -0
  52. package/dist/runtime/shared.d.ts +6 -0
  53. package/dist/runtime/shared.js +19 -0
  54. package/dist/runtime.d.ts +5 -2
  55. package/dist/search.d.ts +189 -0
  56. package/dist/search.js +281 -0
  57. package/dist/session.d.ts +31 -3
  58. package/dist/session.js +153 -41
  59. package/dist/sql.d.ts +228 -78
  60. package/dist/sql.js +745 -240
  61. package/dist/store.d.ts +244 -24
  62. package/dist/store.js +270 -67
  63. package/dist/time.d.ts +5 -1
  64. package/dist/time.js +22 -2
  65. package/dist/tx.d.ts +61 -0
  66. package/dist/tx.js +314 -0
  67. package/dist/wire/codec.d.ts +23 -7
  68. package/dist/wire/codec.js +56 -35
  69. package/dist/wire/frame.d.ts +2 -0
  70. package/dist/wire/frame.js +31 -14
  71. package/dist/wire/messages.d.ts +259 -0
  72. package/dist/wire/messages.js +22 -0
  73. package/dist/wire/msgpack.d.ts +49 -1
  74. package/dist/wire/msgpack.js +293 -56
  75. package/dist/wire/protocol.d.ts +2683 -0
  76. package/dist/wire/protocol.js +10846 -0
  77. package/package.json +39 -38
  78. package/src/backup.ts +81 -0
  79. package/src/blobs.ts +678 -301
  80. package/src/channels.ts +377 -0
  81. package/src/clock.ts +4 -4
  82. package/src/config.ts +902 -612
  83. package/src/connection.ts +279 -21
  84. package/src/console.ts +180 -12
  85. package/src/dotenv.ts +82 -0
  86. package/src/errors.ts +105 -0
  87. package/src/handles.ts +157 -8
  88. package/src/index.ts +158 -52
  89. package/src/instruments.ts +1042 -0
  90. package/src/jobs.ts +1017 -733
  91. package/src/json.ts +133 -0
  92. package/src/kv.ts +424 -847
  93. package/src/limits.ts +202 -0
  94. package/src/locks.ts +320 -0
  95. package/src/logger.ts +209 -40
  96. package/src/metrics.ts +557 -752
  97. package/src/once.ts +86 -83
  98. package/src/query.ts +1005 -0
  99. package/src/records.ts +795 -418
  100. package/src/runtime/addon.ts +99 -0
  101. package/src/runtime/bun.ts +5 -7
  102. package/src/runtime/node.ts +5 -7
  103. package/src/runtime/pipe.ts +178 -0
  104. package/src/runtime/shared.ts +20 -0
  105. package/src/runtime.ts +5 -2
  106. package/src/search.ts +480 -0
  107. package/src/session.ts +187 -41
  108. package/src/sql.ts +993 -306
  109. package/src/store.ts +473 -78
  110. package/src/time.ts +48 -3
  111. package/src/tx.ts +426 -0
  112. package/src/wire/codec.ts +78 -37
  113. package/src/wire/frame.ts +44 -14
  114. package/src/wire/messages.ts +25 -0
  115. package/src/wire/msgpack.ts +327 -56
  116. package/src/wire/protocol.ts +11709 -0
  117. package/dist/limiter.d.ts +0 -45
  118. package/dist/limiter.js +0 -68
  119. package/dist/quota.d.ts +0 -44
  120. package/dist/quota.js +0 -98
  121. package/src/limiter.ts +0 -99
  122. package/src/quota.ts +0 -137
package/dist/records.js CHANGED
@@ -1,346 +1,562 @@
1
- // The application's logs and events in records.db, as Go's records package
2
- // keeps them: one log of the store's whose calls name their streams, read by
3
- // time and by what records hold, followed in the order they were sealed.
4
- import { download } from './connection.js';
5
- import { InvalidError } from './errors.js';
6
- import { newLogger } from './logger.js';
1
+ // Streams of events, as docs/guides/records/README.md has them, over the protocol: a
2
+ // stream opened by name, its events appended whole and read back by time and
3
+ // by what they hold, a page at a time, and every stream read together. A
4
+ // store's logger keeps its lines in a stream through the same append; a tail
5
+ // follows what is written from now on, and a writer of lines turns another
6
+ // program's output into events.
7
+ import { pieceOf } from './connection.js';
8
+ import { jsonOf } from './console.js';
9
+ import { ClosedError, InvalidError } from './errors.js';
10
+ import { checkName, handleOn, viaLink } from './handles.js';
11
+ import { parseJson } from './json.js';
12
+ import { encodeFields, newStoreLogger, } from './logger.js';
7
13
  import { ms } from './time.js';
8
14
  import { currentTrace } from './trace.js';
9
- import { Empty, methods, RecordsQuery as QueryMessage, RecordsBatch, RecordsCursor, RecordsDamage, RecordsDamages, RecordsPage, RecordsRecord, RecordsStream, } from './wire/messages.js';
10
- const levels = { debug: -4, info: 0, warn: 4, error: 8 };
11
- function levelOf(level) {
12
- return typeof level === 'number' ? level : levels[level];
13
- }
14
- /**
15
- * A record's fields, its attrs or its context, as an object, each value read
16
- * as JSON.parse reads the JSON it was kept as; the last of a repeated key is
17
- * kept. The pairs keep what JSON.parse may not: 1.2300 as written, an integer
18
- * past 2^53 whole.
19
- *
20
- * ```ts
21
- * for await (const record of store.records.all({ since: '1h', streams: ['readers'] })) {
22
- * const { page, from } = fields(record.attrs)
23
- * }
24
- * ```
25
- */
26
- export function fields(pairs) {
27
- // fromEntries, since an assignment to __proto__ would set the prototype
28
- return Object.fromEntries(pairs.map(([key, json]) => [key, JSON.parse(textOf(json))]));
29
- }
30
- const nsPerMs = 1000000n;
31
- function nanos(t) {
32
- return typeof t === 'bigint' ? t : BigInt(t.getTime()) * nsPerMs;
33
- }
34
- function idOf(id, bytes, what) {
35
- if (id === undefined) {
36
- return undefined;
15
+ import { Empty, Handle, methods, RecordsAggregateAsked, RecordsAppend, RecordsBatch, RecordsCount, RecordsDamage, RecordsDamages, RecordsDropped, RecordsFieldNamesAsked, RecordsNames, RecordsOpen, RecordsPage, RecordsPiece, RecordsPollAsked, RecordsQuery, RecordsRows, RecordsStreams, RecordsTailBatch, RecordsTopAsked, RecordsTops, } from './wire/protocol.js';
16
+ const newline = 0x0a;
17
+ /** What a writer of lines holds for the server at most: past it, lines are dropped and counted. */
18
+ const linesHeld = 1 << 20;
19
+ /** The bytes one event holds: a longer line is dropped whole, as the store drops one. */
20
+ const eventRoom = 256 << 10;
21
+ const encoder = new TextEncoder();
22
+ const levelNumbers = {
23
+ trace: -8,
24
+ debug: -4,
25
+ info: 0,
26
+ warn: 4,
27
+ error: 8,
28
+ fatal: 12,
29
+ };
30
+ const levelNames = ['trace', 'debug', 'info', 'warn', 'error', 'fatal'];
31
+ /** What every read takes: a stream's handle, or every stream without one. */
32
+ class Reads {
33
+ /** One page, from one moment of the file: `{ events, next }`. */
34
+ async list(query) {
35
+ const asked = queryOf(query);
36
+ const answer = await this.ask('records.list', handle => RecordsQuery.encode({ ...asked, handle }));
37
+ const page = RecordsPage.decode(answer);
38
+ const events = (page.events ?? []).map(eventOf);
39
+ return page.next === undefined ? { events } : { events, next: page.next };
37
40
  }
38
- const raw = typeof id === 'string' ? new Uint8Array(Buffer.from(id, 'hex')) : id;
39
- if (raw.length !== bytes) {
40
- throw new InvalidError(`a ${what} id of ${raw.length} bytes, not ${bytes}`);
41
+ /** Every event the query finds, a page after another. */
42
+ async *all(query) {
43
+ let after = query?.after;
44
+ for (;;) {
45
+ const page = await this.list(after === undefined ? query : { ...query, after });
46
+ yield* page.events;
47
+ if (page.next === undefined) {
48
+ return;
49
+ }
50
+ after = page.next;
51
+ }
41
52
  }
42
- return raw;
43
- }
44
- /** Fields as the wire carries them: one array of keys and JSON texts. */
45
- function fieldsOf(fields) {
46
- if (fields === undefined) {
47
- return undefined;
53
+ /** How many events the query finds. */
54
+ async count(query) {
55
+ const asked = queryOf(query);
56
+ const answer = await this.ask('records.count', handle => RecordsQuery.encode({ ...asked, handle }));
57
+ return RecordsCount.decode(answer).count ?? 0;
48
58
  }
49
- const pairs = Array.isArray(fields) ? fields : Object.entries(fields);
50
- const flat = [];
51
- for (const [key, value] of pairs) {
52
- const json = JSON.stringify(value);
53
- if (json === undefined) {
54
- throw new InvalidError(`the field ${key} holds a value JSON cannot write`);
55
- }
56
- flat.push(key, json);
59
+ /**
60
+ * A key's most frequent values and how many events hold each: ten unless
61
+ * `limit` asks for more, at most 1000. The key is a field's or a
62
+ * context's, or `stream`, `name` or `level`.
63
+ */
64
+ async top(key, query) {
65
+ const asked = queryOf(query);
66
+ const answer = await this.ask('records.top', handle => RecordsTopAsked.encode({ query: { ...asked, handle }, key }));
67
+ return (RecordsTops.decode(answer).values ?? []).map(top => ({
68
+ value: parseJson(top.value ?? 'null'),
69
+ count: top.count ?? 0,
70
+ }));
57
71
  }
58
- return flat.length === 0 ? undefined : flat;
59
- }
60
- const lossy = new TextDecoder();
61
- function textOf(text) {
62
- return text === undefined ? '' : typeof text === 'string' ? text : lossy.decode(text);
63
- }
64
- function pairs(flat) {
65
- const out = [];
66
- for (let i = 0; flat !== undefined && i + 1 < flat.length; i += 2) {
67
- out.push([textOf(flat[i]), flat[i + 1]]);
72
+ /** A row for each span of `every` and each value of the keys of `by`, with its count. */
73
+ async aggregate(query = {}) {
74
+ const { every, by, stats, distinct, ...counted } = query;
75
+ const asked = queryOf(counted);
76
+ const answer = await this.ask('records.aggregate', handle => RecordsAggregateAsked.encode({
77
+ query: { ...asked, handle },
78
+ every: every === undefined ? undefined : ms(every),
79
+ by: by === undefined ? undefined : [...by],
80
+ stats: stats === undefined ? undefined : [...stats],
81
+ distinct: distinct === undefined ? undefined : [...distinct],
82
+ }));
83
+ return (RecordsRows.decode(answer).rows ?? []).map(row => ({
84
+ at: dateOf(row.at ?? 0n),
85
+ count: row.count ?? 0,
86
+ by: mapValues(row.by, json => parseJson(json)),
87
+ stats: mapValues(row.stats, kept => ({
88
+ samples: kept.samples ?? 0,
89
+ sum: parseJson(kept.sum ?? '0'),
90
+ min: parseJson(kept.min ?? '0'),
91
+ max: parseJson(kept.max ?? '0'),
92
+ avg: kept.avg ?? 0,
93
+ })),
94
+ distinct: mapValues(row.distinct, count => count),
95
+ }));
68
96
  }
69
- return out;
70
- }
71
- function recordOf(r) {
72
- return {
73
- at: r.at ?? 0n,
74
- stream: textOf(r.stream),
75
- name: textOf(r.name),
76
- level: r.level,
77
- body: r.body,
78
- traceId: r.traceId,
79
- spanId: r.spanId,
80
- context: pairs(r.context),
81
- attrs: pairs(r.attrs),
82
- };
83
- }
84
- /**
85
- * A page's next: where the range moved, both ends as nanoseconds, so that a
86
- * query over the last `since` continues the range it started with.
87
- *
88
- * from 1700000000000000000, open end → '1700000000000000000:'
89
- */
90
- function cursorOf(from, to) {
91
- return `${from ?? ''}:${to ?? ''}`;
92
- }
93
- function rangeOf(q) {
94
- if (q?.after !== undefined) {
95
- const ends = /^(-?\d*):(-?\d*)$/.exec(q.after);
96
- if (ends === null) {
97
- throw new InvalidError(`${JSON.stringify(q.after)} is no page's next`);
98
- }
99
- return {
100
- from: ends[1] === '' ? undefined : BigInt(ends[1]),
101
- to: ends[2] === '' ? undefined : BigInt(ends[2]),
102
- };
97
+ /** The keys the events hold, in fields and in context, sorted; `prefix` narrows them. */
98
+ async fieldNames(query) {
99
+ const { prefix, ...rest } = query ?? {};
100
+ const asked = queryOf(rest);
101
+ const answer = await this.ask('records.fieldNames', handle => RecordsFieldNamesAsked.encode({ query: { ...asked, handle }, prefix }));
102
+ return RecordsNames.decode(answer).names ?? [];
103
103
  }
104
- if (q?.since !== undefined && q.from !== undefined) {
105
- throw new InvalidError('a range starts since a span before now or from a time, not both');
104
+ /**
105
+ * The events written from now on that the query finds, a batch as they
106
+ * are written: `{ events, missed }`. A reader slower than the writers
107
+ * skips what left the store's hold, and the batch after says how many.
108
+ * It ends when its loop is left, and when the server's store closes.
109
+ */
110
+ async *tail(query) {
111
+ const asked = queryOf(query);
112
+ const stream = await this.opened('records.tail', handle => RecordsQuery.encode({ ...asked, handle }));
113
+ try {
114
+ for (;;) {
115
+ const event = await stream.next();
116
+ stream.consumed(event.body.length);
117
+ if (event.end) {
118
+ return;
119
+ }
120
+ const batch = RecordsTailBatch.decode(event.body);
121
+ yield { events: (batch.events ?? []).map(eventOf), missed: batch.missed ?? 0 };
122
+ }
123
+ }
124
+ finally {
125
+ stream.cancel();
126
+ }
106
127
  }
107
- const from = q?.since !== undefined
108
- ? nanos(new Date(Date.now() - ms(q.since)))
109
- : q?.from === undefined
110
- ? undefined
111
- : nanos(q.from);
112
- return { from, to: q?.to === undefined ? undefined : nanos(q.to) };
113
128
  }
114
- function queryOf(q) {
115
- return {
116
- ...rangeOf(q),
117
- streams: q?.streams !== undefined && q.streams.length > 0 ? q.streams : undefined,
118
- names: q?.names !== undefined && q.names.length > 0 ? q.names : undefined,
119
- minLevel: q?.minLevel === undefined ? undefined : levelOf(q.minLevel),
120
- traceId: idOf(q?.traceId, 16, 'trace'),
121
- attrs: fieldsOf(q?.attrs),
122
- context: fieldsOf(q?.context),
123
- newest: q?.newest === true ? true : undefined,
124
- limit: q?.limit,
125
- budgetBlocks: q?.budget?.blocks,
126
- budgetBytes: q?.budget?.bytes,
127
- budgetRecords: q?.budget?.records,
128
- search: q?.search === '' ? undefined : q?.search,
129
- };
129
+ /** Opens a stream of events: lazily, at its first call on a connection. */
130
+ export function openEvents(link, stream, options) {
131
+ checkName(stream, 'stream');
132
+ const keep = options?.keep === undefined ? undefined : ms(options.keep);
133
+ return new Events(link, stream, RecordsOpen.encode({ stream, keep }));
130
134
  }
131
- export class Records {
135
+ /** A stream of events: written with `append`, which waits for the file, and read back. */
136
+ export class Events extends Reads {
137
+ stream;
132
138
  #link;
133
- #loggers = [];
134
- constructor(link) {
139
+ #via;
140
+ #open;
141
+ /** Events come from `store.events(stream)`. */
142
+ constructor(link, stream, open) {
143
+ super();
144
+ this.stream = stream;
135
145
  this.#link = link;
146
+ this.#via = viaLink(link);
147
+ this.#open = open;
136
148
  }
137
- /**
138
- * A logger of a stream: its calls return at once, and its lines reach the
139
- * server every second, as Go's slog handler's and Python's logging.Handler's do.
140
- */
141
- logger(stream, options) {
142
- const logger = newLogger(lines => this.#writeLines(lines), stream, options);
143
- this.#loggers.push(logger);
144
- return logger;
145
- }
146
- /** Hands over what every logger holds, as the store does when it closes. */
147
- async stop() {
148
- await Promise.all(this.#loggers.map(logger => logger.stop()));
149
+ async append(first, fields, options) {
150
+ const events = typeof first === 'string' ? [oneEvent(first, fields, options)] : first;
151
+ const encoded = events.map(newEventOf);
152
+ await this.#call('records.append', handle => RecordsAppend.encode({ handle, events: encoded }), 'write');
149
153
  }
150
- /**
151
- * Appends records in one transaction, all or none; a refused one names
152
- * itself as `call`, with its stream and name. A read sees them at once. A
153
- * record of no trace of its own takes the one withTrace runs the call in.
154
- */
155
- async append(records) {
156
- const carried = currentTrace();
157
- const list = Array.isArray(records) ? records : [records];
158
- return this.#write(carried === undefined
159
- ? list
160
- : list.map(r => r.traceId === undefined
161
- ? { ...r, traceId: carried.traceId, spanId: carried.spanId }
162
- : r));
163
- }
164
- async #write(records) {
165
- const now = BigInt(Date.now()) * nsPerMs;
166
- await this.#send(records.map(r => ({
167
- at: r.at === undefined ? now : nanos(r.at),
168
- stream: r.stream,
169
- name: r.name,
170
- level: r.level === undefined ? undefined : levelOf(r.level),
171
- body: r.body,
172
- traceId: idOf(r.traceId, 16, 'trace'),
173
- spanId: idOf(r.spanId, 8, 'span'),
174
- context: fieldsOf(r.context),
175
- attrs: fieldsOf(r.attrs),
176
- })));
177
- }
178
- /** Appends a logger's lines, their fields already JSON and their traces taken as each was logged. */
179
- async #writeLines(lines) {
180
- await this.#send(lines.map(line => ({
181
- at: nanos(line.at),
182
- stream: line.stream,
154
+ /** A logger's lines, each an event of this stream. */
155
+ appendLines(lines) {
156
+ const events = lines.map(line => ({
183
157
  name: line.event ?? 'log',
158
+ at: nanosOf(line.at),
184
159
  level: line.level,
185
- body: line.msg,
186
- traceId: idOf(line.traceId, 16, 'trace'),
187
- spanId: idOf(line.spanId, 8, 'span'),
188
- context: line.context.length === 0 ? undefined : [...line.context],
189
- attrs: line.attrs.length === 0 ? undefined : [...line.attrs],
190
- })));
191
- }
192
- async #send(records) {
193
- await this.#link.run('write', connection => connection.session.call(methods['records.append'], RecordsBatch.encode({ records })));
160
+ message: line.msg,
161
+ fields: objectOf(line.attrs),
162
+ context: objectOf(line.context),
163
+ ...wireTrace(line.traceId, line.spanId),
164
+ }));
165
+ return this.#call('records.append', handle => RecordsAppend.encode({ handle, events }), 'write').then(() => { });
194
166
  }
195
167
  /**
196
- * One page of the records a query matches, from one snapshot, in event-time
197
- * order; next, while a limit or a budget ended the page early, is passed back
198
- * as `after` with the same query. A page never splits a timestamp.
168
+ * A writer of another program's output into this stream: each line it is
169
+ * given becomes an event at the time it arrived, a stack trace's or a JSON
170
+ * value's lines one, and a write never waits.
171
+ *
172
+ * const out = store.events('worker').lines()
173
+ * child.stdout.on('data', chunk => out.write(chunk))
174
+ * child.on('close', () => out.end())
199
175
  */
200
- async scan(query) {
201
- const asked = queryOf(query);
202
- const got = await this.#link.run('read', connection => download(connection, methods['records.read'], QueryMessage.encode(asked)));
203
- const page = RecordsPage.decode(got.trailer);
204
- const items = got.items.map(item => recordOf(RecordsRecord.decode(item)));
205
- if (page.more !== true) {
206
- return { items, next: undefined };
207
- }
208
- const moved = (end, sent) => end !== undefined ? BigInt(end) : sent !== undefined ? BigInt(sent) : undefined;
209
- return { items, next: cursorOf(moved(page.from, asked.from), moved(page.to, asked.to)) };
176
+ lines() {
177
+ return new Lines(async () => {
178
+ const connection = await this.#link.connection();
179
+ const handle = await handleOn(connection, methods['records.open'], this.#open);
180
+ const body = Handle.encode({ handle });
181
+ const stream = await streamOf(connection, methods['records.lines'], body, false);
182
+ return { stream, piece: pieceOf(connection) };
183
+ });
210
184
  }
211
- /** Every record a query matches, a page at a time. */
212
- async *all(query) {
213
- let page = await this.scan(query);
214
- yield* page.items;
215
- while (page.next !== undefined) {
216
- page = await this.scan({ ...query, after: page.next });
217
- yield* page.items;
218
- }
185
+ ask(method, body) {
186
+ return this.#call(method, body, 'read');
187
+ }
188
+ opened(method, body) {
189
+ return this.#link.run('read', async (connection) => {
190
+ const handle = await handleOn(connection, methods['records.open'], this.#open);
191
+ return streamOf(connection, methods[method], body(handle), true);
192
+ });
193
+ }
194
+ #call(method, body, idempotence) {
195
+ return this.#via.call(methods['records.open'], this.#open, methods[method], body, idempotence);
196
+ }
197
+ }
198
+ /** Every stream's events, to read: `store.events()`. */
199
+ export class AllEvents extends Reads {
200
+ #link;
201
+ /** AllEvents come from `store.events()`. */
202
+ constructor(link) {
203
+ super();
204
+ this.#link = link;
205
+ }
206
+ /** Every stream with events, by name, with what it keeps, where that begins and its newest event's time. */
207
+ async streams() {
208
+ const answer = await this.ask('records.streams', () => Empty.encode({}));
209
+ return (RecordsStreams.decode(answer).streams ?? []).map(stream => ({
210
+ name: stream.name ?? '',
211
+ keep: spelled(stream.keep ?? 0),
212
+ retainedFrom: dateOf(stream.retainedFrom ?? 0n),
213
+ newest: dateOf(stream.newest ?? 0n),
214
+ }));
219
215
  }
220
216
  /**
221
- * The sealed records after a cursor, in the order they were sealed; the
222
- * cursor to follow from next, and the segments retention removed before
223
- * the cursor reached them. A record reaches a follower once it is sealed.
217
+ * Every event once, a batch a call, from a cursor the program keeps, the
218
+ * oldest event kept without one: `{ events, cursor, expired }`. With no
219
+ * event past the cursor, the batch is empty and its cursor the one given.
224
220
  */
225
- async follow(cursor, limit) {
226
- const got = await this.#link.run('read', connection => download(connection, methods['records.follow'], RecordsCursor.encode({ segment: cursor?.segment, row: cursor?.row, limit })));
227
- const trailer = RecordsCursor.decode(got.trailer);
221
+ async poll(cursor, options) {
222
+ const answer = await this.ask('records.poll', () => RecordsPollAsked.encode({ cursor, limit: options?.limit }));
223
+ const batch = RecordsBatch.decode(answer);
228
224
  return {
229
- records: got.items.map(item => recordOf(RecordsRecord.decode(item))),
230
- cursor: { segment: trailer.segment ?? 0, row: trailer.row ?? 0 },
231
- expired: trailer.expired ?? 0,
225
+ events: (batch.events ?? []).map(eventOf),
226
+ cursor: batch.cursor ?? '',
227
+ expired: batch.expired ?? 0,
232
228
  };
233
229
  }
230
+ ask(method, body) {
231
+ return this.#link.run('read', connection => connection.session.call(methods[method], body(undefined)));
232
+ }
234
233
  /**
235
- * A writer of another program's output, pino's lines or a child's stdout,
236
- * that never waits: each line becomes a record of the stream as the server's
237
- * Lines makes it, a stack trace's lines joined and a JSON or logfmt line's
238
- * fields kept. What does not fit its buffer is dropped and counted.
234
+ * What reads met that no longer reads, which `drop` removes: each head
235
+ * row and segment with its stream and its events' times. What the store
236
+ * has not met since it opened is not listed.
239
237
  */
240
- lines(stream, options) {
241
- if (stream === '') {
242
- throw new InvalidError('lines of no stream');
243
- }
244
- return new LinesWriter(this.#link, stream, options?.buffer ?? 1 << 20);
245
- }
246
- /** The rows the server's records have met that no longer read. */
247
238
  async damaged() {
248
- const body = await this.#link.run('read', connection => connection.session.call(methods['records.damaged'], Empty.encode({})));
249
- return (RecordsDamages.decode(body).damages ?? []).map(d => ({
250
- stream: textOf(d.stream),
251
- segment: d.segment,
252
- headRow: d.headRow,
253
- from: d.from ?? 0n,
254
- to: d.to ?? 0n,
255
- reason: d.reason ?? '',
239
+ const answer = await this.ask('records.damaged', () => Empty.encode({}));
240
+ return (RecordsDamages.decode(answer).damages ?? []).map(damage => ({
241
+ part: damage.part === 'segment' ? 'segment' : 'head row',
242
+ id: damage.id ?? 0,
243
+ stream: damage.stream ?? '',
244
+ first: dateOf(damage.first ?? 0n),
245
+ last: dateOf(damage.last ?? 0n),
246
+ error: damage.error ?? '',
256
247
  }));
257
248
  }
258
- /** Removes a damaged row, a repair an admin connection alone may make. */
249
+ /**
250
+ * Removes a head row or a segment that `damaged` listed, so that the
251
+ * reads over it work again; what still reads is refused.
252
+ *
253
+ * for (const damage of await store.events().damaged()) await store.events().drop(damage)
254
+ */
259
255
  async drop(damage) {
260
- await this.#link.run('write', connection => connection.session.call(methods['records.drop'], RecordsDamage.encode(damage)));
256
+ const body = RecordsDamage.encode({
257
+ ...damage,
258
+ first: nanosOf(damage.first),
259
+ last: nanosOf(damage.last),
260
+ });
261
+ await this.#link.run('write', connection => connection.session.call(methods['records.drop'], body));
262
+ }
263
+ opened(method, body) {
264
+ return this.#link.run('read', connection => streamOf(connection, methods[method], body(undefined), true));
261
265
  }
262
266
  }
263
- /** The most one DATA of lines carries. */
264
- const linesChunk = 64 << 10;
265
267
  /**
266
- * An upload of lines that stays open while it is written: write never waits,
267
- * holding what the stream's credit does not let go yet up to its buffer and
268
- * dropping past it. A connection lost drops what it held and the next write
269
- * uploads on the next connection.
268
+ * A writer of another program's output into a stream, as `events.lines()`
269
+ * begins it: each line becomes an event, and a write never waits. It sends
270
+ * whole lines, so that what it has no room for, `dropped` counts as lines.
270
271
  */
271
- export class LinesWriter {
272
- #link;
273
- #stream;
274
- #most;
275
- #waiting = [];
276
- #held = 0;
277
- #pumping;
278
- #upload;
272
+ export class Lines {
273
+ /** whole lines not sent yet */
274
+ #queued = [];
275
+ #queuedBytes = 0;
276
+ /** a line not ended yet */
277
+ #partial = new Uint8Array(0);
278
+ /** the line being written is longer than an event holds: the rest of it goes too */
279
+ #discarding = false;
280
+ #dropped = 0;
279
281
  #ended = false;
280
- /** bytes dropped since the writer began: a full buffer, or a lost connection */
281
- dropped = 0;
282
- constructor(link, stream, most) {
283
- this.#link = link;
284
- this.#stream = stream;
285
- this.#most = most;
282
+ #failure;
283
+ #wake;
284
+ #sent;
285
+ /** Lines come from `events.lines()`. */
286
+ constructor(open) {
287
+ this.#sent = this.#send(open).catch((err) => {
288
+ this.#failure = err instanceof Error ? err : new Error(String(err));
289
+ for (const bytes of this.#queued.splice(0)) {
290
+ this.#dropped += linesIn(bytes);
291
+ }
292
+ this.#queuedBytes = 0;
293
+ });
286
294
  }
287
- /** Takes a piece of output, cut anywhere. It returns at once. */
295
+ /** The lines dropped: past what the writer holds, and, once it ended, by the store. */
296
+ get dropped() {
297
+ return this.#dropped;
298
+ }
299
+ /** Takes a piece of output, cut anywhere; it never waits. */
288
300
  write(chunk) {
289
301
  if (this.#ended) {
290
- throw new InvalidError('a write after the lines ended');
302
+ throw new ClosedError('lines written after their end');
291
303
  }
292
- const bytes = typeof chunk === 'string' ? new TextEncoder().encode(chunk) : chunk;
293
- if (bytes.length === 0) {
294
- return true;
304
+ let bytes = typeof chunk === 'string' ? encoder.encode(chunk) : chunk;
305
+ if (this.#discarding) {
306
+ const end = bytes.indexOf(newline);
307
+ if (end < 0) {
308
+ return;
309
+ }
310
+ this.#discarding = false;
311
+ bytes = bytes.subarray(end + 1);
295
312
  }
296
- if (this.#held + bytes.length > this.#most) {
297
- this.dropped += bytes.length;
298
- return false;
313
+ const joined = concat(this.#partial, bytes);
314
+ const cut = joined.lastIndexOf(newline) + 1;
315
+ this.#partial = joined.slice(cut);
316
+ if (this.#partial.length > eventRoom) {
317
+ this.#partial = new Uint8Array(0);
318
+ this.#discarding = true;
319
+ this.#dropped++;
299
320
  }
300
- for (let at = 0; at < bytes.length; at += linesChunk) {
301
- this.#waiting.push(bytes.subarray(at, at + linesChunk));
321
+ this.#queue(joined.subarray(0, cut));
322
+ }
323
+ /**
324
+ * Ends the writer: what it holds goes, a last line without its newline
325
+ * too, and it returns once the store wrote them.
326
+ */
327
+ async end() {
328
+ if (!this.#ended) {
329
+ this.#ended = true;
330
+ this.#queue(this.#partial);
331
+ this.#partial = new Uint8Array(0);
332
+ this.#wake?.();
302
333
  }
303
- this.#held += bytes.length;
304
- this.#pumping ??= this.#pump().finally(() => {
305
- this.#pumping = undefined;
306
- });
307
- return true;
308
- }
309
- async #pump() {
310
- while (this.#waiting.length > 0) {
311
- const piece = this.#waiting[0];
312
- try {
313
- const upload = await this.#open();
314
- await upload.send(piece, false);
334
+ await this.#sent;
335
+ if (this.#failure !== undefined) {
336
+ throw this.#failure;
337
+ }
338
+ }
339
+ /** Holds bytes for the server, or drops and counts their lines when it holds no room. */
340
+ #queue(bytes) {
341
+ if (bytes.length === 0) {
342
+ return;
343
+ }
344
+ if (this.#failure !== undefined || this.#queuedBytes + bytes.length > linesHeld) {
345
+ this.#dropped += linesIn(bytes);
346
+ return;
347
+ }
348
+ this.#queued.push(bytes);
349
+ this.#queuedBytes += bytes.length;
350
+ this.#wake?.();
351
+ }
352
+ /** Sends what is held as it comes, in pieces within the stream's credit, then the end. */
353
+ async #send(open) {
354
+ const { stream, piece } = await open();
355
+ for (;;) {
356
+ const bytes = this.#queued.shift();
357
+ if (bytes === undefined) {
358
+ if (this.#ended) {
359
+ break;
360
+ }
361
+ await new Promise(resolve => {
362
+ this.#wake = resolve;
363
+ });
364
+ this.#wake = undefined;
365
+ continue;
366
+ }
367
+ this.#queuedBytes -= bytes.length;
368
+ for (let at = 0; at < bytes.length; at += piece) {
369
+ await stream.send(RecordsPiece.encode({ bytes: bytes.subarray(at, at + piece) }), false);
315
370
  }
316
- catch {
317
- this.#upload = undefined;
318
- this.dropped += this.#held;
319
- this.#waiting = [];
320
- this.#held = 0;
371
+ }
372
+ await stream.send(new Uint8Array(0), true);
373
+ for (;;) {
374
+ const event = await stream.next();
375
+ if (event.end) {
376
+ this.#dropped += RecordsDropped.decode(event.body).dropped ?? 0;
321
377
  return;
322
378
  }
323
- this.#waiting.shift();
324
- this.#held -= piece.length;
325
379
  }
326
380
  }
327
- async #open() {
328
- if (this.#upload !== undefined && !this.#upload.finished) {
329
- return this.#upload;
381
+ }
382
+ /** A stream of the method, open once its RESPONSE came; `end` leaves the client nothing to send. */
383
+ async function streamOf(connection, method, body, end) {
384
+ const stream = await connection.session.open(method, body, end);
385
+ await stream.next();
386
+ return stream;
387
+ }
388
+ function concat(a, b) {
389
+ if (a.length === 0) {
390
+ return b;
391
+ }
392
+ const joined = new Uint8Array(a.length + b.length);
393
+ joined.set(a);
394
+ joined.set(b, a.length);
395
+ return joined;
396
+ }
397
+ /** The lines bytes hold: one a newline ends, and one more past the last. */
398
+ function linesIn(bytes) {
399
+ let lines = bytes[bytes.length - 1] === newline ? 0 : 1;
400
+ for (const byte of bytes) {
401
+ if (byte === newline) {
402
+ lines++;
330
403
  }
331
- const connection = await this.#link.connection();
332
- this.#upload = await connection.session.open(methods['records.lines'], RecordsStream.encode({ stream: this.#stream }), false);
333
- return this.#upload;
334
404
  }
335
- /** Hands over what the writer holds, a line cut short included, and ends the upload. */
336
- async end() {
337
- this.#ended = true;
338
- await this.#pumping;
339
- const upload = this.#upload;
340
- if (upload === undefined || upload.finished) {
341
- return;
405
+ return lines;
406
+ }
407
+ /**
408
+ * A logger of `stream` that keeps its lines there through an append of their
409
+ * own, and what it declares its config `log` with.
410
+ */
411
+ export function storeLogger(link, stream, options = {}) {
412
+ const { keep, ...told } = options;
413
+ const events = openEvents(link, stream, keep === undefined ? undefined : { keep });
414
+ return newStoreLogger(lines => events.appendLines(lines), stream, told);
415
+ }
416
+ /** The event an append of a name, its fields and its options writes. */
417
+ function oneEvent(name, fields, options) {
418
+ return fields === undefined ? { ...options, name } : { ...options, name, fields };
419
+ }
420
+ /** An event as the wire carries it, the current trace's ids when it gives none. */
421
+ function newEventOf(event, place) {
422
+ const named = `event ${place} ${JSON.stringify(event.name)}`;
423
+ if (typeof event.name !== 'string' || event.name === '') {
424
+ throw new InvalidError(`${named}: an event has a name`);
425
+ }
426
+ if (event.at !== undefined && Number.isNaN(event.at.getTime())) {
427
+ throw new InvalidError(`${named}: a time that is no date`);
428
+ }
429
+ const given = event.traceId === undefined
430
+ ? traceOf(currentTrace())
431
+ : { traceId: event.traceId, spanId: event.spanId };
432
+ return {
433
+ name: event.name,
434
+ at: event.at === undefined ? undefined : nanosOf(event.at),
435
+ level: event.level === undefined ? undefined : levelNumber(event.level, named),
436
+ message: event.message,
437
+ fields: event.fields === undefined ? undefined : objectOf(encodeFields(event.fields, undefined)),
438
+ context: event.context === undefined ? undefined : objectOf(encodeFields(event.context, undefined)),
439
+ ...wireTrace(given.traceId, given.spanId),
440
+ };
441
+ }
442
+ /** A query as the wire carries it; a condition given as undefined reads nothing it was not asked. */
443
+ function queryOf(query) {
444
+ const q = query ?? {};
445
+ for (const [key, value] of Object.entries(q)) {
446
+ if (value === undefined) {
447
+ throw new InvalidError(`a query's ${key} given as undefined: leave it out to read without it`);
448
+ }
449
+ }
450
+ const where = q.where === undefined ? undefined : Object.entries(q.where);
451
+ for (const [key, value] of where ?? []) {
452
+ if (value === undefined) {
453
+ throw new InvalidError(`a query's where ${JSON.stringify(key)} given as undefined`);
454
+ }
455
+ }
456
+ return {
457
+ since: q.since === undefined ? undefined : ms(q.since),
458
+ from: q.from === undefined ? undefined : nanosOf(q.from),
459
+ to: q.to === undefined ? undefined : nanosOf(q.to),
460
+ streams: q.streams === undefined ? undefined : [...q.streams],
461
+ names: q.names === undefined ? undefined : [...q.names],
462
+ minLevel: q.minLevel === undefined ? undefined : levelNumber(q.minLevel, 'minLevel'),
463
+ traceId: q.traceId === undefined ? undefined : idBytes(q.traceId, 16, 'traceId'),
464
+ where: where === undefined
465
+ ? undefined
466
+ : Object.fromEntries(where.map(([key, value]) => [key, jsonOf(value)])),
467
+ search: q.search,
468
+ newest: q.newest,
469
+ limit: q.limit,
470
+ after: q.after,
471
+ };
472
+ }
473
+ function eventOf(read) {
474
+ const event = {
475
+ at: dateOf(read.at ?? 0n),
476
+ stream: read.stream ?? '',
477
+ name: read.name ?? '',
478
+ fields: read.fields === undefined ? {} : parseJson(read.fields),
479
+ context: read.context === undefined ? {} : parseJson(read.context),
480
+ };
481
+ if (read.level !== undefined) {
482
+ event.level = levelNames[Math.min(5, Math.max(0, Math.floor(read.level / 4) + 2))];
483
+ }
484
+ if (read.message !== undefined) {
485
+ event.message = read.message;
486
+ }
487
+ if (read.traceId !== undefined) {
488
+ event.traceId = Buffer.from(read.traceId).toString('hex');
489
+ }
490
+ if (read.spanId !== undefined) {
491
+ event.spanId = Buffer.from(read.spanId).toString('hex');
492
+ }
493
+ return event;
494
+ }
495
+ /** Key, JSON, key, JSON… as the JSON object they spell; none for no field. */
496
+ function objectOf(flat) {
497
+ if (flat.length === 0) {
498
+ return undefined;
499
+ }
500
+ const pairs = [];
501
+ for (let at = 0; at < flat.length; at += 2) {
502
+ pairs.push(`${JSON.stringify(flat[at])}:${flat[at + 1]}`);
503
+ }
504
+ return `{${pairs.join(',')}}`;
505
+ }
506
+ function levelNumber(level, what) {
507
+ if (typeof level === 'number') {
508
+ if (!Number.isInteger(level)) {
509
+ throw new InvalidError(`${what}: the level ${level}, where slog counts whole numbers`);
510
+ }
511
+ return level;
512
+ }
513
+ const number = levelNumbers[level];
514
+ if (number === undefined) {
515
+ throw new InvalidError(`${what}: the level ${JSON.stringify(level)}, not one of ${levelNames.join(', ')}`);
516
+ }
517
+ return number;
518
+ }
519
+ /** A trace's ids as hex, when the trace has both. */
520
+ function traceOf(trace) {
521
+ if (trace === undefined) {
522
+ return {};
523
+ }
524
+ const hex = (id) => typeof id === 'string' || id === undefined ? id : Buffer.from(id).toString('hex');
525
+ return { traceId: hex(trace.traceId), spanId: hex(trace.spanId) };
526
+ }
527
+ /** A trace and its span as the wire carries them: both, or neither, since the store keeps a span with its trace. */
528
+ function wireTrace(traceId, spanId) {
529
+ if (traceId === undefined || spanId === undefined) {
530
+ return {};
531
+ }
532
+ return { traceId: idBytes(traceId, 16, 'traceId'), spanId: idBytes(spanId, 8, 'spanId') };
533
+ }
534
+ function idBytes(hex, size, what) {
535
+ if (!new RegExp(`^[0-9a-fA-F]{${2 * size}}$`).test(hex)) {
536
+ throw new InvalidError(`${what} ${JSON.stringify(hex)}: ${2 * size} hex digits`);
537
+ }
538
+ return Uint8Array.from(Buffer.from(hex, 'hex'));
539
+ }
540
+ function nanosOf(date) {
541
+ return BigInt(date.getTime()) * 1000000n;
542
+ }
543
+ function dateOf(nanos) {
544
+ return new Date(Number(nanos / 1000000n));
545
+ }
546
+ function mapValues(map, read) {
547
+ return Object.fromEntries(Object.entries(map ?? {}).map(([key, value]) => [key, read(value)]));
548
+ }
549
+ /** Milliseconds as a span is written: '30d', '12h', '90s', '250ms'. */
550
+ function spelled(millis) {
551
+ for (const [unit, size] of [
552
+ ['d', 86_400_000],
553
+ ['h', 3_600_000],
554
+ ['m', 60_000],
555
+ ['s', 1000],
556
+ ]) {
557
+ if (millis >= size && millis % size === 0) {
558
+ return `${millis / size}${unit}`;
342
559
  }
343
- await upload.send(new Uint8Array(0), true);
344
- await upload.next();
345
560
  }
561
+ return `${millis}ms`;
346
562
  }