@tinyshed/tinystore 0.1.0-rc.5 → 0.1.0-rc.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (122) hide show
  1. package/README.md +37 -37
  2. package/bin/tinystore.js +12 -1
  3. package/dist/backup.d.ts +33 -0
  4. package/dist/backup.js +49 -0
  5. package/dist/blobs.d.ts +160 -79
  6. package/dist/blobs.js +510 -217
  7. package/dist/channels.d.ts +90 -0
  8. package/dist/channels.js +291 -0
  9. package/dist/clock.js +3 -3
  10. package/dist/config.d.ts +209 -86
  11. package/dist/config.js +581 -538
  12. package/dist/connection.d.ts +64 -2
  13. package/dist/connection.js +201 -16
  14. package/dist/console.d.ts +27 -3
  15. package/dist/console.js +165 -12
  16. package/dist/dotenv.d.ts +7 -0
  17. package/dist/dotenv.js +79 -0
  18. package/dist/errors.d.ts +61 -1
  19. package/dist/errors.js +81 -1
  20. package/dist/handles.d.ts +44 -2
  21. package/dist/handles.js +81 -2
  22. package/dist/index.d.ts +17 -11
  23. package/dist/index.js +5 -3
  24. package/dist/instruments.d.ts +177 -0
  25. package/dist/instruments.js +819 -0
  26. package/dist/jobs.d.ts +335 -239
  27. package/dist/jobs.js +716 -536
  28. package/dist/json.d.ts +2 -0
  29. package/dist/json.js +122 -0
  30. package/dist/kv.d.ts +130 -243
  31. package/dist/kv.js +273 -640
  32. package/dist/limits.d.ts +80 -0
  33. package/dist/limits.js +139 -0
  34. package/dist/locks.d.ts +62 -0
  35. package/dist/locks.js +237 -0
  36. package/dist/logger.d.ts +45 -9
  37. package/dist/logger.js +150 -38
  38. package/dist/metrics.d.ts +210 -209
  39. package/dist/metrics.js +319 -556
  40. package/dist/once.d.ts +19 -18
  41. package/dist/once.js +74 -63
  42. package/dist/query.d.ts +181 -0
  43. package/dist/query.js +788 -0
  44. package/dist/records.d.ts +241 -150
  45. package/dist/records.js +507 -291
  46. package/dist/runtime/addon.d.ts +14 -0
  47. package/dist/runtime/addon.js +69 -0
  48. package/dist/runtime/bun.js +5 -7
  49. package/dist/runtime/node.js +5 -7
  50. package/dist/runtime/pipe.d.ts +17 -0
  51. package/dist/runtime/pipe.js +133 -0
  52. package/dist/runtime/shared.d.ts +6 -0
  53. package/dist/runtime/shared.js +19 -0
  54. package/dist/runtime.d.ts +5 -2
  55. package/dist/search.d.ts +189 -0
  56. package/dist/search.js +281 -0
  57. package/dist/session.d.ts +31 -3
  58. package/dist/session.js +153 -41
  59. package/dist/sql.d.ts +228 -78
  60. package/dist/sql.js +745 -240
  61. package/dist/store.d.ts +244 -24
  62. package/dist/store.js +270 -67
  63. package/dist/time.d.ts +5 -1
  64. package/dist/time.js +22 -2
  65. package/dist/tx.d.ts +61 -0
  66. package/dist/tx.js +314 -0
  67. package/dist/wire/codec.d.ts +23 -7
  68. package/dist/wire/codec.js +56 -35
  69. package/dist/wire/frame.d.ts +2 -0
  70. package/dist/wire/frame.js +31 -14
  71. package/dist/wire/messages.d.ts +259 -0
  72. package/dist/wire/messages.js +22 -0
  73. package/dist/wire/msgpack.d.ts +49 -1
  74. package/dist/wire/msgpack.js +293 -56
  75. package/dist/wire/protocol.d.ts +2683 -0
  76. package/dist/wire/protocol.js +10846 -0
  77. package/package.json +39 -38
  78. package/src/backup.ts +81 -0
  79. package/src/blobs.ts +678 -301
  80. package/src/channels.ts +377 -0
  81. package/src/clock.ts +4 -4
  82. package/src/config.ts +902 -612
  83. package/src/connection.ts +279 -21
  84. package/src/console.ts +180 -12
  85. package/src/dotenv.ts +82 -0
  86. package/src/errors.ts +105 -0
  87. package/src/handles.ts +157 -8
  88. package/src/index.ts +158 -52
  89. package/src/instruments.ts +1042 -0
  90. package/src/jobs.ts +1017 -733
  91. package/src/json.ts +133 -0
  92. package/src/kv.ts +424 -847
  93. package/src/limits.ts +202 -0
  94. package/src/locks.ts +320 -0
  95. package/src/logger.ts +209 -40
  96. package/src/metrics.ts +557 -752
  97. package/src/once.ts +86 -83
  98. package/src/query.ts +1005 -0
  99. package/src/records.ts +795 -418
  100. package/src/runtime/addon.ts +99 -0
  101. package/src/runtime/bun.ts +5 -7
  102. package/src/runtime/node.ts +5 -7
  103. package/src/runtime/pipe.ts +178 -0
  104. package/src/runtime/shared.ts +20 -0
  105. package/src/runtime.ts +5 -2
  106. package/src/search.ts +480 -0
  107. package/src/session.ts +187 -41
  108. package/src/sql.ts +993 -306
  109. package/src/store.ts +473 -78
  110. package/src/time.ts +48 -3
  111. package/src/tx.ts +426 -0
  112. package/src/wire/codec.ts +78 -37
  113. package/src/wire/frame.ts +44 -14
  114. package/src/wire/messages.ts +25 -0
  115. package/src/wire/msgpack.ts +327 -56
  116. package/src/wire/protocol.ts +11709 -0
  117. package/dist/limiter.d.ts +0 -45
  118. package/dist/limiter.js +0 -68
  119. package/dist/quota.d.ts +0 -44
  120. package/dist/quota.js +0 -98
  121. package/src/limiter.ts +0 -99
  122. package/src/quota.ts +0 -137
package/src/records.ts CHANGED
@@ -1,537 +1,914 @@
1
- // The application's logs and events in records.db, as Go's records package
2
- // keeps them: one log of the store's whose calls name their streams, read by
3
- // time and by what records hold, followed in the order they were sealed.
4
-
5
- import { type Connection, download, type Link } from './connection.ts'
6
- import type { ConsoleLine } from './console.ts'
7
- import { InvalidError } from './errors.ts'
8
- import type { Page } from './handles.ts'
9
- import { type Logger, type LoggerOptions, newLogger } from './logger.ts'
1
+ // Streams of events, as docs/guides/records/README.md has them, over the protocol: a
2
+ // stream opened by name, its events appended whole and read back by time and
3
+ // by what they hold, a page at a time, and every stream read together. A
4
+ // store's logger keeps its lines in a stream through the same append; a tail
5
+ // follows what is written from now on, and a writer of lines turns another
6
+ // program's output into events.
7
+
8
+ import { type Connection, type Link, pieceOf } from './connection.ts'
9
+ import { type ConsoleLine, jsonOf } from './console.ts'
10
+ import { ClosedError, InvalidError } from './errors.ts'
11
+ import { checkName, handleOn, type Via, viaLink } from './handles.ts'
12
+ import { parseJson } from './json.ts'
13
+ import {
14
+ encodeFields,
15
+ type LogConfig,
16
+ type Logger,
17
+ type LoggerOptions,
18
+ newStoreLogger,
19
+ } from './logger.ts'
10
20
  import type { Stream } from './session.ts'
11
- import { type Duration, ms } from './time.ts'
12
- import { currentTrace } from './trace.ts'
21
+ import { type Duration, type DurationText, ms } from './time.ts'
22
+ import { currentTrace, type Trace } from './trace.ts'
13
23
  import {
14
24
  Empty,
25
+ Handle,
15
26
  methods,
16
- RecordsQuery as QueryMessage,
27
+ RecordsAggregateAsked,
28
+ RecordsAppend,
17
29
  RecordsBatch,
18
- RecordsCursor,
30
+ RecordsCount,
19
31
  RecordsDamage,
20
32
  RecordsDamages,
33
+ RecordsDropped,
34
+ type RecordsEvent,
35
+ RecordsFieldNamesAsked,
36
+ RecordsNames,
37
+ type RecordsNewEvent,
38
+ RecordsOpen,
21
39
  RecordsPage,
22
- RecordsRecord,
23
- RecordsStream,
24
- } from './wire/messages.ts'
25
-
26
- /** slog's levels: -4 debug, 0 info, 4 warn, 8 error, or any other it may name. */
40
+ RecordsPiece,
41
+ RecordsPollAsked,
42
+ RecordsQuery,
43
+ RecordsRows,
44
+ RecordsStreams,
45
+ RecordsTailBatch,
46
+ RecordsTopAsked,
47
+ RecordsTops,
48
+ } from './wire/protocol.ts'
49
+
50
+ /** A level a logger logs at: one of its four names, or slog's number between them. */
27
51
  export type Level = 'debug' | 'info' | 'warn' | 'error' | number
28
52
 
29
- const levels = { debug: -4, info: 0, warn: 4, error: 8 } as const
30
-
31
- function levelOf(level: Level): number {
32
- return typeof level === 'number' ? level : levels[level]
33
- }
53
+ /** An event's level: what loggers and the programs whose lines are kept write. */
54
+ export type EventLevel = 'trace' | 'debug' | 'info' | 'warn' | 'error' | 'fatal'
34
55
 
35
- /**
36
- * Fields: keys and values, each value written as JSON. An object's keys keep
37
- * their order; an array of pairs may repeat a key.
38
- */
56
+ /** What happened, or what it happened in: an object, or its pairs in their order. */
39
57
  export type Fields = Record<string, unknown> | readonly (readonly [string, unknown])[]
40
58
 
41
- /** A record to append: when, where, what, and its fields. */
42
- export interface RecordInput {
43
- /** its time: a Date, or unix nanoseconds as a bigint; now when absent */
44
- at?: Date | bigint
45
- /** the namespace the application names: 'web', 'billing' */
59
+ /** An event as a read gives it. */
60
+ export interface Event {
61
+ /** its time, which the file keeps to the nanosecond */
62
+ at: Date
46
63
  stream: string
47
- /** the event: 'log' for a log line */
64
+ /** what happened; `log` for a logger's line */
48
65
  name: string
49
- level?: Level
50
- /** absent is none, which an empty body is not */
51
- body?: string | Uint8Array
52
- /** sixteen bytes, or their 32 hex digits */
53
- traceId?: Uint8Array | string | undefined
54
- /** eight bytes, or their 16 hex digits */
55
- spanId?: Uint8Array | string | undefined
56
- /** who produced it */
57
- context?: Fields
58
- /** what happened */
59
- attrs?: Fields
66
+ level?: EventLevel
67
+ message?: string
68
+ /** each value as JSON.parse reads it, an integer past 2^53 a bigint */
69
+ fields: Record<string, unknown>
70
+ context: Record<string, unknown>
71
+ /** 32 hex digits */
72
+ traceId?: string
73
+ /** 16 hex digits */
74
+ spanId?: string
60
75
  }
61
76
 
62
- /** A record as it was kept: a field's value is the JSON it was written as, spelled as it was given. */
63
- export interface LogRecord {
64
- /** unix nanoseconds */
65
- at: bigint
66
- stream: string
77
+ /** An event to append: its name, what happened, and what it carries beside. */
78
+ export interface NewEvent {
67
79
  name: string
68
- level: number | undefined
69
- body: string | Uint8Array | undefined
70
- traceId: Uint8Array | undefined
71
- spanId: Uint8Array | undefined
72
- context: [key: string, json: string | Uint8Array][]
73
- attrs: [key: string, json: string | Uint8Array][]
80
+ fields?: Fields
81
+ /** its time: now unless given, from what the stream keeps to ten minutes ahead */
82
+ at?: Date
83
+ context?: Fields
84
+ level?: EventLevel | number
85
+ message?: string
86
+ /** a trace's 32 hex digits, with its span's 16: the current trace's unless given */
87
+ traceId?: string
88
+ spanId?: string
74
89
  }
75
90
 
76
- /**
77
- * A record's fields, its attrs or its context, as an object, each value read
78
- * as JSON.parse reads the JSON it was kept as; the last of a repeated key is
79
- * kept. The pairs keep what JSON.parse may not: 1.2300 as written, an integer
80
- * past 2^53 whole.
81
- *
82
- * ```ts
83
- * for await (const record of store.records.all({ since: '1h', streams: ['readers'] })) {
84
- * const { page, from } = fields(record.attrs)
85
- * }
86
- * ```
87
- */
88
- export function fields(pairs: LogRecord['attrs']): Record<string, unknown> {
89
- // fromEntries, since an assignment to __proto__ would set the prototype
90
- return Object.fromEntries(pairs.map(([key, json]) => [key, JSON.parse(textOf(json))]))
91
+ export type AppendOptions = Omit<NewEvent, 'name' | 'fields'>
92
+
93
+ export interface EventsOptions {
94
+ /** how long the stream keeps its events: its own, kept in the file; 14 days until one says */
95
+ keep?: Duration
91
96
  }
92
97
 
93
- /**
94
- * The records in a range that meet every condition given.
95
- *
96
- * ```ts
97
- * { since: '1h', minLevel: 'warn' } // the last hour's warnings
98
- * { since: '1h', search: 'connection reset' } // whose text holds it, case ignored
99
- * { from: start, to: end, streams: ['api'] }
100
- * ```
101
- */
102
- export interface RecordsQuery {
103
- /** the span before now the range covers: '1h', '15m', or milliseconds; or from and to */
98
+ /** What a read finds: every condition given holds; one given as undefined is InvalidError. */
99
+ export interface Query {
100
+ /** the span before now, '1h'; or from and to */
104
101
  since?: Duration
105
- /** the range's start, included; open when absent */
106
- from?: Date | bigint
107
- /** the range's end, excluded; open when absent */
108
- to?: Date | bigint
109
- /** none or empty is every stream */
110
- streams?: string[]
111
- names?: string[]
112
- /** a record without a level does not match */
113
- minLevel?: Level
114
- traceId?: Uint8Array | string
115
- /** each a record must hold, by key and exact JSON spelling */
116
- attrs?: Fields
117
- context?: Fields
118
- /**
119
- * text a record's body or name holds, its case ignored: every record the
120
- * rest of the query leaves is read for it, so a search over much text ends
121
- * pages early at the budget, and next goes on
122
- */
102
+ /** the range's start, included */
103
+ from?: Date
104
+ /** the range's end, excluded */
105
+ to?: Date
106
+ /** any of these streams, for every stream's read */
107
+ streams?: readonly string[]
108
+ /** any of these names: ['log'] is the lines */
109
+ names?: readonly string[]
110
+ /** this level and worse; an event without a level is not found */
111
+ minLevel?: EventLevel
112
+ traceId?: string
113
+ /** each key with this value, by the JSON it was written as, in fields or in context */
114
+ where?: Record<string, unknown>
115
+ /** this text in the message, the name or a value, its case ignored */
123
116
  search?: string
124
- /** newest first; oldest first when absent */
117
+ /** newest first; oldest first without it */
125
118
  newest?: boolean
126
- /** the records a page holds: 1000, and 10000 at most */
119
+ /** the events of a page: 1000, and 10,000 at most */
127
120
  limit?: number
128
- /** what one read may open, fetch and decode; each narrows the server's */
129
- budget?: { blocks?: number; bytes?: number; records?: number }
130
- /** a page's next, which this page continues from; the rest of the query as it was */
131
- after?: string | undefined
121
+ /** a page's next, with the rest of the query as it was */
122
+ after?: string
132
123
  }
133
124
 
134
- /** Where a follower stands in the sealed segments: kept by the caller between follows. */
135
- export interface Cursor {
136
- segment: number
137
- row: number
138
- }
125
+ /** A query of every event it finds, for what counts them. */
126
+ export type Counted = Omit<Query, 'newest' | 'limit' | 'after'>
139
127
 
140
- /** A row that no longer reads, which a drop removes. */
141
- export interface Damage {
142
- stream: string
143
- /** a sealed segment, dropped whole; or a head row */
144
- segment: number | undefined
145
- headRow: number | undefined
146
- /** the times it held, unix nanoseconds */
147
- from: bigint
148
- to: bigint
149
- /** the invariant its bytes broke */
150
- reason: string
128
+ export interface Page {
129
+ events: Event[]
130
+ /** the page after it; the last page has none */
131
+ next?: string
151
132
  }
152
133
 
153
- const nsPerMs = 1_000_000n
134
+ export interface Top {
135
+ value: unknown
136
+ count: number
137
+ }
154
138
 
155
- function nanos(t: Date | bigint): bigint {
156
- return typeof t === 'bigint' ? t : BigInt(t.getTime()) * nsPerMs
139
+ /** A field's numbers in a row: a sum of integers exact, a bigint past 2^53. */
140
+ export interface Stats {
141
+ samples: number
142
+ sum: number | bigint
143
+ min: number | bigint
144
+ max: number | bigint
145
+ avg: number
157
146
  }
158
147
 
159
- function idOf(
160
- id: Uint8Array | string | undefined,
161
- bytes: number,
162
- what: string,
163
- ): Uint8Array | undefined {
164
- if (id === undefined) {
165
- return undefined
166
- }
167
- const raw = typeof id === 'string' ? new Uint8Array(Buffer.from(id, 'hex')) : id
168
- if (raw.length !== bytes) {
169
- throw new InvalidError(`a ${what} id of ${raw.length} bytes, not ${bytes}`)
170
- }
171
- return raw
148
+ export interface Row {
149
+ /** where the row's span begins, counted from 1970 in UTC */
150
+ at: Date
151
+ count: number
152
+ by: Record<string, unknown>
153
+ stats: Record<string, Stats>
154
+ distinct: Record<string, number>
172
155
  }
173
156
 
174
- /** Fields as the wire carries them: one array of keys and JSON texts. */
175
- function fieldsOf(fields: Fields | undefined): string[] | undefined {
176
- if (fields === undefined) {
177
- return undefined
178
- }
179
- const pairs = Array.isArray(fields) ? fields : Object.entries(fields)
180
- const flat: string[] = []
181
- for (const [key, value] of pairs) {
182
- const json = JSON.stringify(value)
183
- if (json === undefined) {
184
- throw new InvalidError(`the field ${key} holds a value JSON cannot write`)
185
- }
186
- flat.push(key, json)
187
- }
188
- return flat.length === 0 ? undefined : flat
157
+ export interface Aggregate extends Counted {
158
+ /** the span of a row: the range is one without it */
159
+ every?: Duration
160
+ /** keys whose every value has rows of its own */
161
+ by?: readonly string[]
162
+ /** fields whose numbers a row sums */
163
+ stats?: readonly string[]
164
+ /** fields whose different values a row counts */
165
+ distinct?: readonly string[]
189
166
  }
190
167
 
191
- const lossy = new TextDecoder()
168
+ export interface StreamInfo {
169
+ name: string
170
+ keep: DurationText
171
+ /** its oldest event, or where its keep begins while it holds older */
172
+ retainedFrom: Date
173
+ newest: Date
174
+ }
192
175
 
193
- function textOf(text: string | Uint8Array | undefined): string {
194
- return text === undefined ? '' : typeof text === 'string' ? text : lossy.decode(text)
176
+ export interface PollBatch {
177
+ events: Event[]
178
+ /** where the next poll begins, kept once these events are safe */
179
+ cursor: string
180
+ /** the events keep removed before the cursor reached them */
181
+ expired: number
195
182
  }
196
183
 
197
- function pairs(flat: (string | Uint8Array)[] | undefined): [string, string | Uint8Array][] {
198
- const out: [string, string | Uint8Array][] = []
199
- for (let i = 0; flat !== undefined && i + 1 < flat.length; i += 2) {
200
- out.push([textOf(flat[i]), flat[i + 1]!])
201
- }
202
- return out
184
+ /** What a tail finds: a query's conditions, and no range, order or limit. */
185
+ export type TailQuery = Pick<
186
+ Query,
187
+ 'streams' | 'names' | 'minLevel' | 'traceId' | 'where' | 'search'
188
+ >
189
+
190
+ /** A tail's batch: the events written since the last, and how many it missed. */
191
+ export interface TailBatch {
192
+ events: Event[]
193
+ /** the events that left the store's hold before the tail read them */
194
+ missed: number
203
195
  }
204
196
 
205
- function recordOf(r: ReturnType<typeof RecordsRecord.decode>): LogRecord {
206
- return {
207
- at: r.at ?? 0n,
208
- stream: textOf(r.stream),
209
- name: textOf(r.name),
210
- level: r.level,
211
- body: r.body,
212
- traceId: r.traceId,
213
- spanId: r.spanId,
214
- context: pairs(r.context),
215
- attrs: pairs(r.attrs),
216
- }
197
+ /** A head row or a segment that no longer reads, as `damaged` lists it and `drop` takes it. */
198
+ export interface Damage {
199
+ /** a head row holds events not packed yet, a segment packed ones */
200
+ part: 'head row' | 'segment'
201
+ id: number
202
+ stream: string
203
+ /** its events' oldest and newest times */
204
+ first: Date
205
+ last: Date
206
+ /** what broke */
207
+ error: string
217
208
  }
218
209
 
219
- /**
220
- * A page's next: where the range moved, both ends as nanoseconds, so that a
221
- * query over the last `since` continues the range it started with.
222
- *
223
- * from 1700000000000000000, open end → '1700000000000000000:'
224
- */
225
- function cursorOf(from: bigint | undefined, to: bigint | undefined): string {
226
- return `${from ?? ''}:${to ?? ''}`
210
+ type Method = keyof typeof methods
211
+
212
+ const newline = 0x0a
213
+ /** What a writer of lines holds for the server at most: past it, lines are dropped and counted. */
214
+ const linesHeld = 1 << 20
215
+ /** The bytes one event holds: a longer line is dropped whole, as the store drops one. */
216
+ const eventRoom = 256 << 10
217
+ const encoder = new TextEncoder()
218
+
219
+ const levelNumbers: Record<EventLevel, number> = {
220
+ trace: -8,
221
+ debug: -4,
222
+ info: 0,
223
+ warn: 4,
224
+ error: 8,
225
+ fatal: 12,
227
226
  }
227
+ const levelNames: readonly EventLevel[] = ['trace', 'debug', 'info', 'warn', 'error', 'fatal']
228
+
229
+ /** What every read takes: a stream's handle, or every stream without one. */
230
+ abstract class Reads {
231
+ protected abstract ask(
232
+ method: Method,
233
+ body: (handle: number | undefined) => Uint8Array,
234
+ ): Promise<Uint8Array>
235
+
236
+ protected abstract opened(
237
+ method: Method,
238
+ body: (handle: number | undefined) => Uint8Array,
239
+ ): Promise<Stream>
240
+
241
+ /** One page, from one moment of the file: `{ events, next }`. */
242
+ async list(query?: Query): Promise<Page> {
243
+ const asked = queryOf(query)
244
+ const answer = await this.ask('records.list', handle =>
245
+ RecordsQuery.encode({ ...asked, handle }),
246
+ )
247
+ const page = RecordsPage.decode(answer)
248
+ const events = (page.events ?? []).map(eventOf)
249
+ return page.next === undefined ? { events } : { events, next: page.next }
250
+ }
228
251
 
229
- function rangeOf(q: RecordsQuery | undefined): {
230
- from: bigint | undefined
231
- to: bigint | undefined
232
- } {
233
- if (q?.after !== undefined) {
234
- const ends = /^(-?\d*):(-?\d*)$/.exec(q.after)
235
- if (ends === null) {
236
- throw new InvalidError(`${JSON.stringify(q.after)} is no page's next`)
237
- }
238
- return {
239
- from: ends[1] === '' ? undefined : BigInt(ends[1]!),
240
- to: ends[2] === '' ? undefined : BigInt(ends[2]!),
252
+ /** Every event the query finds, a page after another. */
253
+ async *all(query?: Query): AsyncGenerator<Event> {
254
+ let after = query?.after
255
+ for (;;) {
256
+ const page = await this.list(after === undefined ? query : { ...query, after })
257
+ yield* page.events
258
+ if (page.next === undefined) {
259
+ return
260
+ }
261
+ after = page.next
241
262
  }
242
263
  }
243
- if (q?.since !== undefined && q.from !== undefined) {
244
- throw new InvalidError('a range starts since a span before now or from a time, not both')
264
+
265
+ /** How many events the query finds. */
266
+ async count(query?: Counted): Promise<number> {
267
+ const asked = queryOf(query)
268
+ const answer = await this.ask('records.count', handle =>
269
+ RecordsQuery.encode({ ...asked, handle }),
270
+ )
271
+ return RecordsCount.decode(answer).count ?? 0
272
+ }
273
+
274
+ /**
275
+ * A key's most frequent values and how many events hold each: ten unless
276
+ * `limit` asks for more, at most 1000. The key is a field's or a
277
+ * context's, or `stream`, `name` or `level`.
278
+ */
279
+ async top(key: string, query?: Counted & { limit?: number }): Promise<Top[]> {
280
+ const asked = queryOf(query)
281
+ const answer = await this.ask('records.top', handle =>
282
+ RecordsTopAsked.encode({ query: { ...asked, handle }, key }),
283
+ )
284
+ return (RecordsTops.decode(answer).values ?? []).map(top => ({
285
+ value: parseJson(top.value ?? 'null'),
286
+ count: top.count ?? 0,
287
+ }))
288
+ }
289
+
290
+ /** A row for each span of `every` and each value of the keys of `by`, with its count. */
291
+ async aggregate(query: Aggregate = {}): Promise<Row[]> {
292
+ const { every, by, stats, distinct, ...counted } = query
293
+ const asked = queryOf(counted)
294
+ const answer = await this.ask('records.aggregate', handle =>
295
+ RecordsAggregateAsked.encode({
296
+ query: { ...asked, handle },
297
+ every: every === undefined ? undefined : ms(every),
298
+ by: by === undefined ? undefined : [...by],
299
+ stats: stats === undefined ? undefined : [...stats],
300
+ distinct: distinct === undefined ? undefined : [...distinct],
301
+ }),
302
+ )
303
+ return (RecordsRows.decode(answer).rows ?? []).map(row => ({
304
+ at: dateOf(row.at ?? 0n),
305
+ count: row.count ?? 0,
306
+ by: mapValues(row.by, json => parseJson(json)),
307
+ stats: mapValues(row.stats, kept => ({
308
+ samples: kept.samples ?? 0,
309
+ sum: parseJson(kept.sum ?? '0') as number | bigint,
310
+ min: parseJson(kept.min ?? '0') as number | bigint,
311
+ max: parseJson(kept.max ?? '0') as number | bigint,
312
+ avg: kept.avg ?? 0,
313
+ })),
314
+ distinct: mapValues(row.distinct, count => count),
315
+ }))
316
+ }
317
+
318
+ /** The keys the events hold, in fields and in context, sorted; `prefix` narrows them. */
319
+ async fieldNames(
320
+ query?: Omit<Query, 'newest' | 'after'> & { prefix?: string },
321
+ ): Promise<string[]> {
322
+ const { prefix, ...rest } = query ?? {}
323
+ const asked = queryOf(rest)
324
+ const answer = await this.ask('records.fieldNames', handle =>
325
+ RecordsFieldNamesAsked.encode({ query: { ...asked, handle }, prefix }),
326
+ )
327
+ return RecordsNames.decode(answer).names ?? []
328
+ }
329
+
330
+ /**
331
+ * The events written from now on that the query finds, a batch as they
332
+ * are written: `{ events, missed }`. A reader slower than the writers
333
+ * skips what left the store's hold, and the batch after says how many.
334
+ * It ends when its loop is left, and when the server's store closes.
335
+ */
336
+ async *tail(query?: TailQuery): AsyncGenerator<TailBatch> {
337
+ const asked = queryOf(query)
338
+ const stream = await this.opened('records.tail', handle =>
339
+ RecordsQuery.encode({ ...asked, handle }),
340
+ )
341
+ try {
342
+ for (;;) {
343
+ const event = await stream.next()
344
+ stream.consumed(event.body.length)
345
+ if (event.end) {
346
+ return
347
+ }
348
+ const batch = RecordsTailBatch.decode(event.body)
349
+ yield { events: (batch.events ?? []).map(eventOf), missed: batch.missed ?? 0 }
350
+ }
351
+ } finally {
352
+ stream.cancel()
353
+ }
245
354
  }
246
- const from =
247
- q?.since !== undefined
248
- ? nanos(new Date(Date.now() - ms(q.since)))
249
- : q?.from === undefined
250
- ? undefined
251
- : nanos(q.from)
252
- return { from, to: q?.to === undefined ? undefined : nanos(q.to) }
253
355
  }
254
356
 
255
- function queryOf(q: RecordsQuery | undefined): Parameters<typeof QueryMessage.encode>[0] {
256
- return {
257
- ...rangeOf(q),
258
- streams: q?.streams !== undefined && q.streams.length > 0 ? q.streams : undefined,
259
- names: q?.names !== undefined && q.names.length > 0 ? q.names : undefined,
260
- minLevel: q?.minLevel === undefined ? undefined : levelOf(q.minLevel),
261
- traceId: idOf(q?.traceId, 16, 'trace'),
262
- attrs: fieldsOf(q?.attrs),
263
- context: fieldsOf(q?.context),
264
- newest: q?.newest === true ? true : undefined,
265
- limit: q?.limit,
266
- budgetBlocks: q?.budget?.blocks,
267
- budgetBytes: q?.budget?.bytes,
268
- budgetRecords: q?.budget?.records,
269
- search: q?.search === '' ? undefined : q?.search,
270
- }
271
- }
272
-
273
- export class Records {
357
+ /** Opens a stream of events: lazily, at its first call on a connection. */
358
+ export function openEvents(link: Link, stream: string, options?: EventsOptions): Events {
359
+ checkName(stream, 'stream')
360
+ const keep = options?.keep === undefined ? undefined : ms(options.keep)
361
+ return new Events(link, stream, RecordsOpen.encode({ stream, keep }))
362
+ }
363
+
364
+ /** A stream of events: written with `append`, which waits for the file, and read back. */
365
+ export class Events extends Reads {
366
+ readonly stream: string
274
367
  readonly #link: Link
275
- readonly #loggers: Logger[] = []
368
+ readonly #via: Via
369
+ readonly #open: Uint8Array
276
370
 
277
- constructor(link: Link) {
371
+ /** Events come from `store.events(stream)`. */
372
+ constructor(link: Link, stream: string, open: Uint8Array) {
373
+ super()
374
+ this.stream = stream
278
375
  this.#link = link
376
+ this.#via = viaLink(link)
377
+ this.#open = open
279
378
  }
280
379
 
281
380
  /**
282
- * A logger of a stream: its calls return at once, and its lines reach the
283
- * server every second, as Go's slog handler's and Python's logging.Handler's do.
381
+ * Writes an event, or a list of them whole or not at all: it returns once
382
+ * they are written, and a read finds them at once. A refused event names
383
+ * its place in the list and its name.
284
384
  */
285
- logger(stream: string, options?: LoggerOptions): Logger {
286
- const logger = newLogger(lines => this.#writeLines(lines), stream, options)
287
- this.#loggers.push(logger)
288
- return logger
385
+ append(name: string, fields?: Fields, options?: AppendOptions): Promise<void>
386
+ append(events: readonly NewEvent[]): Promise<void>
387
+ async append(
388
+ first: string | readonly NewEvent[],
389
+ fields?: Fields,
390
+ options?: AppendOptions,
391
+ ): Promise<void> {
392
+ const events = typeof first === 'string' ? [oneEvent(first, fields, options)] : first
393
+ const encoded = events.map(newEventOf)
394
+ await this.#call(
395
+ 'records.append',
396
+ handle => RecordsAppend.encode({ handle, events: encoded }),
397
+ 'write',
398
+ )
289
399
  }
290
400
 
291
- /** Hands over what every logger holds, as the store does when it closes. */
292
- async stop(): Promise<void> {
293
- await Promise.all(this.#loggers.map(logger => logger.stop()))
401
+ /** A logger's lines, each an event of this stream. */
402
+ appendLines(lines: readonly ConsoleLine[]): Promise<void> {
403
+ const events = lines.map(line => ({
404
+ name: line.event ?? 'log',
405
+ at: nanosOf(line.at),
406
+ level: line.level,
407
+ message: line.msg,
408
+ fields: objectOf(line.attrs),
409
+ context: objectOf(line.context),
410
+ ...wireTrace(line.traceId, line.spanId),
411
+ }))
412
+ return this.#call(
413
+ 'records.append',
414
+ handle => RecordsAppend.encode({ handle, events }),
415
+ 'write',
416
+ ).then(() => {})
294
417
  }
295
418
 
296
419
  /**
297
- * Appends records in one transaction, all or none; a refused one names
298
- * itself as `call`, with its stream and name. A read sees them at once. A
299
- * record of no trace of its own takes the one withTrace runs the call in.
420
+ * A writer of another program's output into this stream: each line it is
421
+ * given becomes an event at the time it arrived, a stack trace's or a JSON
422
+ * value's lines one, and a write never waits.
423
+ *
424
+ * const out = store.events('worker').lines()
425
+ * child.stdout.on('data', chunk => out.write(chunk))
426
+ * child.on('close', () => out.end())
300
427
  */
301
- async append(records: RecordInput | readonly RecordInput[]): Promise<void> {
302
- const carried = currentTrace()
303
- const list = Array.isArray(records) ? records : [records as RecordInput]
304
- return this.#write(
305
- carried === undefined
306
- ? list
307
- : list.map(r =>
308
- r.traceId === undefined
309
- ? { ...r, traceId: carried.traceId, spanId: carried.spanId }
310
- : r,
311
- ),
312
- )
428
+ lines(): Lines {
429
+ return new Lines(async () => {
430
+ const connection = await this.#link.connection()
431
+ const handle = await handleOn(connection, methods['records.open'], this.#open)
432
+ const body = Handle.encode({ handle })
433
+ const stream = await streamOf(connection, methods['records.lines'], body, false)
434
+ return { stream, piece: pieceOf(connection) }
435
+ })
313
436
  }
314
437
 
315
- async #write(records: readonly RecordInput[]): Promise<void> {
316
- const now = BigInt(Date.now()) * nsPerMs
317
- await this.#send(
318
- records.map(r => ({
319
- at: r.at === undefined ? now : nanos(r.at),
320
- stream: r.stream,
321
- name: r.name,
322
- level: r.level === undefined ? undefined : levelOf(r.level),
323
- body: r.body,
324
- traceId: idOf(r.traceId, 16, 'trace'),
325
- spanId: idOf(r.spanId, 8, 'span'),
326
- context: fieldsOf(r.context),
327
- attrs: fieldsOf(r.attrs),
328
- })),
329
- )
438
+ protected ask(
439
+ method: Method,
440
+ body: (handle: number | undefined) => Uint8Array,
441
+ ): Promise<Uint8Array> {
442
+ return this.#call(method, body, 'read')
330
443
  }
331
444
 
332
- /** Appends a logger's lines, their fields already JSON and their traces taken as each was logged. */
333
- async #writeLines(lines: readonly ConsoleLine[]): Promise<void> {
334
- await this.#send(
335
- lines.map(line => ({
336
- at: nanos(line.at),
337
- stream: line.stream,
338
- name: line.event ?? 'log',
339
- level: line.level,
340
- body: line.msg,
341
- traceId: idOf(line.traceId, 16, 'trace'),
342
- spanId: idOf(line.spanId, 8, 'span'),
343
- context: line.context.length === 0 ? undefined : [...line.context],
344
- attrs: line.attrs.length === 0 ? undefined : [...line.attrs],
345
- })),
346
- )
445
+ protected opened(
446
+ method: Method,
447
+ body: (handle: number | undefined) => Uint8Array,
448
+ ): Promise<Stream> {
449
+ return this.#link.run('read', async connection => {
450
+ const handle = await handleOn(connection, methods['records.open'], this.#open)
451
+ return streamOf(connection, methods[method], body(handle), true)
452
+ })
347
453
  }
348
454
 
349
- async #send(records: Parameters<typeof RecordsBatch.encode>[0]['records']): Promise<void> {
350
- await this.#link.run('write', connection =>
351
- connection.session.call(methods['records.append'], RecordsBatch.encode({ records })),
352
- )
455
+ #call(
456
+ method: Method,
457
+ body: (handle: number) => Uint8Array,
458
+ idempotence: 'read' | 'write',
459
+ ): Promise<Uint8Array> {
460
+ return this.#via.call(methods['records.open'], this.#open, methods[method], body, idempotence)
353
461
  }
462
+ }
354
463
 
355
- /**
356
- * One page of the records a query matches, from one snapshot, in event-time
357
- * order; next, while a limit or a budget ended the page early, is passed back
358
- * as `after` with the same query. A page never splits a timestamp.
359
- */
360
- async scan(query?: RecordsQuery): Promise<Page<LogRecord, string>> {
361
- const asked = queryOf(query)
362
- const got = await this.#link.run('read', connection =>
363
- download(connection, methods['records.read'], QueryMessage.encode(asked)),
364
- )
365
- const page = RecordsPage.decode(got.trailer)
366
- const items = got.items.map(item => recordOf(RecordsRecord.decode(item)))
367
- if (page.more !== true) {
368
- return { items, next: undefined }
369
- }
370
- const moved = (end: number | bigint | undefined, sent: number | bigint | undefined) =>
371
- end !== undefined ? BigInt(end) : sent !== undefined ? BigInt(sent) : undefined
372
- return { items, next: cursorOf(moved(page.from, asked.from), moved(page.to, asked.to)) }
373
- }
374
-
375
- /** Every record a query matches, a page at a time. */
376
- async *all(query?: Omit<RecordsQuery, 'after'>): AsyncGenerator<LogRecord> {
377
- let page = await this.scan(query)
378
- yield* page.items
379
- while (page.next !== undefined) {
380
- page = await this.scan({ ...query, after: page.next })
381
- yield* page.items
382
- }
464
+ /** Every stream's events, to read: `store.events()`. */
465
+ export class AllEvents extends Reads {
466
+ readonly #link: Link
467
+
468
+ /** AllEvents come from `store.events()`. */
469
+ constructor(link: Link) {
470
+ super()
471
+ this.#link = link
472
+ }
473
+
474
+ /** Every stream with events, by name, with what it keeps, where that begins and its newest event's time. */
475
+ async streams(): Promise<StreamInfo[]> {
476
+ const answer = await this.ask('records.streams', () => Empty.encode({}))
477
+ return (RecordsStreams.decode(answer).streams ?? []).map(stream => ({
478
+ name: stream.name ?? '',
479
+ keep: spelled(stream.keep ?? 0),
480
+ retainedFrom: dateOf(stream.retainedFrom ?? 0n),
481
+ newest: dateOf(stream.newest ?? 0n),
482
+ }))
383
483
  }
384
484
 
385
485
  /**
386
- * The sealed records after a cursor, in the order they were sealed; the
387
- * cursor to follow from next, and the segments retention removed before
388
- * the cursor reached them. A record reaches a follower once it is sealed.
486
+ * Every event once, a batch a call, from a cursor the program keeps, the
487
+ * oldest event kept without one: `{ events, cursor, expired }`. With no
488
+ * event past the cursor, the batch is empty and its cursor the one given.
389
489
  */
390
- async follow(
391
- cursor?: Cursor,
392
- limit?: number,
393
- ): Promise<{ records: LogRecord[]; cursor: Cursor; expired: number }> {
394
- const got = await this.#link.run('read', connection =>
395
- download(
396
- connection,
397
- methods['records.follow'],
398
- RecordsCursor.encode({ segment: cursor?.segment, row: cursor?.row, limit }),
399
- ),
490
+ async poll(cursor?: string, options?: { limit?: number }): Promise<PollBatch> {
491
+ const answer = await this.ask('records.poll', () =>
492
+ RecordsPollAsked.encode({ cursor, limit: options?.limit }),
400
493
  )
401
- const trailer = RecordsCursor.decode(got.trailer)
494
+ const batch = RecordsBatch.decode(answer)
402
495
  return {
403
- records: got.items.map(item => recordOf(RecordsRecord.decode(item))),
404
- cursor: { segment: trailer.segment ?? 0, row: trailer.row ?? 0 },
405
- expired: trailer.expired ?? 0,
496
+ events: (batch.events ?? []).map(eventOf),
497
+ cursor: batch.cursor ?? '',
498
+ expired: batch.expired ?? 0,
406
499
  }
407
500
  }
408
501
 
409
- /**
410
- * A writer of another program's output, pino's lines or a child's stdout,
411
- * that never waits: each line becomes a record of the stream as the server's
412
- * Lines makes it, a stack trace's lines joined and a JSON or logfmt line's
413
- * fields kept. What does not fit its buffer is dropped and counted.
414
- */
415
- lines(stream: string, options?: { buffer?: number }): LinesWriter {
416
- if (stream === '') {
417
- throw new InvalidError('lines of no stream')
418
- }
419
- return new LinesWriter(this.#link, stream, options?.buffer ?? 1 << 20)
502
+ protected ask(
503
+ method: Method,
504
+ body: (handle: number | undefined) => Uint8Array,
505
+ ): Promise<Uint8Array> {
506
+ return this.#link.run('read', connection =>
507
+ connection.session.call(methods[method], body(undefined)),
508
+ )
420
509
  }
421
510
 
422
- /** The rows the server's records have met that no longer read. */
511
+ /**
512
+ * What reads met that no longer reads, which `drop` removes: each head
513
+ * row and segment with its stream and its events' times. What the store
514
+ * has not met since it opened is not listed.
515
+ */
423
516
  async damaged(): Promise<Damage[]> {
424
- const body = await this.#link.run('read', connection =>
425
- connection.session.call(methods['records.damaged'], Empty.encode({})),
426
- )
427
- return (RecordsDamages.decode(body).damages ?? []).map(d => ({
428
- stream: textOf(d.stream),
429
- segment: d.segment,
430
- headRow: d.headRow,
431
- from: d.from ?? 0n,
432
- to: d.to ?? 0n,
433
- reason: d.reason ?? '',
517
+ const answer = await this.ask('records.damaged', () => Empty.encode({}))
518
+ return (RecordsDamages.decode(answer).damages ?? []).map(damage => ({
519
+ part: damage.part === 'segment' ? 'segment' : 'head row',
520
+ id: damage.id ?? 0,
521
+ stream: damage.stream ?? '',
522
+ first: dateOf(damage.first ?? 0n),
523
+ last: dateOf(damage.last ?? 0n),
524
+ error: damage.error ?? '',
434
525
  }))
435
526
  }
436
527
 
437
- /** Removes a damaged row, a repair an admin connection alone may make. */
528
+ /**
529
+ * Removes a head row or a segment that `damaged` listed, so that the
530
+ * reads over it work again; what still reads is refused.
531
+ *
532
+ * for (const damage of await store.events().damaged()) await store.events().drop(damage)
533
+ */
438
534
  async drop(damage: Damage): Promise<void> {
535
+ const body = RecordsDamage.encode({
536
+ ...damage,
537
+ first: nanosOf(damage.first),
538
+ last: nanosOf(damage.last),
539
+ })
439
540
  await this.#link.run('write', connection =>
440
- connection.session.call(methods['records.drop'], RecordsDamage.encode(damage)),
541
+ connection.session.call(methods['records.drop'], body),
542
+ )
543
+ }
544
+
545
+ protected opened(
546
+ method: Method,
547
+ body: (handle: number | undefined) => Uint8Array,
548
+ ): Promise<Stream> {
549
+ return this.#link.run('read', connection =>
550
+ streamOf(connection, methods[method], body(undefined), true),
441
551
  )
442
552
  }
443
553
  }
444
554
 
445
- /** The most one DATA of lines carries. */
446
- const linesChunk = 64 << 10
555
+ /** A writer's stream, and the bytes a piece of it carries. */
556
+ interface Opened {
557
+ stream: Stream
558
+ piece: number
559
+ }
447
560
 
448
561
  /**
449
- * An upload of lines that stays open while it is written: write never waits,
450
- * holding what the stream's credit does not let go yet up to its buffer and
451
- * dropping past it. A connection lost drops what it held and the next write
452
- * uploads on the next connection.
562
+ * A writer of another program's output into a stream, as `events.lines()`
563
+ * begins it: each line becomes an event, and a write never waits. It sends
564
+ * whole lines, so that what it has no room for, `dropped` counts as lines.
453
565
  */
454
- export class LinesWriter {
455
- readonly #link: Link
456
- readonly #stream: string
457
- readonly #most: number
458
- #waiting: Uint8Array[] = []
459
- #held = 0
460
- #pumping: Promise<void> | undefined
461
- #upload: Stream | undefined
566
+ export class Lines {
567
+ /** whole lines not sent yet */
568
+ readonly #queued: Uint8Array[] = []
569
+ #queuedBytes = 0
570
+ /** a line not ended yet */
571
+ #partial = new Uint8Array(0)
572
+ /** the line being written is longer than an event holds: the rest of it goes too */
573
+ #discarding = false
574
+ #dropped = 0
462
575
  #ended = false
463
- /** bytes dropped since the writer began: a full buffer, or a lost connection */
464
- dropped = 0
576
+ #failure: Error | undefined
577
+ #wake: (() => void) | undefined
578
+ readonly #sent: Promise<void>
579
+
580
+ /** Lines come from `events.lines()`. */
581
+ constructor(open: () => Promise<Opened>) {
582
+ this.#sent = this.#send(open).catch((err: unknown) => {
583
+ this.#failure = err instanceof Error ? err : new Error(String(err))
584
+ for (const bytes of this.#queued.splice(0)) {
585
+ this.#dropped += linesIn(bytes)
586
+ }
587
+ this.#queuedBytes = 0
588
+ })
589
+ }
465
590
 
466
- constructor(link: Link, stream: string, most: number) {
467
- this.#link = link
468
- this.#stream = stream
469
- this.#most = most
591
+ /** The lines dropped: past what the writer holds, and, once it ended, by the store. */
592
+ get dropped(): number {
593
+ return this.#dropped
470
594
  }
471
595
 
472
- /** Takes a piece of output, cut anywhere. It returns at once. */
473
- write(chunk: string | Uint8Array): boolean {
596
+ /** Takes a piece of output, cut anywhere; it never waits. */
597
+ write(chunk: Uint8Array | string): void {
474
598
  if (this.#ended) {
475
- throw new InvalidError('a write after the lines ended')
599
+ throw new ClosedError('lines written after their end')
600
+ }
601
+ let bytes = typeof chunk === 'string' ? encoder.encode(chunk) : chunk
602
+ if (this.#discarding) {
603
+ const end = bytes.indexOf(newline)
604
+ if (end < 0) {
605
+ return
606
+ }
607
+ this.#discarding = false
608
+ bytes = bytes.subarray(end + 1)
476
609
  }
477
- const bytes = typeof chunk === 'string' ? new TextEncoder().encode(chunk) : chunk
610
+ const joined = concat(this.#partial, bytes)
611
+ const cut = joined.lastIndexOf(newline) + 1
612
+ this.#partial = joined.slice(cut)
613
+ if (this.#partial.length > eventRoom) {
614
+ this.#partial = new Uint8Array(0)
615
+ this.#discarding = true
616
+ this.#dropped++
617
+ }
618
+ this.#queue(joined.subarray(0, cut))
619
+ }
620
+
621
+ /**
622
+ * Ends the writer: what it holds goes, a last line without its newline
623
+ * too, and it returns once the store wrote them.
624
+ */
625
+ async end(): Promise<void> {
626
+ if (!this.#ended) {
627
+ this.#ended = true
628
+ this.#queue(this.#partial)
629
+ this.#partial = new Uint8Array(0)
630
+ this.#wake?.()
631
+ }
632
+ await this.#sent
633
+ if (this.#failure !== undefined) {
634
+ throw this.#failure
635
+ }
636
+ }
637
+
638
+ /** Holds bytes for the server, or drops and counts their lines when it holds no room. */
639
+ #queue(bytes: Uint8Array): void {
478
640
  if (bytes.length === 0) {
479
- return true
641
+ return
480
642
  }
481
- if (this.#held + bytes.length > this.#most) {
482
- this.dropped += bytes.length
483
- return false
643
+ if (this.#failure !== undefined || this.#queuedBytes + bytes.length > linesHeld) {
644
+ this.#dropped += linesIn(bytes)
645
+ return
484
646
  }
485
- for (let at = 0; at < bytes.length; at += linesChunk) {
486
- this.#waiting.push(bytes.subarray(at, at + linesChunk))
647
+ this.#queued.push(bytes)
648
+ this.#queuedBytes += bytes.length
649
+ this.#wake?.()
650
+ }
651
+
652
+ /** Sends what is held as it comes, in pieces within the stream's credit, then the end. */
653
+ async #send(open: () => Promise<Opened>): Promise<void> {
654
+ const { stream, piece } = await open()
655
+ for (;;) {
656
+ const bytes = this.#queued.shift()
657
+ if (bytes === undefined) {
658
+ if (this.#ended) {
659
+ break
660
+ }
661
+ await new Promise<void>(resolve => {
662
+ this.#wake = resolve
663
+ })
664
+ this.#wake = undefined
665
+ continue
666
+ }
667
+ this.#queuedBytes -= bytes.length
668
+ for (let at = 0; at < bytes.length; at += piece) {
669
+ await stream.send(RecordsPiece.encode({ bytes: bytes.subarray(at, at + piece) }), false)
670
+ }
487
671
  }
488
- this.#held += bytes.length
489
- this.#pumping ??= this.#pump().finally(() => {
490
- this.#pumping = undefined
491
- })
492
- return true
493
- }
494
-
495
- async #pump(): Promise<void> {
496
- while (this.#waiting.length > 0) {
497
- const piece = this.#waiting[0]!
498
- try {
499
- const upload = await this.#open()
500
- await upload.send(piece, false)
501
- } catch {
502
- this.#upload = undefined
503
- this.dropped += this.#held
504
- this.#waiting = []
505
- this.#held = 0
672
+ await stream.send(new Uint8Array(0), true)
673
+ for (;;) {
674
+ const event = await stream.next()
675
+ if (event.end) {
676
+ this.#dropped += RecordsDropped.decode(event.body).dropped ?? 0
506
677
  return
507
678
  }
508
- this.#waiting.shift()
509
- this.#held -= piece.length
510
679
  }
511
680
  }
681
+ }
682
+
683
+ /** A stream of the method, open once its RESPONSE came; `end` leaves the client nothing to send. */
684
+ async function streamOf(
685
+ connection: Connection,
686
+ method: number,
687
+ body: Uint8Array,
688
+ end: boolean,
689
+ ): Promise<Stream> {
690
+ const stream = await connection.session.open(method, body, end)
691
+ await stream.next()
692
+ return stream
693
+ }
694
+
695
+ function concat(a: Uint8Array, b: Uint8Array): Uint8Array {
696
+ if (a.length === 0) {
697
+ return b
698
+ }
699
+ const joined = new Uint8Array(a.length + b.length)
700
+ joined.set(a)
701
+ joined.set(b, a.length)
702
+ return joined
703
+ }
704
+
705
+ /** The lines bytes hold: one a newline ends, and one more past the last. */
706
+ function linesIn(bytes: Uint8Array): number {
707
+ let lines = bytes[bytes.length - 1] === newline ? 0 : 1
708
+ for (const byte of bytes) {
709
+ if (byte === newline) {
710
+ lines++
711
+ }
712
+ }
713
+ return lines
714
+ }
715
+
716
+ /** What a store's logger is told: a logger's options, and how long its stream keeps. */
717
+ export interface StoreLoggerOptions extends LoggerOptions {
718
+ /** how long the stream keeps its events, as store.events takes it */
719
+ keep?: Duration
720
+ }
721
+
722
+ /**
723
+ * A logger of `stream` that keeps its lines there through an append of their
724
+ * own, and what it declares its config `log` with.
725
+ */
726
+ export function storeLogger(
727
+ link: Link,
728
+ stream: string,
729
+ options: StoreLoggerOptions = {},
730
+ ): [Logger, LogConfig] {
731
+ const { keep, ...told } = options
732
+ const events = openEvents(link, stream, keep === undefined ? undefined : { keep })
733
+ return newStoreLogger(lines => events.appendLines(lines), stream, told)
734
+ }
735
+
736
+ /** The event an append of a name, its fields and its options writes. */
737
+ function oneEvent(
738
+ name: string,
739
+ fields: Fields | undefined,
740
+ options: AppendOptions | undefined,
741
+ ): NewEvent {
742
+ return fields === undefined ? { ...options, name } : { ...options, name, fields }
743
+ }
744
+
745
+ /** An event as the wire carries it, the current trace's ids when it gives none. */
746
+ function newEventOf(event: NewEvent, place: number): Parameters<typeof RecordsNewEvent.encode>[0] {
747
+ const named = `event ${place} ${JSON.stringify(event.name)}`
748
+ if (typeof event.name !== 'string' || event.name === '') {
749
+ throw new InvalidError(`${named}: an event has a name`)
750
+ }
751
+ if (event.at !== undefined && Number.isNaN(event.at.getTime())) {
752
+ throw new InvalidError(`${named}: a time that is no date`)
753
+ }
754
+ const given =
755
+ event.traceId === undefined
756
+ ? traceOf(currentTrace())
757
+ : { traceId: event.traceId, spanId: event.spanId }
758
+ return {
759
+ name: event.name,
760
+ at: event.at === undefined ? undefined : nanosOf(event.at),
761
+ level: event.level === undefined ? undefined : levelNumber(event.level, named),
762
+ message: event.message,
763
+ fields:
764
+ event.fields === undefined ? undefined : objectOf(encodeFields(event.fields, undefined)),
765
+ context:
766
+ event.context === undefined ? undefined : objectOf(encodeFields(event.context, undefined)),
767
+ ...wireTrace(given.traceId, given.spanId),
768
+ }
769
+ }
770
+
771
+ /** A query as the wire carries it; a condition given as undefined reads nothing it was not asked. */
772
+ function queryOf(query: Query | undefined): Parameters<typeof RecordsQuery.encode>[0] {
773
+ const q = query ?? {}
774
+ for (const [key, value] of Object.entries(q)) {
775
+ if (value === undefined) {
776
+ throw new InvalidError(`a query's ${key} given as undefined: leave it out to read without it`)
777
+ }
778
+ }
779
+ const where = q.where === undefined ? undefined : Object.entries(q.where)
780
+ for (const [key, value] of where ?? []) {
781
+ if (value === undefined) {
782
+ throw new InvalidError(`a query's where ${JSON.stringify(key)} given as undefined`)
783
+ }
784
+ }
785
+ return {
786
+ since: q.since === undefined ? undefined : ms(q.since),
787
+ from: q.from === undefined ? undefined : nanosOf(q.from),
788
+ to: q.to === undefined ? undefined : nanosOf(q.to),
789
+ streams: q.streams === undefined ? undefined : [...q.streams],
790
+ names: q.names === undefined ? undefined : [...q.names],
791
+ minLevel: q.minLevel === undefined ? undefined : levelNumber(q.minLevel, 'minLevel'),
792
+ traceId: q.traceId === undefined ? undefined : idBytes(q.traceId, 16, 'traceId'),
793
+ where:
794
+ where === undefined
795
+ ? undefined
796
+ : Object.fromEntries(where.map(([key, value]) => [key, jsonOf(value)])),
797
+ search: q.search,
798
+ newest: q.newest,
799
+ limit: q.limit,
800
+ after: q.after,
801
+ }
802
+ }
803
+
804
+ function eventOf(read: ReturnType<typeof RecordsEvent.decode>): Event {
805
+ const event: Event = {
806
+ at: dateOf(read.at ?? 0n),
807
+ stream: read.stream ?? '',
808
+ name: read.name ?? '',
809
+ fields: read.fields === undefined ? {} : (parseJson(read.fields) as Record<string, unknown>),
810
+ context: read.context === undefined ? {} : (parseJson(read.context) as Record<string, unknown>),
811
+ }
812
+ if (read.level !== undefined) {
813
+ event.level = levelNames[Math.min(5, Math.max(0, Math.floor(read.level / 4) + 2))] as EventLevel
814
+ }
815
+ if (read.message !== undefined) {
816
+ event.message = read.message
817
+ }
818
+ if (read.traceId !== undefined) {
819
+ event.traceId = Buffer.from(read.traceId).toString('hex')
820
+ }
821
+ if (read.spanId !== undefined) {
822
+ event.spanId = Buffer.from(read.spanId).toString('hex')
823
+ }
824
+ return event
825
+ }
512
826
 
513
- async #open(): Promise<Stream> {
514
- if (this.#upload !== undefined && !this.#upload.finished) {
515
- return this.#upload
827
+ /** Key, JSON, key, JSON… as the JSON object they spell; none for no field. */
828
+ function objectOf(flat: readonly string[]): string | undefined {
829
+ if (flat.length === 0) {
830
+ return undefined
831
+ }
832
+ const pairs: string[] = []
833
+ for (let at = 0; at < flat.length; at += 2) {
834
+ pairs.push(`${JSON.stringify(flat[at])}:${flat[at + 1]}`)
835
+ }
836
+ return `{${pairs.join(',')}}`
837
+ }
838
+
839
+ function levelNumber(level: EventLevel | number, what: string): number {
840
+ if (typeof level === 'number') {
841
+ if (!Number.isInteger(level)) {
842
+ throw new InvalidError(`${what}: the level ${level}, where slog counts whole numbers`)
516
843
  }
517
- const connection: Connection = await this.#link.connection()
518
- this.#upload = await connection.session.open(
519
- methods['records.lines'],
520
- RecordsStream.encode({ stream: this.#stream }),
521
- false,
844
+ return level
845
+ }
846
+ const number = levelNumbers[level]
847
+ if (number === undefined) {
848
+ throw new InvalidError(
849
+ `${what}: the level ${JSON.stringify(level)}, not one of ${levelNames.join(', ')}`,
522
850
  )
523
- return this.#upload
524
851
  }
852
+ return number
853
+ }
525
854
 
526
- /** Hands over what the writer holds, a line cut short included, and ends the upload. */
527
- async end(): Promise<void> {
528
- this.#ended = true
529
- await this.#pumping
530
- const upload = this.#upload
531
- if (upload === undefined || upload.finished) {
532
- return
855
+ /** A trace's ids as hex, when the trace has both. */
856
+ function traceOf(trace: Trace | undefined): {
857
+ traceId?: string | undefined
858
+ spanId?: string | undefined
859
+ } {
860
+ if (trace === undefined) {
861
+ return {}
862
+ }
863
+ const hex = (id: Uint8Array | string | undefined) =>
864
+ typeof id === 'string' || id === undefined ? id : Buffer.from(id).toString('hex')
865
+ return { traceId: hex(trace.traceId), spanId: hex(trace.spanId) }
866
+ }
867
+
868
+ /** A trace and its span as the wire carries them: both, or neither, since the store keeps a span with its trace. */
869
+ function wireTrace(
870
+ traceId: string | undefined,
871
+ spanId: string | undefined,
872
+ ): { traceId?: Uint8Array; spanId?: Uint8Array } {
873
+ if (traceId === undefined || spanId === undefined) {
874
+ return {}
875
+ }
876
+ return { traceId: idBytes(traceId, 16, 'traceId'), spanId: idBytes(spanId, 8, 'spanId') }
877
+ }
878
+
879
+ function idBytes(hex: string, size: number, what: string): Uint8Array {
880
+ if (!new RegExp(`^[0-9a-fA-F]{${2 * size}}$`).test(hex)) {
881
+ throw new InvalidError(`${what} ${JSON.stringify(hex)}: ${2 * size} hex digits`)
882
+ }
883
+ return Uint8Array.from(Buffer.from(hex, 'hex'))
884
+ }
885
+
886
+ function nanosOf(date: Date): bigint {
887
+ return BigInt(date.getTime()) * 1_000_000n
888
+ }
889
+
890
+ function dateOf(nanos: bigint): Date {
891
+ return new Date(Number(nanos / 1_000_000n))
892
+ }
893
+
894
+ function mapValues<T, U>(
895
+ map: Record<string, T> | undefined,
896
+ read: (value: T) => U,
897
+ ): Record<string, U> {
898
+ return Object.fromEntries(Object.entries(map ?? {}).map(([key, value]) => [key, read(value)]))
899
+ }
900
+
901
+ /** Milliseconds as a span is written: '30d', '12h', '90s', '250ms'. */
902
+ function spelled(millis: number): DurationText {
903
+ for (const [unit, size] of [
904
+ ['d', 86_400_000],
905
+ ['h', 3_600_000],
906
+ ['m', 60_000],
907
+ ['s', 1000],
908
+ ] as const) {
909
+ if (millis >= size && millis % size === 0) {
910
+ return `${millis / size}${unit}` as DurationText
533
911
  }
534
- await upload.send(new Uint8Array(0), true)
535
- await upload.next()
536
912
  }
913
+ return `${millis}ms` as DurationText
537
914
  }