@tinyshed/tinystore 0.1.0-rc.4 → 0.1.0-rc.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +37 -37
- package/bin/tinystore.js +12 -1
- package/dist/backup.d.ts +33 -0
- package/dist/backup.js +49 -0
- package/dist/blobs.d.ts +160 -79
- package/dist/blobs.js +510 -217
- package/dist/channels.d.ts +90 -0
- package/dist/channels.js +291 -0
- package/dist/clock.js +3 -3
- package/dist/config.d.ts +209 -86
- package/dist/config.js +581 -538
- package/dist/connection.d.ts +64 -2
- package/dist/connection.js +201 -16
- package/dist/console.d.ts +27 -3
- package/dist/console.js +165 -12
- package/dist/dotenv.d.ts +7 -0
- package/dist/dotenv.js +79 -0
- package/dist/errors.d.ts +61 -1
- package/dist/errors.js +81 -1
- package/dist/handles.d.ts +44 -2
- package/dist/handles.js +81 -2
- package/dist/index.d.ts +17 -11
- package/dist/index.js +5 -3
- package/dist/instruments.d.ts +177 -0
- package/dist/instruments.js +819 -0
- package/dist/jobs.d.ts +335 -239
- package/dist/jobs.js +716 -536
- package/dist/json.d.ts +2 -0
- package/dist/json.js +122 -0
- package/dist/kv.d.ts +130 -243
- package/dist/kv.js +273 -640
- package/dist/limits.d.ts +80 -0
- package/dist/limits.js +139 -0
- package/dist/locks.d.ts +62 -0
- package/dist/locks.js +237 -0
- package/dist/logger.d.ts +45 -9
- package/dist/logger.js +150 -38
- package/dist/metrics.d.ts +210 -209
- package/dist/metrics.js +319 -556
- package/dist/once.d.ts +19 -18
- package/dist/once.js +74 -63
- package/dist/query.d.ts +181 -0
- package/dist/query.js +788 -0
- package/dist/records.d.ts +241 -150
- package/dist/records.js +507 -291
- package/dist/runtime/addon.d.ts +14 -0
- package/dist/runtime/addon.js +69 -0
- package/dist/runtime/bun.js +5 -7
- package/dist/runtime/node.js +5 -7
- package/dist/runtime/pipe.d.ts +17 -0
- package/dist/runtime/pipe.js +133 -0
- package/dist/runtime/shared.d.ts +6 -0
- package/dist/runtime/shared.js +19 -0
- package/dist/runtime.d.ts +5 -2
- package/dist/search.d.ts +189 -0
- package/dist/search.js +281 -0
- package/dist/session.d.ts +31 -3
- package/dist/session.js +153 -41
- package/dist/sql.d.ts +228 -78
- package/dist/sql.js +745 -240
- package/dist/store.d.ts +244 -24
- package/dist/store.js +270 -67
- package/dist/time.d.ts +5 -1
- package/dist/time.js +22 -2
- package/dist/tx.d.ts +61 -0
- package/dist/tx.js +314 -0
- package/dist/wire/codec.d.ts +23 -7
- package/dist/wire/codec.js +56 -35
- package/dist/wire/frame.d.ts +2 -0
- package/dist/wire/frame.js +31 -14
- package/dist/wire/messages.d.ts +259 -0
- package/dist/wire/messages.js +22 -0
- package/dist/wire/msgpack.d.ts +49 -1
- package/dist/wire/msgpack.js +293 -56
- package/dist/wire/protocol.d.ts +2683 -0
- package/dist/wire/protocol.js +10846 -0
- package/package.json +39 -38
- package/src/backup.ts +81 -0
- package/src/blobs.ts +678 -301
- package/src/channels.ts +377 -0
- package/src/clock.ts +4 -4
- package/src/config.ts +902 -612
- package/src/connection.ts +279 -21
- package/src/console.ts +180 -12
- package/src/dotenv.ts +82 -0
- package/src/errors.ts +105 -0
- package/src/handles.ts +157 -8
- package/src/index.ts +158 -52
- package/src/instruments.ts +1042 -0
- package/src/jobs.ts +1017 -733
- package/src/json.ts +133 -0
- package/src/kv.ts +424 -847
- package/src/limits.ts +202 -0
- package/src/locks.ts +320 -0
- package/src/logger.ts +209 -40
- package/src/metrics.ts +557 -752
- package/src/once.ts +86 -83
- package/src/query.ts +1005 -0
- package/src/records.ts +795 -418
- package/src/runtime/addon.ts +99 -0
- package/src/runtime/bun.ts +5 -7
- package/src/runtime/node.ts +5 -7
- package/src/runtime/pipe.ts +178 -0
- package/src/runtime/shared.ts +20 -0
- package/src/runtime.ts +5 -2
- package/src/search.ts +480 -0
- package/src/session.ts +187 -41
- package/src/sql.ts +993 -306
- package/src/store.ts +473 -78
- package/src/time.ts +48 -3
- package/src/tx.ts +426 -0
- package/src/wire/codec.ts +78 -37
- package/src/wire/frame.ts +44 -14
- package/src/wire/messages.ts +25 -0
- package/src/wire/msgpack.ts +327 -56
- package/src/wire/protocol.ts +11709 -0
- package/dist/limiter.d.ts +0 -45
- package/dist/limiter.js +0 -68
- package/dist/quota.d.ts +0 -44
- package/dist/quota.js +0 -98
- package/src/limiter.ts +0 -99
- package/src/quota.ts +0 -137
package/src/records.ts
CHANGED
|
@@ -1,537 +1,914 @@
|
|
|
1
|
-
//
|
|
2
|
-
//
|
|
3
|
-
//
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
import type
|
|
9
|
-
import { type
|
|
1
|
+
// Streams of events, as docs/guides/records/README.md has them, over the protocol: a
|
|
2
|
+
// stream opened by name, its events appended whole and read back by time and
|
|
3
|
+
// by what they hold, a page at a time, and every stream read together. A
|
|
4
|
+
// store's logger keeps its lines in a stream through the same append; a tail
|
|
5
|
+
// follows what is written from now on, and a writer of lines turns another
|
|
6
|
+
// program's output into events.
|
|
7
|
+
|
|
8
|
+
import { type Connection, type Link, pieceOf } from './connection.ts'
|
|
9
|
+
import { type ConsoleLine, jsonOf } from './console.ts'
|
|
10
|
+
import { ClosedError, InvalidError } from './errors.ts'
|
|
11
|
+
import { checkName, handleOn, type Via, viaLink } from './handles.ts'
|
|
12
|
+
import { parseJson } from './json.ts'
|
|
13
|
+
import {
|
|
14
|
+
encodeFields,
|
|
15
|
+
type LogConfig,
|
|
16
|
+
type Logger,
|
|
17
|
+
type LoggerOptions,
|
|
18
|
+
newStoreLogger,
|
|
19
|
+
} from './logger.ts'
|
|
10
20
|
import type { Stream } from './session.ts'
|
|
11
|
-
import { type Duration, ms } from './time.ts'
|
|
12
|
-
import { currentTrace } from './trace.ts'
|
|
21
|
+
import { type Duration, type DurationText, ms } from './time.ts'
|
|
22
|
+
import { currentTrace, type Trace } from './trace.ts'
|
|
13
23
|
import {
|
|
14
24
|
Empty,
|
|
25
|
+
Handle,
|
|
15
26
|
methods,
|
|
16
|
-
|
|
27
|
+
RecordsAggregateAsked,
|
|
28
|
+
RecordsAppend,
|
|
17
29
|
RecordsBatch,
|
|
18
|
-
|
|
30
|
+
RecordsCount,
|
|
19
31
|
RecordsDamage,
|
|
20
32
|
RecordsDamages,
|
|
33
|
+
RecordsDropped,
|
|
34
|
+
type RecordsEvent,
|
|
35
|
+
RecordsFieldNamesAsked,
|
|
36
|
+
RecordsNames,
|
|
37
|
+
type RecordsNewEvent,
|
|
38
|
+
RecordsOpen,
|
|
21
39
|
RecordsPage,
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
40
|
+
RecordsPiece,
|
|
41
|
+
RecordsPollAsked,
|
|
42
|
+
RecordsQuery,
|
|
43
|
+
RecordsRows,
|
|
44
|
+
RecordsStreams,
|
|
45
|
+
RecordsTailBatch,
|
|
46
|
+
RecordsTopAsked,
|
|
47
|
+
RecordsTops,
|
|
48
|
+
} from './wire/protocol.ts'
|
|
49
|
+
|
|
50
|
+
/** A level a logger logs at: one of its four names, or slog's number between them. */
|
|
27
51
|
export type Level = 'debug' | 'info' | 'warn' | 'error' | number
|
|
28
52
|
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
function levelOf(level: Level): number {
|
|
32
|
-
return typeof level === 'number' ? level : levels[level]
|
|
33
|
-
}
|
|
53
|
+
/** An event's level: what loggers and the programs whose lines are kept write. */
|
|
54
|
+
export type EventLevel = 'trace' | 'debug' | 'info' | 'warn' | 'error' | 'fatal'
|
|
34
55
|
|
|
35
|
-
/**
|
|
36
|
-
* Fields: keys and values, each value written as JSON. An object's keys keep
|
|
37
|
-
* their order; an array of pairs may repeat a key.
|
|
38
|
-
*/
|
|
56
|
+
/** What happened, or what it happened in: an object, or its pairs in their order. */
|
|
39
57
|
export type Fields = Record<string, unknown> | readonly (readonly [string, unknown])[]
|
|
40
58
|
|
|
41
|
-
/**
|
|
42
|
-
export interface
|
|
43
|
-
/** its time
|
|
44
|
-
at
|
|
45
|
-
/** the namespace the application names: 'web', 'billing' */
|
|
59
|
+
/** An event as a read gives it. */
|
|
60
|
+
export interface Event {
|
|
61
|
+
/** its time, which the file keeps to the nanosecond */
|
|
62
|
+
at: Date
|
|
46
63
|
stream: string
|
|
47
|
-
/**
|
|
64
|
+
/** what happened; `log` for a logger's line */
|
|
48
65
|
name: string
|
|
49
|
-
level?:
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
/**
|
|
55
|
-
|
|
56
|
-
/**
|
|
57
|
-
|
|
58
|
-
/** what happened */
|
|
59
|
-
attrs?: Fields
|
|
66
|
+
level?: EventLevel
|
|
67
|
+
message?: string
|
|
68
|
+
/** each value as JSON.parse reads it, an integer past 2^53 a bigint */
|
|
69
|
+
fields: Record<string, unknown>
|
|
70
|
+
context: Record<string, unknown>
|
|
71
|
+
/** 32 hex digits */
|
|
72
|
+
traceId?: string
|
|
73
|
+
/** 16 hex digits */
|
|
74
|
+
spanId?: string
|
|
60
75
|
}
|
|
61
76
|
|
|
62
|
-
/**
|
|
63
|
-
export interface
|
|
64
|
-
/** unix nanoseconds */
|
|
65
|
-
at: bigint
|
|
66
|
-
stream: string
|
|
77
|
+
/** An event to append: its name, what happened, and what it carries beside. */
|
|
78
|
+
export interface NewEvent {
|
|
67
79
|
name: string
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
80
|
+
fields?: Fields
|
|
81
|
+
/** its time: now unless given, from what the stream keeps to ten minutes ahead */
|
|
82
|
+
at?: Date
|
|
83
|
+
context?: Fields
|
|
84
|
+
level?: EventLevel | number
|
|
85
|
+
message?: string
|
|
86
|
+
/** a trace's 32 hex digits, with its span's 16: the current trace's unless given */
|
|
87
|
+
traceId?: string
|
|
88
|
+
spanId?: string
|
|
74
89
|
}
|
|
75
90
|
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
*
|
|
82
|
-
* ```ts
|
|
83
|
-
* for await (const record of store.records.all({ since: '1h', streams: ['readers'] })) {
|
|
84
|
-
* const { page, from } = fields(record.attrs)
|
|
85
|
-
* }
|
|
86
|
-
* ```
|
|
87
|
-
*/
|
|
88
|
-
export function fields(pairs: LogRecord['attrs']): Record<string, unknown> {
|
|
89
|
-
// fromEntries, since an assignment to __proto__ would set the prototype
|
|
90
|
-
return Object.fromEntries(pairs.map(([key, json]) => [key, JSON.parse(textOf(json))]))
|
|
91
|
+
export type AppendOptions = Omit<NewEvent, 'name' | 'fields'>
|
|
92
|
+
|
|
93
|
+
export interface EventsOptions {
|
|
94
|
+
/** how long the stream keeps its events: its own, kept in the file; 14 days until one says */
|
|
95
|
+
keep?: Duration
|
|
91
96
|
}
|
|
92
97
|
|
|
93
|
-
/**
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
* ```ts
|
|
97
|
-
* { since: '1h', minLevel: 'warn' } // the last hour's warnings
|
|
98
|
-
* { since: '1h', search: 'connection reset' } // whose text holds it, case ignored
|
|
99
|
-
* { from: start, to: end, streams: ['api'] }
|
|
100
|
-
* ```
|
|
101
|
-
*/
|
|
102
|
-
export interface RecordsQuery {
|
|
103
|
-
/** the span before now the range covers: '1h', '15m', or milliseconds; or from and to */
|
|
98
|
+
/** What a read finds: every condition given holds; one given as undefined is InvalidError. */
|
|
99
|
+
export interface Query {
|
|
100
|
+
/** the span before now, '1h'; or from and to */
|
|
104
101
|
since?: Duration
|
|
105
|
-
/** the range's start, included
|
|
106
|
-
from?: Date
|
|
107
|
-
/** the range's end, excluded
|
|
108
|
-
to?: Date
|
|
109
|
-
/**
|
|
110
|
-
streams?: string[]
|
|
111
|
-
names
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
/**
|
|
119
|
-
* text a record's body or name holds, its case ignored: every record the
|
|
120
|
-
* rest of the query leaves is read for it, so a search over much text ends
|
|
121
|
-
* pages early at the budget, and next goes on
|
|
122
|
-
*/
|
|
102
|
+
/** the range's start, included */
|
|
103
|
+
from?: Date
|
|
104
|
+
/** the range's end, excluded */
|
|
105
|
+
to?: Date
|
|
106
|
+
/** any of these streams, for every stream's read */
|
|
107
|
+
streams?: readonly string[]
|
|
108
|
+
/** any of these names: ['log'] is the lines */
|
|
109
|
+
names?: readonly string[]
|
|
110
|
+
/** this level and worse; an event without a level is not found */
|
|
111
|
+
minLevel?: EventLevel
|
|
112
|
+
traceId?: string
|
|
113
|
+
/** each key with this value, by the JSON it was written as, in fields or in context */
|
|
114
|
+
where?: Record<string, unknown>
|
|
115
|
+
/** this text in the message, the name or a value, its case ignored */
|
|
123
116
|
search?: string
|
|
124
|
-
/** newest first; oldest first
|
|
117
|
+
/** newest first; oldest first without it */
|
|
125
118
|
newest?: boolean
|
|
126
|
-
/** the
|
|
119
|
+
/** the events of a page: 1000, and 10,000 at most */
|
|
127
120
|
limit?: number
|
|
128
|
-
/**
|
|
129
|
-
|
|
130
|
-
/** a page's next, which this page continues from; the rest of the query as it was */
|
|
131
|
-
after?: string | undefined
|
|
121
|
+
/** a page's next, with the rest of the query as it was */
|
|
122
|
+
after?: string
|
|
132
123
|
}
|
|
133
124
|
|
|
134
|
-
/**
|
|
135
|
-
export
|
|
136
|
-
segment: number
|
|
137
|
-
row: number
|
|
138
|
-
}
|
|
125
|
+
/** A query of every event it finds, for what counts them. */
|
|
126
|
+
export type Counted = Omit<Query, 'newest' | 'limit' | 'after'>
|
|
139
127
|
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
segment: number | undefined
|
|
145
|
-
headRow: number | undefined
|
|
146
|
-
/** the times it held, unix nanoseconds */
|
|
147
|
-
from: bigint
|
|
148
|
-
to: bigint
|
|
149
|
-
/** the invariant its bytes broke */
|
|
150
|
-
reason: string
|
|
128
|
+
export interface Page {
|
|
129
|
+
events: Event[]
|
|
130
|
+
/** the page after it; the last page has none */
|
|
131
|
+
next?: string
|
|
151
132
|
}
|
|
152
133
|
|
|
153
|
-
|
|
134
|
+
export interface Top {
|
|
135
|
+
value: unknown
|
|
136
|
+
count: number
|
|
137
|
+
}
|
|
154
138
|
|
|
155
|
-
|
|
156
|
-
|
|
139
|
+
/** A field's numbers in a row: a sum of integers exact, a bigint past 2^53. */
|
|
140
|
+
export interface Stats {
|
|
141
|
+
samples: number
|
|
142
|
+
sum: number | bigint
|
|
143
|
+
min: number | bigint
|
|
144
|
+
max: number | bigint
|
|
145
|
+
avg: number
|
|
157
146
|
}
|
|
158
147
|
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
}
|
|
167
|
-
const raw = typeof id === 'string' ? new Uint8Array(Buffer.from(id, 'hex')) : id
|
|
168
|
-
if (raw.length !== bytes) {
|
|
169
|
-
throw new InvalidError(`a ${what} id of ${raw.length} bytes, not ${bytes}`)
|
|
170
|
-
}
|
|
171
|
-
return raw
|
|
148
|
+
export interface Row {
|
|
149
|
+
/** where the row's span begins, counted from 1970 in UTC */
|
|
150
|
+
at: Date
|
|
151
|
+
count: number
|
|
152
|
+
by: Record<string, unknown>
|
|
153
|
+
stats: Record<string, Stats>
|
|
154
|
+
distinct: Record<string, number>
|
|
172
155
|
}
|
|
173
156
|
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
if (json === undefined) {
|
|
184
|
-
throw new InvalidError(`the field ${key} holds a value JSON cannot write`)
|
|
185
|
-
}
|
|
186
|
-
flat.push(key, json)
|
|
187
|
-
}
|
|
188
|
-
return flat.length === 0 ? undefined : flat
|
|
157
|
+
export interface Aggregate extends Counted {
|
|
158
|
+
/** the span of a row: the range is one without it */
|
|
159
|
+
every?: Duration
|
|
160
|
+
/** keys whose every value has rows of its own */
|
|
161
|
+
by?: readonly string[]
|
|
162
|
+
/** fields whose numbers a row sums */
|
|
163
|
+
stats?: readonly string[]
|
|
164
|
+
/** fields whose different values a row counts */
|
|
165
|
+
distinct?: readonly string[]
|
|
189
166
|
}
|
|
190
167
|
|
|
191
|
-
|
|
168
|
+
export interface StreamInfo {
|
|
169
|
+
name: string
|
|
170
|
+
keep: DurationText
|
|
171
|
+
/** its oldest event, or where its keep begins while it holds older */
|
|
172
|
+
retainedFrom: Date
|
|
173
|
+
newest: Date
|
|
174
|
+
}
|
|
192
175
|
|
|
193
|
-
|
|
194
|
-
|
|
176
|
+
export interface PollBatch {
|
|
177
|
+
events: Event[]
|
|
178
|
+
/** where the next poll begins, kept once these events are safe */
|
|
179
|
+
cursor: string
|
|
180
|
+
/** the events keep removed before the cursor reached them */
|
|
181
|
+
expired: number
|
|
195
182
|
}
|
|
196
183
|
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
184
|
+
/** What a tail finds: a query's conditions, and no range, order or limit. */
|
|
185
|
+
export type TailQuery = Pick<
|
|
186
|
+
Query,
|
|
187
|
+
'streams' | 'names' | 'minLevel' | 'traceId' | 'where' | 'search'
|
|
188
|
+
>
|
|
189
|
+
|
|
190
|
+
/** A tail's batch: the events written since the last, and how many it missed. */
|
|
191
|
+
export interface TailBatch {
|
|
192
|
+
events: Event[]
|
|
193
|
+
/** the events that left the store's hold before the tail read them */
|
|
194
|
+
missed: number
|
|
203
195
|
}
|
|
204
196
|
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
}
|
|
197
|
+
/** A head row or a segment that no longer reads, as `damaged` lists it and `drop` takes it. */
|
|
198
|
+
export interface Damage {
|
|
199
|
+
/** a head row holds events not packed yet, a segment packed ones */
|
|
200
|
+
part: 'head row' | 'segment'
|
|
201
|
+
id: number
|
|
202
|
+
stream: string
|
|
203
|
+
/** its events' oldest and newest times */
|
|
204
|
+
first: Date
|
|
205
|
+
last: Date
|
|
206
|
+
/** what broke */
|
|
207
|
+
error: string
|
|
217
208
|
}
|
|
218
209
|
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
*/
|
|
225
|
-
|
|
226
|
-
|
|
210
|
+
type Method = keyof typeof methods
|
|
211
|
+
|
|
212
|
+
const newline = 0x0a
|
|
213
|
+
/** What a writer of lines holds for the server at most: past it, lines are dropped and counted. */
|
|
214
|
+
const linesHeld = 1 << 20
|
|
215
|
+
/** The bytes one event holds: a longer line is dropped whole, as the store drops one. */
|
|
216
|
+
const eventRoom = 256 << 10
|
|
217
|
+
const encoder = new TextEncoder()
|
|
218
|
+
|
|
219
|
+
const levelNumbers: Record<EventLevel, number> = {
|
|
220
|
+
trace: -8,
|
|
221
|
+
debug: -4,
|
|
222
|
+
info: 0,
|
|
223
|
+
warn: 4,
|
|
224
|
+
error: 8,
|
|
225
|
+
fatal: 12,
|
|
227
226
|
}
|
|
227
|
+
const levelNames: readonly EventLevel[] = ['trace', 'debug', 'info', 'warn', 'error', 'fatal']
|
|
228
|
+
|
|
229
|
+
/** What every read takes: a stream's handle, or every stream without one. */
|
|
230
|
+
abstract class Reads {
|
|
231
|
+
protected abstract ask(
|
|
232
|
+
method: Method,
|
|
233
|
+
body: (handle: number | undefined) => Uint8Array,
|
|
234
|
+
): Promise<Uint8Array>
|
|
235
|
+
|
|
236
|
+
protected abstract opened(
|
|
237
|
+
method: Method,
|
|
238
|
+
body: (handle: number | undefined) => Uint8Array,
|
|
239
|
+
): Promise<Stream>
|
|
240
|
+
|
|
241
|
+
/** One page, from one moment of the file: `{ events, next }`. */
|
|
242
|
+
async list(query?: Query): Promise<Page> {
|
|
243
|
+
const asked = queryOf(query)
|
|
244
|
+
const answer = await this.ask('records.list', handle =>
|
|
245
|
+
RecordsQuery.encode({ ...asked, handle }),
|
|
246
|
+
)
|
|
247
|
+
const page = RecordsPage.decode(answer)
|
|
248
|
+
const events = (page.events ?? []).map(eventOf)
|
|
249
|
+
return page.next === undefined ? { events } : { events, next: page.next }
|
|
250
|
+
}
|
|
228
251
|
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
from: ends[1] === '' ? undefined : BigInt(ends[1]!),
|
|
240
|
-
to: ends[2] === '' ? undefined : BigInt(ends[2]!),
|
|
252
|
+
/** Every event the query finds, a page after another. */
|
|
253
|
+
async *all(query?: Query): AsyncGenerator<Event> {
|
|
254
|
+
let after = query?.after
|
|
255
|
+
for (;;) {
|
|
256
|
+
const page = await this.list(after === undefined ? query : { ...query, after })
|
|
257
|
+
yield* page.events
|
|
258
|
+
if (page.next === undefined) {
|
|
259
|
+
return
|
|
260
|
+
}
|
|
261
|
+
after = page.next
|
|
241
262
|
}
|
|
242
263
|
}
|
|
243
|
-
|
|
244
|
-
|
|
264
|
+
|
|
265
|
+
/** How many events the query finds. */
|
|
266
|
+
async count(query?: Counted): Promise<number> {
|
|
267
|
+
const asked = queryOf(query)
|
|
268
|
+
const answer = await this.ask('records.count', handle =>
|
|
269
|
+
RecordsQuery.encode({ ...asked, handle }),
|
|
270
|
+
)
|
|
271
|
+
return RecordsCount.decode(answer).count ?? 0
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
/**
|
|
275
|
+
* A key's most frequent values and how many events hold each: ten unless
|
|
276
|
+
* `limit` asks for more, at most 1000. The key is a field's or a
|
|
277
|
+
* context's, or `stream`, `name` or `level`.
|
|
278
|
+
*/
|
|
279
|
+
async top(key: string, query?: Counted & { limit?: number }): Promise<Top[]> {
|
|
280
|
+
const asked = queryOf(query)
|
|
281
|
+
const answer = await this.ask('records.top', handle =>
|
|
282
|
+
RecordsTopAsked.encode({ query: { ...asked, handle }, key }),
|
|
283
|
+
)
|
|
284
|
+
return (RecordsTops.decode(answer).values ?? []).map(top => ({
|
|
285
|
+
value: parseJson(top.value ?? 'null'),
|
|
286
|
+
count: top.count ?? 0,
|
|
287
|
+
}))
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
/** A row for each span of `every` and each value of the keys of `by`, with its count. */
|
|
291
|
+
async aggregate(query: Aggregate = {}): Promise<Row[]> {
|
|
292
|
+
const { every, by, stats, distinct, ...counted } = query
|
|
293
|
+
const asked = queryOf(counted)
|
|
294
|
+
const answer = await this.ask('records.aggregate', handle =>
|
|
295
|
+
RecordsAggregateAsked.encode({
|
|
296
|
+
query: { ...asked, handle },
|
|
297
|
+
every: every === undefined ? undefined : ms(every),
|
|
298
|
+
by: by === undefined ? undefined : [...by],
|
|
299
|
+
stats: stats === undefined ? undefined : [...stats],
|
|
300
|
+
distinct: distinct === undefined ? undefined : [...distinct],
|
|
301
|
+
}),
|
|
302
|
+
)
|
|
303
|
+
return (RecordsRows.decode(answer).rows ?? []).map(row => ({
|
|
304
|
+
at: dateOf(row.at ?? 0n),
|
|
305
|
+
count: row.count ?? 0,
|
|
306
|
+
by: mapValues(row.by, json => parseJson(json)),
|
|
307
|
+
stats: mapValues(row.stats, kept => ({
|
|
308
|
+
samples: kept.samples ?? 0,
|
|
309
|
+
sum: parseJson(kept.sum ?? '0') as number | bigint,
|
|
310
|
+
min: parseJson(kept.min ?? '0') as number | bigint,
|
|
311
|
+
max: parseJson(kept.max ?? '0') as number | bigint,
|
|
312
|
+
avg: kept.avg ?? 0,
|
|
313
|
+
})),
|
|
314
|
+
distinct: mapValues(row.distinct, count => count),
|
|
315
|
+
}))
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
/** The keys the events hold, in fields and in context, sorted; `prefix` narrows them. */
|
|
319
|
+
async fieldNames(
|
|
320
|
+
query?: Omit<Query, 'newest' | 'after'> & { prefix?: string },
|
|
321
|
+
): Promise<string[]> {
|
|
322
|
+
const { prefix, ...rest } = query ?? {}
|
|
323
|
+
const asked = queryOf(rest)
|
|
324
|
+
const answer = await this.ask('records.fieldNames', handle =>
|
|
325
|
+
RecordsFieldNamesAsked.encode({ query: { ...asked, handle }, prefix }),
|
|
326
|
+
)
|
|
327
|
+
return RecordsNames.decode(answer).names ?? []
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
/**
|
|
331
|
+
* The events written from now on that the query finds, a batch as they
|
|
332
|
+
* are written: `{ events, missed }`. A reader slower than the writers
|
|
333
|
+
* skips what left the store's hold, and the batch after says how many.
|
|
334
|
+
* It ends when its loop is left, and when the server's store closes.
|
|
335
|
+
*/
|
|
336
|
+
async *tail(query?: TailQuery): AsyncGenerator<TailBatch> {
|
|
337
|
+
const asked = queryOf(query)
|
|
338
|
+
const stream = await this.opened('records.tail', handle =>
|
|
339
|
+
RecordsQuery.encode({ ...asked, handle }),
|
|
340
|
+
)
|
|
341
|
+
try {
|
|
342
|
+
for (;;) {
|
|
343
|
+
const event = await stream.next()
|
|
344
|
+
stream.consumed(event.body.length)
|
|
345
|
+
if (event.end) {
|
|
346
|
+
return
|
|
347
|
+
}
|
|
348
|
+
const batch = RecordsTailBatch.decode(event.body)
|
|
349
|
+
yield { events: (batch.events ?? []).map(eventOf), missed: batch.missed ?? 0 }
|
|
350
|
+
}
|
|
351
|
+
} finally {
|
|
352
|
+
stream.cancel()
|
|
353
|
+
}
|
|
245
354
|
}
|
|
246
|
-
const from =
|
|
247
|
-
q?.since !== undefined
|
|
248
|
-
? nanos(new Date(Date.now() - ms(q.since)))
|
|
249
|
-
: q?.from === undefined
|
|
250
|
-
? undefined
|
|
251
|
-
: nanos(q.from)
|
|
252
|
-
return { from, to: q?.to === undefined ? undefined : nanos(q.to) }
|
|
253
355
|
}
|
|
254
356
|
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
limit: q?.limit,
|
|
266
|
-
budgetBlocks: q?.budget?.blocks,
|
|
267
|
-
budgetBytes: q?.budget?.bytes,
|
|
268
|
-
budgetRecords: q?.budget?.records,
|
|
269
|
-
search: q?.search === '' ? undefined : q?.search,
|
|
270
|
-
}
|
|
271
|
-
}
|
|
272
|
-
|
|
273
|
-
export class Records {
|
|
357
|
+
/** Opens a stream of events: lazily, at its first call on a connection. */
|
|
358
|
+
export function openEvents(link: Link, stream: string, options?: EventsOptions): Events {
|
|
359
|
+
checkName(stream, 'stream')
|
|
360
|
+
const keep = options?.keep === undefined ? undefined : ms(options.keep)
|
|
361
|
+
return new Events(link, stream, RecordsOpen.encode({ stream, keep }))
|
|
362
|
+
}
|
|
363
|
+
|
|
364
|
+
/** A stream of events: written with `append`, which waits for the file, and read back. */
|
|
365
|
+
export class Events extends Reads {
|
|
366
|
+
readonly stream: string
|
|
274
367
|
readonly #link: Link
|
|
275
|
-
readonly #
|
|
368
|
+
readonly #via: Via
|
|
369
|
+
readonly #open: Uint8Array
|
|
276
370
|
|
|
277
|
-
|
|
371
|
+
/** Events come from `store.events(stream)`. */
|
|
372
|
+
constructor(link: Link, stream: string, open: Uint8Array) {
|
|
373
|
+
super()
|
|
374
|
+
this.stream = stream
|
|
278
375
|
this.#link = link
|
|
376
|
+
this.#via = viaLink(link)
|
|
377
|
+
this.#open = open
|
|
279
378
|
}
|
|
280
379
|
|
|
281
380
|
/**
|
|
282
|
-
*
|
|
283
|
-
*
|
|
381
|
+
* Writes an event, or a list of them whole or not at all: it returns once
|
|
382
|
+
* they are written, and a read finds them at once. A refused event names
|
|
383
|
+
* its place in the list and its name.
|
|
284
384
|
*/
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
385
|
+
append(name: string, fields?: Fields, options?: AppendOptions): Promise<void>
|
|
386
|
+
append(events: readonly NewEvent[]): Promise<void>
|
|
387
|
+
async append(
|
|
388
|
+
first: string | readonly NewEvent[],
|
|
389
|
+
fields?: Fields,
|
|
390
|
+
options?: AppendOptions,
|
|
391
|
+
): Promise<void> {
|
|
392
|
+
const events = typeof first === 'string' ? [oneEvent(first, fields, options)] : first
|
|
393
|
+
const encoded = events.map(newEventOf)
|
|
394
|
+
await this.#call(
|
|
395
|
+
'records.append',
|
|
396
|
+
handle => RecordsAppend.encode({ handle, events: encoded }),
|
|
397
|
+
'write',
|
|
398
|
+
)
|
|
289
399
|
}
|
|
290
400
|
|
|
291
|
-
/**
|
|
292
|
-
|
|
293
|
-
|
|
401
|
+
/** A logger's lines, each an event of this stream. */
|
|
402
|
+
appendLines(lines: readonly ConsoleLine[]): Promise<void> {
|
|
403
|
+
const events = lines.map(line => ({
|
|
404
|
+
name: line.event ?? 'log',
|
|
405
|
+
at: nanosOf(line.at),
|
|
406
|
+
level: line.level,
|
|
407
|
+
message: line.msg,
|
|
408
|
+
fields: objectOf(line.attrs),
|
|
409
|
+
context: objectOf(line.context),
|
|
410
|
+
...wireTrace(line.traceId, line.spanId),
|
|
411
|
+
}))
|
|
412
|
+
return this.#call(
|
|
413
|
+
'records.append',
|
|
414
|
+
handle => RecordsAppend.encode({ handle, events }),
|
|
415
|
+
'write',
|
|
416
|
+
).then(() => {})
|
|
294
417
|
}
|
|
295
418
|
|
|
296
419
|
/**
|
|
297
|
-
*
|
|
298
|
-
*
|
|
299
|
-
*
|
|
420
|
+
* A writer of another program's output into this stream: each line it is
|
|
421
|
+
* given becomes an event at the time it arrived, a stack trace's or a JSON
|
|
422
|
+
* value's lines one, and a write never waits.
|
|
423
|
+
*
|
|
424
|
+
* const out = store.events('worker').lines()
|
|
425
|
+
* child.stdout.on('data', chunk => out.write(chunk))
|
|
426
|
+
* child.on('close', () => out.end())
|
|
300
427
|
*/
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
? { ...r, traceId: carried.traceId, spanId: carried.spanId }
|
|
310
|
-
: r,
|
|
311
|
-
),
|
|
312
|
-
)
|
|
428
|
+
lines(): Lines {
|
|
429
|
+
return new Lines(async () => {
|
|
430
|
+
const connection = await this.#link.connection()
|
|
431
|
+
const handle = await handleOn(connection, methods['records.open'], this.#open)
|
|
432
|
+
const body = Handle.encode({ handle })
|
|
433
|
+
const stream = await streamOf(connection, methods['records.lines'], body, false)
|
|
434
|
+
return { stream, piece: pieceOf(connection) }
|
|
435
|
+
})
|
|
313
436
|
}
|
|
314
437
|
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
stream: r.stream,
|
|
321
|
-
name: r.name,
|
|
322
|
-
level: r.level === undefined ? undefined : levelOf(r.level),
|
|
323
|
-
body: r.body,
|
|
324
|
-
traceId: idOf(r.traceId, 16, 'trace'),
|
|
325
|
-
spanId: idOf(r.spanId, 8, 'span'),
|
|
326
|
-
context: fieldsOf(r.context),
|
|
327
|
-
attrs: fieldsOf(r.attrs),
|
|
328
|
-
})),
|
|
329
|
-
)
|
|
438
|
+
protected ask(
|
|
439
|
+
method: Method,
|
|
440
|
+
body: (handle: number | undefined) => Uint8Array,
|
|
441
|
+
): Promise<Uint8Array> {
|
|
442
|
+
return this.#call(method, body, 'read')
|
|
330
443
|
}
|
|
331
444
|
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
body: line.msg,
|
|
341
|
-
traceId: idOf(line.traceId, 16, 'trace'),
|
|
342
|
-
spanId: idOf(line.spanId, 8, 'span'),
|
|
343
|
-
context: line.context.length === 0 ? undefined : [...line.context],
|
|
344
|
-
attrs: line.attrs.length === 0 ? undefined : [...line.attrs],
|
|
345
|
-
})),
|
|
346
|
-
)
|
|
445
|
+
protected opened(
|
|
446
|
+
method: Method,
|
|
447
|
+
body: (handle: number | undefined) => Uint8Array,
|
|
448
|
+
): Promise<Stream> {
|
|
449
|
+
return this.#link.run('read', async connection => {
|
|
450
|
+
const handle = await handleOn(connection, methods['records.open'], this.#open)
|
|
451
|
+
return streamOf(connection, methods[method], body(handle), true)
|
|
452
|
+
})
|
|
347
453
|
}
|
|
348
454
|
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
455
|
+
#call(
|
|
456
|
+
method: Method,
|
|
457
|
+
body: (handle: number) => Uint8Array,
|
|
458
|
+
idempotence: 'read' | 'write',
|
|
459
|
+
): Promise<Uint8Array> {
|
|
460
|
+
return this.#via.call(methods['records.open'], this.#open, methods[method], body, idempotence)
|
|
353
461
|
}
|
|
462
|
+
}
|
|
354
463
|
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
/** Every record a query matches, a page at a time. */
|
|
376
|
-
async *all(query?: Omit<RecordsQuery, 'after'>): AsyncGenerator<LogRecord> {
|
|
377
|
-
let page = await this.scan(query)
|
|
378
|
-
yield* page.items
|
|
379
|
-
while (page.next !== undefined) {
|
|
380
|
-
page = await this.scan({ ...query, after: page.next })
|
|
381
|
-
yield* page.items
|
|
382
|
-
}
|
|
464
|
+
/** Every stream's events, to read: `store.events()`. */
|
|
465
|
+
export class AllEvents extends Reads {
|
|
466
|
+
readonly #link: Link
|
|
467
|
+
|
|
468
|
+
/** AllEvents come from `store.events()`. */
|
|
469
|
+
constructor(link: Link) {
|
|
470
|
+
super()
|
|
471
|
+
this.#link = link
|
|
472
|
+
}
|
|
473
|
+
|
|
474
|
+
/** Every stream with events, by name, with what it keeps, where that begins and its newest event's time. */
|
|
475
|
+
async streams(): Promise<StreamInfo[]> {
|
|
476
|
+
const answer = await this.ask('records.streams', () => Empty.encode({}))
|
|
477
|
+
return (RecordsStreams.decode(answer).streams ?? []).map(stream => ({
|
|
478
|
+
name: stream.name ?? '',
|
|
479
|
+
keep: spelled(stream.keep ?? 0),
|
|
480
|
+
retainedFrom: dateOf(stream.retainedFrom ?? 0n),
|
|
481
|
+
newest: dateOf(stream.newest ?? 0n),
|
|
482
|
+
}))
|
|
383
483
|
}
|
|
384
484
|
|
|
385
485
|
/**
|
|
386
|
-
*
|
|
387
|
-
*
|
|
388
|
-
* the cursor
|
|
486
|
+
* Every event once, a batch a call, from a cursor the program keeps, the
|
|
487
|
+
* oldest event kept without one: `{ events, cursor, expired }`. With no
|
|
488
|
+
* event past the cursor, the batch is empty and its cursor the one given.
|
|
389
489
|
*/
|
|
390
|
-
async
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
): Promise<{ records: LogRecord[]; cursor: Cursor; expired: number }> {
|
|
394
|
-
const got = await this.#link.run('read', connection =>
|
|
395
|
-
download(
|
|
396
|
-
connection,
|
|
397
|
-
methods['records.follow'],
|
|
398
|
-
RecordsCursor.encode({ segment: cursor?.segment, row: cursor?.row, limit }),
|
|
399
|
-
),
|
|
490
|
+
async poll(cursor?: string, options?: { limit?: number }): Promise<PollBatch> {
|
|
491
|
+
const answer = await this.ask('records.poll', () =>
|
|
492
|
+
RecordsPollAsked.encode({ cursor, limit: options?.limit }),
|
|
400
493
|
)
|
|
401
|
-
const
|
|
494
|
+
const batch = RecordsBatch.decode(answer)
|
|
402
495
|
return {
|
|
403
|
-
|
|
404
|
-
cursor:
|
|
405
|
-
expired:
|
|
496
|
+
events: (batch.events ?? []).map(eventOf),
|
|
497
|
+
cursor: batch.cursor ?? '',
|
|
498
|
+
expired: batch.expired ?? 0,
|
|
406
499
|
}
|
|
407
500
|
}
|
|
408
501
|
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
if (stream === '') {
|
|
417
|
-
throw new InvalidError('lines of no stream')
|
|
418
|
-
}
|
|
419
|
-
return new LinesWriter(this.#link, stream, options?.buffer ?? 1 << 20)
|
|
502
|
+
protected ask(
|
|
503
|
+
method: Method,
|
|
504
|
+
body: (handle: number | undefined) => Uint8Array,
|
|
505
|
+
): Promise<Uint8Array> {
|
|
506
|
+
return this.#link.run('read', connection =>
|
|
507
|
+
connection.session.call(methods[method], body(undefined)),
|
|
508
|
+
)
|
|
420
509
|
}
|
|
421
510
|
|
|
422
|
-
/**
|
|
511
|
+
/**
|
|
512
|
+
* What reads met that no longer reads, which `drop` removes: each head
|
|
513
|
+
* row and segment with its stream and its events' times. What the store
|
|
514
|
+
* has not met since it opened is not listed.
|
|
515
|
+
*/
|
|
423
516
|
async damaged(): Promise<Damage[]> {
|
|
424
|
-
const
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
stream:
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
to: d.to ?? 0n,
|
|
433
|
-
reason: d.reason ?? '',
|
|
517
|
+
const answer = await this.ask('records.damaged', () => Empty.encode({}))
|
|
518
|
+
return (RecordsDamages.decode(answer).damages ?? []).map(damage => ({
|
|
519
|
+
part: damage.part === 'segment' ? 'segment' : 'head row',
|
|
520
|
+
id: damage.id ?? 0,
|
|
521
|
+
stream: damage.stream ?? '',
|
|
522
|
+
first: dateOf(damage.first ?? 0n),
|
|
523
|
+
last: dateOf(damage.last ?? 0n),
|
|
524
|
+
error: damage.error ?? '',
|
|
434
525
|
}))
|
|
435
526
|
}
|
|
436
527
|
|
|
437
|
-
/**
|
|
528
|
+
/**
|
|
529
|
+
* Removes a head row or a segment that `damaged` listed, so that the
|
|
530
|
+
* reads over it work again; what still reads is refused.
|
|
531
|
+
*
|
|
532
|
+
* for (const damage of await store.events().damaged()) await store.events().drop(damage)
|
|
533
|
+
*/
|
|
438
534
|
async drop(damage: Damage): Promise<void> {
|
|
535
|
+
const body = RecordsDamage.encode({
|
|
536
|
+
...damage,
|
|
537
|
+
first: nanosOf(damage.first),
|
|
538
|
+
last: nanosOf(damage.last),
|
|
539
|
+
})
|
|
439
540
|
await this.#link.run('write', connection =>
|
|
440
|
-
connection.session.call(methods['records.drop'],
|
|
541
|
+
connection.session.call(methods['records.drop'], body),
|
|
542
|
+
)
|
|
543
|
+
}
|
|
544
|
+
|
|
545
|
+
protected opened(
|
|
546
|
+
method: Method,
|
|
547
|
+
body: (handle: number | undefined) => Uint8Array,
|
|
548
|
+
): Promise<Stream> {
|
|
549
|
+
return this.#link.run('read', connection =>
|
|
550
|
+
streamOf(connection, methods[method], body(undefined), true),
|
|
441
551
|
)
|
|
442
552
|
}
|
|
443
553
|
}
|
|
444
554
|
|
|
445
|
-
/**
|
|
446
|
-
|
|
555
|
+
/** A writer's stream, and the bytes a piece of it carries. */
|
|
556
|
+
interface Opened {
|
|
557
|
+
stream: Stream
|
|
558
|
+
piece: number
|
|
559
|
+
}
|
|
447
560
|
|
|
448
561
|
/**
|
|
449
|
-
*
|
|
450
|
-
*
|
|
451
|
-
*
|
|
452
|
-
* uploads on the next connection.
|
|
562
|
+
* A writer of another program's output into a stream, as `events.lines()`
|
|
563
|
+
* begins it: each line becomes an event, and a write never waits. It sends
|
|
564
|
+
* whole lines, so that what it has no room for, `dropped` counts as lines.
|
|
453
565
|
*/
|
|
454
|
-
export class
|
|
455
|
-
|
|
456
|
-
readonly #
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
#
|
|
460
|
-
|
|
461
|
-
#
|
|
566
|
+
export class Lines {
|
|
567
|
+
/** whole lines not sent yet */
|
|
568
|
+
readonly #queued: Uint8Array[] = []
|
|
569
|
+
#queuedBytes = 0
|
|
570
|
+
/** a line not ended yet */
|
|
571
|
+
#partial = new Uint8Array(0)
|
|
572
|
+
/** the line being written is longer than an event holds: the rest of it goes too */
|
|
573
|
+
#discarding = false
|
|
574
|
+
#dropped = 0
|
|
462
575
|
#ended = false
|
|
463
|
-
|
|
464
|
-
|
|
576
|
+
#failure: Error | undefined
|
|
577
|
+
#wake: (() => void) | undefined
|
|
578
|
+
readonly #sent: Promise<void>
|
|
579
|
+
|
|
580
|
+
/** Lines come from `events.lines()`. */
|
|
581
|
+
constructor(open: () => Promise<Opened>) {
|
|
582
|
+
this.#sent = this.#send(open).catch((err: unknown) => {
|
|
583
|
+
this.#failure = err instanceof Error ? err : new Error(String(err))
|
|
584
|
+
for (const bytes of this.#queued.splice(0)) {
|
|
585
|
+
this.#dropped += linesIn(bytes)
|
|
586
|
+
}
|
|
587
|
+
this.#queuedBytes = 0
|
|
588
|
+
})
|
|
589
|
+
}
|
|
465
590
|
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
this.#
|
|
469
|
-
this.#most = most
|
|
591
|
+
/** The lines dropped: past what the writer holds, and, once it ended, by the store. */
|
|
592
|
+
get dropped(): number {
|
|
593
|
+
return this.#dropped
|
|
470
594
|
}
|
|
471
595
|
|
|
472
|
-
/** Takes a piece of output, cut anywhere
|
|
473
|
-
write(chunk:
|
|
596
|
+
/** Takes a piece of output, cut anywhere; it never waits. */
|
|
597
|
+
write(chunk: Uint8Array | string): void {
|
|
474
598
|
if (this.#ended) {
|
|
475
|
-
throw new
|
|
599
|
+
throw new ClosedError('lines written after their end')
|
|
600
|
+
}
|
|
601
|
+
let bytes = typeof chunk === 'string' ? encoder.encode(chunk) : chunk
|
|
602
|
+
if (this.#discarding) {
|
|
603
|
+
const end = bytes.indexOf(newline)
|
|
604
|
+
if (end < 0) {
|
|
605
|
+
return
|
|
606
|
+
}
|
|
607
|
+
this.#discarding = false
|
|
608
|
+
bytes = bytes.subarray(end + 1)
|
|
476
609
|
}
|
|
477
|
-
const
|
|
610
|
+
const joined = concat(this.#partial, bytes)
|
|
611
|
+
const cut = joined.lastIndexOf(newline) + 1
|
|
612
|
+
this.#partial = joined.slice(cut)
|
|
613
|
+
if (this.#partial.length > eventRoom) {
|
|
614
|
+
this.#partial = new Uint8Array(0)
|
|
615
|
+
this.#discarding = true
|
|
616
|
+
this.#dropped++
|
|
617
|
+
}
|
|
618
|
+
this.#queue(joined.subarray(0, cut))
|
|
619
|
+
}
|
|
620
|
+
|
|
621
|
+
/**
|
|
622
|
+
* Ends the writer: what it holds goes, a last line without its newline
|
|
623
|
+
* too, and it returns once the store wrote them.
|
|
624
|
+
*/
|
|
625
|
+
async end(): Promise<void> {
|
|
626
|
+
if (!this.#ended) {
|
|
627
|
+
this.#ended = true
|
|
628
|
+
this.#queue(this.#partial)
|
|
629
|
+
this.#partial = new Uint8Array(0)
|
|
630
|
+
this.#wake?.()
|
|
631
|
+
}
|
|
632
|
+
await this.#sent
|
|
633
|
+
if (this.#failure !== undefined) {
|
|
634
|
+
throw this.#failure
|
|
635
|
+
}
|
|
636
|
+
}
|
|
637
|
+
|
|
638
|
+
/** Holds bytes for the server, or drops and counts their lines when it holds no room. */
|
|
639
|
+
#queue(bytes: Uint8Array): void {
|
|
478
640
|
if (bytes.length === 0) {
|
|
479
|
-
return
|
|
641
|
+
return
|
|
480
642
|
}
|
|
481
|
-
if (this.#
|
|
482
|
-
this
|
|
483
|
-
return
|
|
643
|
+
if (this.#failure !== undefined || this.#queuedBytes + bytes.length > linesHeld) {
|
|
644
|
+
this.#dropped += linesIn(bytes)
|
|
645
|
+
return
|
|
484
646
|
}
|
|
485
|
-
|
|
486
|
-
|
|
647
|
+
this.#queued.push(bytes)
|
|
648
|
+
this.#queuedBytes += bytes.length
|
|
649
|
+
this.#wake?.()
|
|
650
|
+
}
|
|
651
|
+
|
|
652
|
+
/** Sends what is held as it comes, in pieces within the stream's credit, then the end. */
|
|
653
|
+
async #send(open: () => Promise<Opened>): Promise<void> {
|
|
654
|
+
const { stream, piece } = await open()
|
|
655
|
+
for (;;) {
|
|
656
|
+
const bytes = this.#queued.shift()
|
|
657
|
+
if (bytes === undefined) {
|
|
658
|
+
if (this.#ended) {
|
|
659
|
+
break
|
|
660
|
+
}
|
|
661
|
+
await new Promise<void>(resolve => {
|
|
662
|
+
this.#wake = resolve
|
|
663
|
+
})
|
|
664
|
+
this.#wake = undefined
|
|
665
|
+
continue
|
|
666
|
+
}
|
|
667
|
+
this.#queuedBytes -= bytes.length
|
|
668
|
+
for (let at = 0; at < bytes.length; at += piece) {
|
|
669
|
+
await stream.send(RecordsPiece.encode({ bytes: bytes.subarray(at, at + piece) }), false)
|
|
670
|
+
}
|
|
487
671
|
}
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
}
|
|
494
|
-
|
|
495
|
-
async #pump(): Promise<void> {
|
|
496
|
-
while (this.#waiting.length > 0) {
|
|
497
|
-
const piece = this.#waiting[0]!
|
|
498
|
-
try {
|
|
499
|
-
const upload = await this.#open()
|
|
500
|
-
await upload.send(piece, false)
|
|
501
|
-
} catch {
|
|
502
|
-
this.#upload = undefined
|
|
503
|
-
this.dropped += this.#held
|
|
504
|
-
this.#waiting = []
|
|
505
|
-
this.#held = 0
|
|
672
|
+
await stream.send(new Uint8Array(0), true)
|
|
673
|
+
for (;;) {
|
|
674
|
+
const event = await stream.next()
|
|
675
|
+
if (event.end) {
|
|
676
|
+
this.#dropped += RecordsDropped.decode(event.body).dropped ?? 0
|
|
506
677
|
return
|
|
507
678
|
}
|
|
508
|
-
this.#waiting.shift()
|
|
509
|
-
this.#held -= piece.length
|
|
510
679
|
}
|
|
511
680
|
}
|
|
681
|
+
}
|
|
682
|
+
|
|
683
|
+
/** A stream of the method, open once its RESPONSE came; `end` leaves the client nothing to send. */
|
|
684
|
+
async function streamOf(
|
|
685
|
+
connection: Connection,
|
|
686
|
+
method: number,
|
|
687
|
+
body: Uint8Array,
|
|
688
|
+
end: boolean,
|
|
689
|
+
): Promise<Stream> {
|
|
690
|
+
const stream = await connection.session.open(method, body, end)
|
|
691
|
+
await stream.next()
|
|
692
|
+
return stream
|
|
693
|
+
}
|
|
694
|
+
|
|
695
|
+
function concat(a: Uint8Array, b: Uint8Array): Uint8Array {
|
|
696
|
+
if (a.length === 0) {
|
|
697
|
+
return b
|
|
698
|
+
}
|
|
699
|
+
const joined = new Uint8Array(a.length + b.length)
|
|
700
|
+
joined.set(a)
|
|
701
|
+
joined.set(b, a.length)
|
|
702
|
+
return joined
|
|
703
|
+
}
|
|
704
|
+
|
|
705
|
+
/** The lines bytes hold: one a newline ends, and one more past the last. */
|
|
706
|
+
function linesIn(bytes: Uint8Array): number {
|
|
707
|
+
let lines = bytes[bytes.length - 1] === newline ? 0 : 1
|
|
708
|
+
for (const byte of bytes) {
|
|
709
|
+
if (byte === newline) {
|
|
710
|
+
lines++
|
|
711
|
+
}
|
|
712
|
+
}
|
|
713
|
+
return lines
|
|
714
|
+
}
|
|
715
|
+
|
|
716
|
+
/** What a store's logger is told: a logger's options, and how long its stream keeps. */
|
|
717
|
+
export interface StoreLoggerOptions extends LoggerOptions {
|
|
718
|
+
/** how long the stream keeps its events, as store.events takes it */
|
|
719
|
+
keep?: Duration
|
|
720
|
+
}
|
|
721
|
+
|
|
722
|
+
/**
|
|
723
|
+
* A logger of `stream` that keeps its lines there through an append of their
|
|
724
|
+
* own, and what it declares its config `log` with.
|
|
725
|
+
*/
|
|
726
|
+
export function storeLogger(
|
|
727
|
+
link: Link,
|
|
728
|
+
stream: string,
|
|
729
|
+
options: StoreLoggerOptions = {},
|
|
730
|
+
): [Logger, LogConfig] {
|
|
731
|
+
const { keep, ...told } = options
|
|
732
|
+
const events = openEvents(link, stream, keep === undefined ? undefined : { keep })
|
|
733
|
+
return newStoreLogger(lines => events.appendLines(lines), stream, told)
|
|
734
|
+
}
|
|
735
|
+
|
|
736
|
+
/** The event an append of a name, its fields and its options writes. */
|
|
737
|
+
function oneEvent(
|
|
738
|
+
name: string,
|
|
739
|
+
fields: Fields | undefined,
|
|
740
|
+
options: AppendOptions | undefined,
|
|
741
|
+
): NewEvent {
|
|
742
|
+
return fields === undefined ? { ...options, name } : { ...options, name, fields }
|
|
743
|
+
}
|
|
744
|
+
|
|
745
|
+
/** An event as the wire carries it, the current trace's ids when it gives none. */
|
|
746
|
+
function newEventOf(event: NewEvent, place: number): Parameters<typeof RecordsNewEvent.encode>[0] {
|
|
747
|
+
const named = `event ${place} ${JSON.stringify(event.name)}`
|
|
748
|
+
if (typeof event.name !== 'string' || event.name === '') {
|
|
749
|
+
throw new InvalidError(`${named}: an event has a name`)
|
|
750
|
+
}
|
|
751
|
+
if (event.at !== undefined && Number.isNaN(event.at.getTime())) {
|
|
752
|
+
throw new InvalidError(`${named}: a time that is no date`)
|
|
753
|
+
}
|
|
754
|
+
const given =
|
|
755
|
+
event.traceId === undefined
|
|
756
|
+
? traceOf(currentTrace())
|
|
757
|
+
: { traceId: event.traceId, spanId: event.spanId }
|
|
758
|
+
return {
|
|
759
|
+
name: event.name,
|
|
760
|
+
at: event.at === undefined ? undefined : nanosOf(event.at),
|
|
761
|
+
level: event.level === undefined ? undefined : levelNumber(event.level, named),
|
|
762
|
+
message: event.message,
|
|
763
|
+
fields:
|
|
764
|
+
event.fields === undefined ? undefined : objectOf(encodeFields(event.fields, undefined)),
|
|
765
|
+
context:
|
|
766
|
+
event.context === undefined ? undefined : objectOf(encodeFields(event.context, undefined)),
|
|
767
|
+
...wireTrace(given.traceId, given.spanId),
|
|
768
|
+
}
|
|
769
|
+
}
|
|
770
|
+
|
|
771
|
+
/** A query as the wire carries it; a condition given as undefined reads nothing it was not asked. */
|
|
772
|
+
function queryOf(query: Query | undefined): Parameters<typeof RecordsQuery.encode>[0] {
|
|
773
|
+
const q = query ?? {}
|
|
774
|
+
for (const [key, value] of Object.entries(q)) {
|
|
775
|
+
if (value === undefined) {
|
|
776
|
+
throw new InvalidError(`a query's ${key} given as undefined: leave it out to read without it`)
|
|
777
|
+
}
|
|
778
|
+
}
|
|
779
|
+
const where = q.where === undefined ? undefined : Object.entries(q.where)
|
|
780
|
+
for (const [key, value] of where ?? []) {
|
|
781
|
+
if (value === undefined) {
|
|
782
|
+
throw new InvalidError(`a query's where ${JSON.stringify(key)} given as undefined`)
|
|
783
|
+
}
|
|
784
|
+
}
|
|
785
|
+
return {
|
|
786
|
+
since: q.since === undefined ? undefined : ms(q.since),
|
|
787
|
+
from: q.from === undefined ? undefined : nanosOf(q.from),
|
|
788
|
+
to: q.to === undefined ? undefined : nanosOf(q.to),
|
|
789
|
+
streams: q.streams === undefined ? undefined : [...q.streams],
|
|
790
|
+
names: q.names === undefined ? undefined : [...q.names],
|
|
791
|
+
minLevel: q.minLevel === undefined ? undefined : levelNumber(q.minLevel, 'minLevel'),
|
|
792
|
+
traceId: q.traceId === undefined ? undefined : idBytes(q.traceId, 16, 'traceId'),
|
|
793
|
+
where:
|
|
794
|
+
where === undefined
|
|
795
|
+
? undefined
|
|
796
|
+
: Object.fromEntries(where.map(([key, value]) => [key, jsonOf(value)])),
|
|
797
|
+
search: q.search,
|
|
798
|
+
newest: q.newest,
|
|
799
|
+
limit: q.limit,
|
|
800
|
+
after: q.after,
|
|
801
|
+
}
|
|
802
|
+
}
|
|
803
|
+
|
|
804
|
+
function eventOf(read: ReturnType<typeof RecordsEvent.decode>): Event {
|
|
805
|
+
const event: Event = {
|
|
806
|
+
at: dateOf(read.at ?? 0n),
|
|
807
|
+
stream: read.stream ?? '',
|
|
808
|
+
name: read.name ?? '',
|
|
809
|
+
fields: read.fields === undefined ? {} : (parseJson(read.fields) as Record<string, unknown>),
|
|
810
|
+
context: read.context === undefined ? {} : (parseJson(read.context) as Record<string, unknown>),
|
|
811
|
+
}
|
|
812
|
+
if (read.level !== undefined) {
|
|
813
|
+
event.level = levelNames[Math.min(5, Math.max(0, Math.floor(read.level / 4) + 2))] as EventLevel
|
|
814
|
+
}
|
|
815
|
+
if (read.message !== undefined) {
|
|
816
|
+
event.message = read.message
|
|
817
|
+
}
|
|
818
|
+
if (read.traceId !== undefined) {
|
|
819
|
+
event.traceId = Buffer.from(read.traceId).toString('hex')
|
|
820
|
+
}
|
|
821
|
+
if (read.spanId !== undefined) {
|
|
822
|
+
event.spanId = Buffer.from(read.spanId).toString('hex')
|
|
823
|
+
}
|
|
824
|
+
return event
|
|
825
|
+
}
|
|
512
826
|
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
827
|
+
/** Key, JSON, key, JSON… as the JSON object they spell; none for no field. */
|
|
828
|
+
function objectOf(flat: readonly string[]): string | undefined {
|
|
829
|
+
if (flat.length === 0) {
|
|
830
|
+
return undefined
|
|
831
|
+
}
|
|
832
|
+
const pairs: string[] = []
|
|
833
|
+
for (let at = 0; at < flat.length; at += 2) {
|
|
834
|
+
pairs.push(`${JSON.stringify(flat[at])}:${flat[at + 1]}`)
|
|
835
|
+
}
|
|
836
|
+
return `{${pairs.join(',')}}`
|
|
837
|
+
}
|
|
838
|
+
|
|
839
|
+
function levelNumber(level: EventLevel | number, what: string): number {
|
|
840
|
+
if (typeof level === 'number') {
|
|
841
|
+
if (!Number.isInteger(level)) {
|
|
842
|
+
throw new InvalidError(`${what}: the level ${level}, where slog counts whole numbers`)
|
|
516
843
|
}
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
844
|
+
return level
|
|
845
|
+
}
|
|
846
|
+
const number = levelNumbers[level]
|
|
847
|
+
if (number === undefined) {
|
|
848
|
+
throw new InvalidError(
|
|
849
|
+
`${what}: the level ${JSON.stringify(level)}, not one of ${levelNames.join(', ')}`,
|
|
522
850
|
)
|
|
523
|
-
return this.#upload
|
|
524
851
|
}
|
|
852
|
+
return number
|
|
853
|
+
}
|
|
525
854
|
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
855
|
+
/** A trace's ids as hex, when the trace has both. */
|
|
856
|
+
function traceOf(trace: Trace | undefined): {
|
|
857
|
+
traceId?: string | undefined
|
|
858
|
+
spanId?: string | undefined
|
|
859
|
+
} {
|
|
860
|
+
if (trace === undefined) {
|
|
861
|
+
return {}
|
|
862
|
+
}
|
|
863
|
+
const hex = (id: Uint8Array | string | undefined) =>
|
|
864
|
+
typeof id === 'string' || id === undefined ? id : Buffer.from(id).toString('hex')
|
|
865
|
+
return { traceId: hex(trace.traceId), spanId: hex(trace.spanId) }
|
|
866
|
+
}
|
|
867
|
+
|
|
868
|
+
/** A trace and its span as the wire carries them: both, or neither, since the store keeps a span with its trace. */
|
|
869
|
+
function wireTrace(
|
|
870
|
+
traceId: string | undefined,
|
|
871
|
+
spanId: string | undefined,
|
|
872
|
+
): { traceId?: Uint8Array; spanId?: Uint8Array } {
|
|
873
|
+
if (traceId === undefined || spanId === undefined) {
|
|
874
|
+
return {}
|
|
875
|
+
}
|
|
876
|
+
return { traceId: idBytes(traceId, 16, 'traceId'), spanId: idBytes(spanId, 8, 'spanId') }
|
|
877
|
+
}
|
|
878
|
+
|
|
879
|
+
function idBytes(hex: string, size: number, what: string): Uint8Array {
|
|
880
|
+
if (!new RegExp(`^[0-9a-fA-F]{${2 * size}}$`).test(hex)) {
|
|
881
|
+
throw new InvalidError(`${what} ${JSON.stringify(hex)}: ${2 * size} hex digits`)
|
|
882
|
+
}
|
|
883
|
+
return Uint8Array.from(Buffer.from(hex, 'hex'))
|
|
884
|
+
}
|
|
885
|
+
|
|
886
|
+
function nanosOf(date: Date): bigint {
|
|
887
|
+
return BigInt(date.getTime()) * 1_000_000n
|
|
888
|
+
}
|
|
889
|
+
|
|
890
|
+
function dateOf(nanos: bigint): Date {
|
|
891
|
+
return new Date(Number(nanos / 1_000_000n))
|
|
892
|
+
}
|
|
893
|
+
|
|
894
|
+
function mapValues<T, U>(
|
|
895
|
+
map: Record<string, T> | undefined,
|
|
896
|
+
read: (value: T) => U,
|
|
897
|
+
): Record<string, U> {
|
|
898
|
+
return Object.fromEntries(Object.entries(map ?? {}).map(([key, value]) => [key, read(value)]))
|
|
899
|
+
}
|
|
900
|
+
|
|
901
|
+
/** Milliseconds as a span is written: '30d', '12h', '90s', '250ms'. */
|
|
902
|
+
function spelled(millis: number): DurationText {
|
|
903
|
+
for (const [unit, size] of [
|
|
904
|
+
['d', 86_400_000],
|
|
905
|
+
['h', 3_600_000],
|
|
906
|
+
['m', 60_000],
|
|
907
|
+
['s', 1000],
|
|
908
|
+
] as const) {
|
|
909
|
+
if (millis >= size && millis % size === 0) {
|
|
910
|
+
return `${millis / size}${unit}` as DurationText
|
|
533
911
|
}
|
|
534
|
-
await upload.send(new Uint8Array(0), true)
|
|
535
|
-
await upload.next()
|
|
536
912
|
}
|
|
913
|
+
return `${millis}ms` as DurationText
|
|
537
914
|
}
|