@tinyshed/tinystore 0.1.0-rc.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +262 -0
- package/bin/tinystore.js +50 -0
- package/dist/background.d.ts +30 -0
- package/dist/background.js +93 -0
- package/dist/blobs.d.ts +98 -0
- package/dist/blobs.js +299 -0
- package/dist/cancel.d.ts +14 -0
- package/dist/cancel.js +22 -0
- package/dist/clock.d.ts +21 -0
- package/dist/clock.js +35 -0
- package/dist/config.d.ts +60 -0
- package/dist/config.js +417 -0
- package/dist/connection.d.ts +78 -0
- package/dist/connection.js +405 -0
- package/dist/console.d.ts +48 -0
- package/dist/console.js +195 -0
- package/dist/errors.d.ts +103 -0
- package/dist/errors.js +158 -0
- package/dist/handles.d.ts +28 -0
- package/dist/handles.js +37 -0
- package/dist/index.d.ts +21 -0
- package/dist/index.js +7 -0
- package/dist/jobs.d.ts +304 -0
- package/dist/jobs.js +684 -0
- package/dist/kv.d.ts +283 -0
- package/dist/kv.js +815 -0
- package/dist/limiter.d.ts +45 -0
- package/dist/limiter.js +68 -0
- package/dist/logger.d.ts +96 -0
- package/dist/logger.js +243 -0
- package/dist/metrics.d.ts +249 -0
- package/dist/metrics.js +610 -0
- package/dist/once.d.ts +31 -0
- package/dist/once.js +116 -0
- package/dist/quota.d.ts +44 -0
- package/dist/quota.js +98 -0
- package/dist/records.d.ts +182 -0
- package/dist/records.js +346 -0
- package/dist/runtime/bun.d.ts +2 -0
- package/dist/runtime/bun.js +164 -0
- package/dist/runtime/node.d.ts +2 -0
- package/dist/runtime/node.js +157 -0
- package/dist/runtime/shared.d.ts +25 -0
- package/dist/runtime/shared.js +45 -0
- package/dist/runtime.d.ts +48 -0
- package/dist/runtime.js +4 -0
- package/dist/schema.d.ts +39 -0
- package/dist/schema.js +21 -0
- package/dist/session.d.ts +120 -0
- package/dist/session.js +546 -0
- package/dist/sql.d.ts +121 -0
- package/dist/sql.js +342 -0
- package/dist/store.d.ts +103 -0
- package/dist/store.js +153 -0
- package/dist/time.d.ts +35 -0
- package/dist/time.js +44 -0
- package/dist/trace.d.ts +18 -0
- package/dist/trace.js +22 -0
- package/dist/wire/codec.d.ts +90 -0
- package/dist/wire/codec.js +383 -0
- package/dist/wire/frame.d.ts +41 -0
- package/dist/wire/frame.js +102 -0
- package/dist/wire/messages.d.ts +1403 -0
- package/dist/wire/messages.js +472 -0
- package/dist/wire/msgpack.d.ts +93 -0
- package/dist/wire/msgpack.js +680 -0
- package/package.json +40 -0
- package/src/background.ts +108 -0
- package/src/blobs.ts +437 -0
- package/src/cancel.ts +26 -0
- package/src/clock.ts +45 -0
- package/src/config.ts +501 -0
- package/src/connection.ts +512 -0
- package/src/console.ts +228 -0
- package/src/errors.ts +211 -0
- package/src/handles.ts +68 -0
- package/src/index.ts +92 -0
- package/src/jobs.ts +963 -0
- package/src/kv.ts +1093 -0
- package/src/limiter.ts +99 -0
- package/src/logger.ts +326 -0
- package/src/metrics.ts +896 -0
- package/src/once.ts +142 -0
- package/src/quota.ts +137 -0
- package/src/records.ts +537 -0
- package/src/runtime/bun.ts +190 -0
- package/src/runtime/node.ts +177 -0
- package/src/runtime/shared.ts +61 -0
- package/src/runtime.ts +57 -0
- package/src/schema.ts +69 -0
- package/src/session.ts +665 -0
- package/src/sql.ts +478 -0
- package/src/store.ts +219 -0
- package/src/time.ts +87 -0
- package/src/trace.ts +32 -0
- package/src/wire/codec.ts +445 -0
- package/src/wire/frame.ts +139 -0
- package/src/wire/messages.ts +562 -0
- package/src/wire/msgpack.ts +712 -0
package/src/records.ts
ADDED
|
@@ -0,0 +1,537 @@
|
|
|
1
|
+
// The application's logs and events in records.db, as Go's records package
|
|
2
|
+
// keeps them: one log of the store's whose calls name their streams, read by
|
|
3
|
+
// time and by what records hold, followed in the order they were sealed.
|
|
4
|
+
|
|
5
|
+
import { type Connection, download, type Link } from './connection.ts'
|
|
6
|
+
import type { ConsoleLine } from './console.ts'
|
|
7
|
+
import { InvalidError } from './errors.ts'
|
|
8
|
+
import type { Page } from './handles.ts'
|
|
9
|
+
import { type Logger, type LoggerOptions, newLogger } from './logger.ts'
|
|
10
|
+
import type { Stream } from './session.ts'
|
|
11
|
+
import { type Duration, ms } from './time.ts'
|
|
12
|
+
import { currentTrace } from './trace.ts'
|
|
13
|
+
import {
|
|
14
|
+
Empty,
|
|
15
|
+
methods,
|
|
16
|
+
RecordsQuery as QueryMessage,
|
|
17
|
+
RecordsBatch,
|
|
18
|
+
RecordsCursor,
|
|
19
|
+
RecordsDamage,
|
|
20
|
+
RecordsDamages,
|
|
21
|
+
RecordsPage,
|
|
22
|
+
RecordsRecord,
|
|
23
|
+
RecordsStream,
|
|
24
|
+
} from './wire/messages.ts'
|
|
25
|
+
|
|
26
|
+
/** slog's levels: -4 debug, 0 info, 4 warn, 8 error, or any other it may name. */
|
|
27
|
+
export type Level = 'debug' | 'info' | 'warn' | 'error' | number
|
|
28
|
+
|
|
29
|
+
const levels = { debug: -4, info: 0, warn: 4, error: 8 } as const
|
|
30
|
+
|
|
31
|
+
function levelOf(level: Level): number {
|
|
32
|
+
return typeof level === 'number' ? level : levels[level]
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* Fields: keys and values, each value written as JSON. An object's keys keep
|
|
37
|
+
* their order; an array of pairs may repeat a key.
|
|
38
|
+
*/
|
|
39
|
+
export type Fields = Record<string, unknown> | readonly (readonly [string, unknown])[]
|
|
40
|
+
|
|
41
|
+
/** A record to append: when, where, what, and its fields. */
|
|
42
|
+
export interface RecordInput {
|
|
43
|
+
/** its time: a Date, or unix nanoseconds as a bigint; now when absent */
|
|
44
|
+
at?: Date | bigint
|
|
45
|
+
/** the namespace the application names: 'web', 'billing' */
|
|
46
|
+
stream: string
|
|
47
|
+
/** the event: 'log' for a log line */
|
|
48
|
+
name: string
|
|
49
|
+
level?: Level
|
|
50
|
+
/** absent is none, which an empty body is not */
|
|
51
|
+
body?: string | Uint8Array
|
|
52
|
+
/** sixteen bytes, or their 32 hex digits */
|
|
53
|
+
traceId?: Uint8Array | string | undefined
|
|
54
|
+
/** eight bytes, or their 16 hex digits */
|
|
55
|
+
spanId?: Uint8Array | string | undefined
|
|
56
|
+
/** who produced it */
|
|
57
|
+
context?: Fields
|
|
58
|
+
/** what happened */
|
|
59
|
+
attrs?: Fields
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/** A record as it was kept: a field's value is the JSON it was written as, spelled as it was given. */
|
|
63
|
+
export interface LogRecord {
|
|
64
|
+
/** unix nanoseconds */
|
|
65
|
+
at: bigint
|
|
66
|
+
stream: string
|
|
67
|
+
name: string
|
|
68
|
+
level: number | undefined
|
|
69
|
+
body: string | Uint8Array | undefined
|
|
70
|
+
traceId: Uint8Array | undefined
|
|
71
|
+
spanId: Uint8Array | undefined
|
|
72
|
+
context: [key: string, json: string | Uint8Array][]
|
|
73
|
+
attrs: [key: string, json: string | Uint8Array][]
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* A record's fields, its attrs or its context, as an object, each value read
|
|
78
|
+
* as JSON.parse reads the JSON it was kept as; the last of a repeated key is
|
|
79
|
+
* kept. The pairs keep what JSON.parse may not: 1.2300 as written, an integer
|
|
80
|
+
* past 2^53 whole.
|
|
81
|
+
*
|
|
82
|
+
* ```ts
|
|
83
|
+
* for await (const record of store.records.all({ since: '1h', streams: ['readers'] })) {
|
|
84
|
+
* const { page, from } = fields(record.attrs)
|
|
85
|
+
* }
|
|
86
|
+
* ```
|
|
87
|
+
*/
|
|
88
|
+
export function fields(pairs: LogRecord['attrs']): Record<string, unknown> {
|
|
89
|
+
// fromEntries, since an assignment to __proto__ would set the prototype
|
|
90
|
+
return Object.fromEntries(pairs.map(([key, json]) => [key, JSON.parse(textOf(json))]))
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/**
|
|
94
|
+
* The records in a range that meet every condition given.
|
|
95
|
+
*
|
|
96
|
+
* ```ts
|
|
97
|
+
* { since: '1h', minLevel: 'warn' } // the last hour's warnings
|
|
98
|
+
* { since: '1h', search: 'connection reset' } // whose text holds it, case ignored
|
|
99
|
+
* { from: start, to: end, streams: ['api'] }
|
|
100
|
+
* ```
|
|
101
|
+
*/
|
|
102
|
+
export interface RecordsQuery {
|
|
103
|
+
/** the span before now the range covers: '1h', '15m', or milliseconds; or from and to */
|
|
104
|
+
since?: Duration
|
|
105
|
+
/** the range's start, included; open when absent */
|
|
106
|
+
from?: Date | bigint
|
|
107
|
+
/** the range's end, excluded; open when absent */
|
|
108
|
+
to?: Date | bigint
|
|
109
|
+
/** none or empty is every stream */
|
|
110
|
+
streams?: string[]
|
|
111
|
+
names?: string[]
|
|
112
|
+
/** a record without a level does not match */
|
|
113
|
+
minLevel?: Level
|
|
114
|
+
traceId?: Uint8Array | string
|
|
115
|
+
/** each a record must hold, by key and exact JSON spelling */
|
|
116
|
+
attrs?: Fields
|
|
117
|
+
context?: Fields
|
|
118
|
+
/**
|
|
119
|
+
* text a record's body or name holds, its case ignored: every record the
|
|
120
|
+
* rest of the query leaves is read for it, so a search over much text ends
|
|
121
|
+
* pages early at the budget, and next goes on
|
|
122
|
+
*/
|
|
123
|
+
search?: string
|
|
124
|
+
/** newest first; oldest first when absent */
|
|
125
|
+
newest?: boolean
|
|
126
|
+
/** the records a page holds: 1000, and 10000 at most */
|
|
127
|
+
limit?: number
|
|
128
|
+
/** what one read may open, fetch and decode; each narrows the server's */
|
|
129
|
+
budget?: { blocks?: number; bytes?: number; records?: number }
|
|
130
|
+
/** a page's next, which this page continues from; the rest of the query as it was */
|
|
131
|
+
after?: string | undefined
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
/** Where a follower stands in the sealed segments: kept by the caller between follows. */
|
|
135
|
+
export interface Cursor {
|
|
136
|
+
segment: number
|
|
137
|
+
row: number
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
/** A row that no longer reads, which a drop removes. */
|
|
141
|
+
export interface Damage {
|
|
142
|
+
stream: string
|
|
143
|
+
/** a sealed segment, dropped whole; or a head row */
|
|
144
|
+
segment: number | undefined
|
|
145
|
+
headRow: number | undefined
|
|
146
|
+
/** the times it held, unix nanoseconds */
|
|
147
|
+
from: bigint
|
|
148
|
+
to: bigint
|
|
149
|
+
/** the invariant its bytes broke */
|
|
150
|
+
reason: string
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
const nsPerMs = 1_000_000n
|
|
154
|
+
|
|
155
|
+
function nanos(t: Date | bigint): bigint {
|
|
156
|
+
return typeof t === 'bigint' ? t : BigInt(t.getTime()) * nsPerMs
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
function idOf(
|
|
160
|
+
id: Uint8Array | string | undefined,
|
|
161
|
+
bytes: number,
|
|
162
|
+
what: string,
|
|
163
|
+
): Uint8Array | undefined {
|
|
164
|
+
if (id === undefined) {
|
|
165
|
+
return undefined
|
|
166
|
+
}
|
|
167
|
+
const raw = typeof id === 'string' ? new Uint8Array(Buffer.from(id, 'hex')) : id
|
|
168
|
+
if (raw.length !== bytes) {
|
|
169
|
+
throw new InvalidError(`a ${what} id of ${raw.length} bytes, not ${bytes}`)
|
|
170
|
+
}
|
|
171
|
+
return raw
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
/** Fields as the wire carries them: one array of keys and JSON texts. */
|
|
175
|
+
function fieldsOf(fields: Fields | undefined): string[] | undefined {
|
|
176
|
+
if (fields === undefined) {
|
|
177
|
+
return undefined
|
|
178
|
+
}
|
|
179
|
+
const pairs = Array.isArray(fields) ? fields : Object.entries(fields)
|
|
180
|
+
const flat: string[] = []
|
|
181
|
+
for (const [key, value] of pairs) {
|
|
182
|
+
const json = JSON.stringify(value)
|
|
183
|
+
if (json === undefined) {
|
|
184
|
+
throw new InvalidError(`the field ${key} holds a value JSON cannot write`)
|
|
185
|
+
}
|
|
186
|
+
flat.push(key, json)
|
|
187
|
+
}
|
|
188
|
+
return flat.length === 0 ? undefined : flat
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
const lossy = new TextDecoder()
|
|
192
|
+
|
|
193
|
+
function textOf(text: string | Uint8Array | undefined): string {
|
|
194
|
+
return text === undefined ? '' : typeof text === 'string' ? text : lossy.decode(text)
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
function pairs(flat: (string | Uint8Array)[] | undefined): [string, string | Uint8Array][] {
|
|
198
|
+
const out: [string, string | Uint8Array][] = []
|
|
199
|
+
for (let i = 0; flat !== undefined && i + 1 < flat.length; i += 2) {
|
|
200
|
+
out.push([textOf(flat[i]), flat[i + 1]!])
|
|
201
|
+
}
|
|
202
|
+
return out
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
function recordOf(r: ReturnType<typeof RecordsRecord.decode>): LogRecord {
|
|
206
|
+
return {
|
|
207
|
+
at: r.at ?? 0n,
|
|
208
|
+
stream: textOf(r.stream),
|
|
209
|
+
name: textOf(r.name),
|
|
210
|
+
level: r.level,
|
|
211
|
+
body: r.body,
|
|
212
|
+
traceId: r.traceId,
|
|
213
|
+
spanId: r.spanId,
|
|
214
|
+
context: pairs(r.context),
|
|
215
|
+
attrs: pairs(r.attrs),
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
/**
|
|
220
|
+
* A page's next: where the range moved, both ends as nanoseconds, so that a
|
|
221
|
+
* query over the last `since` continues the range it started with.
|
|
222
|
+
*
|
|
223
|
+
* from 1700000000000000000, open end → '1700000000000000000:'
|
|
224
|
+
*/
|
|
225
|
+
function cursorOf(from: bigint | undefined, to: bigint | undefined): string {
|
|
226
|
+
return `${from ?? ''}:${to ?? ''}`
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
function rangeOf(q: RecordsQuery | undefined): {
|
|
230
|
+
from: bigint | undefined
|
|
231
|
+
to: bigint | undefined
|
|
232
|
+
} {
|
|
233
|
+
if (q?.after !== undefined) {
|
|
234
|
+
const ends = /^(-?\d*):(-?\d*)$/.exec(q.after)
|
|
235
|
+
if (ends === null) {
|
|
236
|
+
throw new InvalidError(`${JSON.stringify(q.after)} is no page's next`)
|
|
237
|
+
}
|
|
238
|
+
return {
|
|
239
|
+
from: ends[1] === '' ? undefined : BigInt(ends[1]!),
|
|
240
|
+
to: ends[2] === '' ? undefined : BigInt(ends[2]!),
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
if (q?.since !== undefined && q.from !== undefined) {
|
|
244
|
+
throw new InvalidError('a range starts since a span before now or from a time, not both')
|
|
245
|
+
}
|
|
246
|
+
const from =
|
|
247
|
+
q?.since !== undefined
|
|
248
|
+
? nanos(new Date(Date.now() - ms(q.since)))
|
|
249
|
+
: q?.from === undefined
|
|
250
|
+
? undefined
|
|
251
|
+
: nanos(q.from)
|
|
252
|
+
return { from, to: q?.to === undefined ? undefined : nanos(q.to) }
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
function queryOf(q: RecordsQuery | undefined): Parameters<typeof QueryMessage.encode>[0] {
|
|
256
|
+
return {
|
|
257
|
+
...rangeOf(q),
|
|
258
|
+
streams: q?.streams !== undefined && q.streams.length > 0 ? q.streams : undefined,
|
|
259
|
+
names: q?.names !== undefined && q.names.length > 0 ? q.names : undefined,
|
|
260
|
+
minLevel: q?.minLevel === undefined ? undefined : levelOf(q.minLevel),
|
|
261
|
+
traceId: idOf(q?.traceId, 16, 'trace'),
|
|
262
|
+
attrs: fieldsOf(q?.attrs),
|
|
263
|
+
context: fieldsOf(q?.context),
|
|
264
|
+
newest: q?.newest === true ? true : undefined,
|
|
265
|
+
limit: q?.limit,
|
|
266
|
+
budgetBlocks: q?.budget?.blocks,
|
|
267
|
+
budgetBytes: q?.budget?.bytes,
|
|
268
|
+
budgetRecords: q?.budget?.records,
|
|
269
|
+
search: q?.search === '' ? undefined : q?.search,
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
export class Records {
|
|
274
|
+
readonly #link: Link
|
|
275
|
+
readonly #loggers: Logger[] = []
|
|
276
|
+
|
|
277
|
+
constructor(link: Link) {
|
|
278
|
+
this.#link = link
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
/**
|
|
282
|
+
* A logger of a stream: its calls return at once, and its lines reach the
|
|
283
|
+
* server every second, as Go's slog handler's and Python's logging.Handler's do.
|
|
284
|
+
*/
|
|
285
|
+
logger(stream: string, options?: LoggerOptions): Logger {
|
|
286
|
+
const logger = newLogger(lines => this.#writeLines(lines), stream, options)
|
|
287
|
+
this.#loggers.push(logger)
|
|
288
|
+
return logger
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
/** Hands over what every logger holds, as the store does when it closes. */
|
|
292
|
+
async stop(): Promise<void> {
|
|
293
|
+
await Promise.all(this.#loggers.map(logger => logger.stop()))
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
/**
|
|
297
|
+
* Appends records in one transaction, all or none; a refused one names
|
|
298
|
+
* itself as `call`, with its stream and name. A read sees them at once. A
|
|
299
|
+
* record of no trace of its own takes the one withTrace runs the call in.
|
|
300
|
+
*/
|
|
301
|
+
async append(records: RecordInput | readonly RecordInput[]): Promise<void> {
|
|
302
|
+
const carried = currentTrace()
|
|
303
|
+
const list = Array.isArray(records) ? records : [records as RecordInput]
|
|
304
|
+
return this.#write(
|
|
305
|
+
carried === undefined
|
|
306
|
+
? list
|
|
307
|
+
: list.map(r =>
|
|
308
|
+
r.traceId === undefined
|
|
309
|
+
? { ...r, traceId: carried.traceId, spanId: carried.spanId }
|
|
310
|
+
: r,
|
|
311
|
+
),
|
|
312
|
+
)
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
async #write(records: readonly RecordInput[]): Promise<void> {
|
|
316
|
+
const now = BigInt(Date.now()) * nsPerMs
|
|
317
|
+
await this.#send(
|
|
318
|
+
records.map(r => ({
|
|
319
|
+
at: r.at === undefined ? now : nanos(r.at),
|
|
320
|
+
stream: r.stream,
|
|
321
|
+
name: r.name,
|
|
322
|
+
level: r.level === undefined ? undefined : levelOf(r.level),
|
|
323
|
+
body: r.body,
|
|
324
|
+
traceId: idOf(r.traceId, 16, 'trace'),
|
|
325
|
+
spanId: idOf(r.spanId, 8, 'span'),
|
|
326
|
+
context: fieldsOf(r.context),
|
|
327
|
+
attrs: fieldsOf(r.attrs),
|
|
328
|
+
})),
|
|
329
|
+
)
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
/** Appends a logger's lines, their fields already JSON and their traces taken as each was logged. */
|
|
333
|
+
async #writeLines(lines: readonly ConsoleLine[]): Promise<void> {
|
|
334
|
+
await this.#send(
|
|
335
|
+
lines.map(line => ({
|
|
336
|
+
at: nanos(line.at),
|
|
337
|
+
stream: line.stream,
|
|
338
|
+
name: line.event ?? 'log',
|
|
339
|
+
level: line.level,
|
|
340
|
+
body: line.msg,
|
|
341
|
+
traceId: idOf(line.traceId, 16, 'trace'),
|
|
342
|
+
spanId: idOf(line.spanId, 8, 'span'),
|
|
343
|
+
context: line.context.length === 0 ? undefined : [...line.context],
|
|
344
|
+
attrs: line.attrs.length === 0 ? undefined : [...line.attrs],
|
|
345
|
+
})),
|
|
346
|
+
)
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
async #send(records: Parameters<typeof RecordsBatch.encode>[0]['records']): Promise<void> {
|
|
350
|
+
await this.#link.run('write', connection =>
|
|
351
|
+
connection.session.call(methods['records.append'], RecordsBatch.encode({ records })),
|
|
352
|
+
)
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
/**
|
|
356
|
+
* One page of the records a query matches, from one snapshot, in event-time
|
|
357
|
+
* order; next, while a limit or a budget ended the page early, is passed back
|
|
358
|
+
* as `after` with the same query. A page never splits a timestamp.
|
|
359
|
+
*/
|
|
360
|
+
async scan(query?: RecordsQuery): Promise<Page<LogRecord, string>> {
|
|
361
|
+
const asked = queryOf(query)
|
|
362
|
+
const got = await this.#link.run('read', connection =>
|
|
363
|
+
download(connection, methods['records.read'], QueryMessage.encode(asked)),
|
|
364
|
+
)
|
|
365
|
+
const page = RecordsPage.decode(got.trailer)
|
|
366
|
+
const items = got.items.map(item => recordOf(RecordsRecord.decode(item)))
|
|
367
|
+
if (page.more !== true) {
|
|
368
|
+
return { items, next: undefined }
|
|
369
|
+
}
|
|
370
|
+
const moved = (end: number | bigint | undefined, sent: number | bigint | undefined) =>
|
|
371
|
+
end !== undefined ? BigInt(end) : sent !== undefined ? BigInt(sent) : undefined
|
|
372
|
+
return { items, next: cursorOf(moved(page.from, asked.from), moved(page.to, asked.to)) }
|
|
373
|
+
}
|
|
374
|
+
|
|
375
|
+
/** Every record a query matches, a page at a time. */
|
|
376
|
+
async *all(query?: Omit<RecordsQuery, 'after'>): AsyncGenerator<LogRecord> {
|
|
377
|
+
let page = await this.scan(query)
|
|
378
|
+
yield* page.items
|
|
379
|
+
while (page.next !== undefined) {
|
|
380
|
+
page = await this.scan({ ...query, after: page.next })
|
|
381
|
+
yield* page.items
|
|
382
|
+
}
|
|
383
|
+
}
|
|
384
|
+
|
|
385
|
+
/**
|
|
386
|
+
* The sealed records after a cursor, in the order they were sealed; the
|
|
387
|
+
* cursor to follow from next, and the segments retention removed before
|
|
388
|
+
* the cursor reached them. A record reaches a follower once it is sealed.
|
|
389
|
+
*/
|
|
390
|
+
async follow(
|
|
391
|
+
cursor?: Cursor,
|
|
392
|
+
limit?: number,
|
|
393
|
+
): Promise<{ records: LogRecord[]; cursor: Cursor; expired: number }> {
|
|
394
|
+
const got = await this.#link.run('read', connection =>
|
|
395
|
+
download(
|
|
396
|
+
connection,
|
|
397
|
+
methods['records.follow'],
|
|
398
|
+
RecordsCursor.encode({ segment: cursor?.segment, row: cursor?.row, limit }),
|
|
399
|
+
),
|
|
400
|
+
)
|
|
401
|
+
const trailer = RecordsCursor.decode(got.trailer)
|
|
402
|
+
return {
|
|
403
|
+
records: got.items.map(item => recordOf(RecordsRecord.decode(item))),
|
|
404
|
+
cursor: { segment: trailer.segment ?? 0, row: trailer.row ?? 0 },
|
|
405
|
+
expired: trailer.expired ?? 0,
|
|
406
|
+
}
|
|
407
|
+
}
|
|
408
|
+
|
|
409
|
+
/**
|
|
410
|
+
* A writer of another program's output, pino's lines or a child's stdout,
|
|
411
|
+
* that never waits: each line becomes a record of the stream as the server's
|
|
412
|
+
* Lines makes it, a stack trace's lines joined and a JSON or logfmt line's
|
|
413
|
+
* fields kept. What does not fit its buffer is dropped and counted.
|
|
414
|
+
*/
|
|
415
|
+
lines(stream: string, options?: { buffer?: number }): LinesWriter {
|
|
416
|
+
if (stream === '') {
|
|
417
|
+
throw new InvalidError('lines of no stream')
|
|
418
|
+
}
|
|
419
|
+
return new LinesWriter(this.#link, stream, options?.buffer ?? 1 << 20)
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
/** The rows the server's records have met that no longer read. */
|
|
423
|
+
async damaged(): Promise<Damage[]> {
|
|
424
|
+
const body = await this.#link.run('read', connection =>
|
|
425
|
+
connection.session.call(methods['records.damaged'], Empty.encode({})),
|
|
426
|
+
)
|
|
427
|
+
return (RecordsDamages.decode(body).damages ?? []).map(d => ({
|
|
428
|
+
stream: textOf(d.stream),
|
|
429
|
+
segment: d.segment,
|
|
430
|
+
headRow: d.headRow,
|
|
431
|
+
from: d.from ?? 0n,
|
|
432
|
+
to: d.to ?? 0n,
|
|
433
|
+
reason: d.reason ?? '',
|
|
434
|
+
}))
|
|
435
|
+
}
|
|
436
|
+
|
|
437
|
+
/** Removes a damaged row, a repair an admin connection alone may make. */
|
|
438
|
+
async drop(damage: Damage): Promise<void> {
|
|
439
|
+
await this.#link.run('write', connection =>
|
|
440
|
+
connection.session.call(methods['records.drop'], RecordsDamage.encode(damage)),
|
|
441
|
+
)
|
|
442
|
+
}
|
|
443
|
+
}
|
|
444
|
+
|
|
445
|
+
/** The most one DATA of lines carries. */
|
|
446
|
+
const linesChunk = 64 << 10
|
|
447
|
+
|
|
448
|
+
/**
|
|
449
|
+
* An upload of lines that stays open while it is written: write never waits,
|
|
450
|
+
* holding what the stream's credit does not let go yet up to its buffer and
|
|
451
|
+
* dropping past it. A connection lost drops what it held and the next write
|
|
452
|
+
* uploads on the next connection.
|
|
453
|
+
*/
|
|
454
|
+
export class LinesWriter {
|
|
455
|
+
readonly #link: Link
|
|
456
|
+
readonly #stream: string
|
|
457
|
+
readonly #most: number
|
|
458
|
+
#waiting: Uint8Array[] = []
|
|
459
|
+
#held = 0
|
|
460
|
+
#pumping: Promise<void> | undefined
|
|
461
|
+
#upload: Stream | undefined
|
|
462
|
+
#ended = false
|
|
463
|
+
/** bytes dropped since the writer began: a full buffer, or a lost connection */
|
|
464
|
+
dropped = 0
|
|
465
|
+
|
|
466
|
+
constructor(link: Link, stream: string, most: number) {
|
|
467
|
+
this.#link = link
|
|
468
|
+
this.#stream = stream
|
|
469
|
+
this.#most = most
|
|
470
|
+
}
|
|
471
|
+
|
|
472
|
+
/** Takes a piece of output, cut anywhere. It returns at once. */
|
|
473
|
+
write(chunk: string | Uint8Array): boolean {
|
|
474
|
+
if (this.#ended) {
|
|
475
|
+
throw new InvalidError('a write after the lines ended')
|
|
476
|
+
}
|
|
477
|
+
const bytes = typeof chunk === 'string' ? new TextEncoder().encode(chunk) : chunk
|
|
478
|
+
if (bytes.length === 0) {
|
|
479
|
+
return true
|
|
480
|
+
}
|
|
481
|
+
if (this.#held + bytes.length > this.#most) {
|
|
482
|
+
this.dropped += bytes.length
|
|
483
|
+
return false
|
|
484
|
+
}
|
|
485
|
+
for (let at = 0; at < bytes.length; at += linesChunk) {
|
|
486
|
+
this.#waiting.push(bytes.subarray(at, at + linesChunk))
|
|
487
|
+
}
|
|
488
|
+
this.#held += bytes.length
|
|
489
|
+
this.#pumping ??= this.#pump().finally(() => {
|
|
490
|
+
this.#pumping = undefined
|
|
491
|
+
})
|
|
492
|
+
return true
|
|
493
|
+
}
|
|
494
|
+
|
|
495
|
+
async #pump(): Promise<void> {
|
|
496
|
+
while (this.#waiting.length > 0) {
|
|
497
|
+
const piece = this.#waiting[0]!
|
|
498
|
+
try {
|
|
499
|
+
const upload = await this.#open()
|
|
500
|
+
await upload.send(piece, false)
|
|
501
|
+
} catch {
|
|
502
|
+
this.#upload = undefined
|
|
503
|
+
this.dropped += this.#held
|
|
504
|
+
this.#waiting = []
|
|
505
|
+
this.#held = 0
|
|
506
|
+
return
|
|
507
|
+
}
|
|
508
|
+
this.#waiting.shift()
|
|
509
|
+
this.#held -= piece.length
|
|
510
|
+
}
|
|
511
|
+
}
|
|
512
|
+
|
|
513
|
+
async #open(): Promise<Stream> {
|
|
514
|
+
if (this.#upload !== undefined && !this.#upload.finished) {
|
|
515
|
+
return this.#upload
|
|
516
|
+
}
|
|
517
|
+
const connection: Connection = await this.#link.connection()
|
|
518
|
+
this.#upload = await connection.session.open(
|
|
519
|
+
methods['records.lines'],
|
|
520
|
+
RecordsStream.encode({ stream: this.#stream }),
|
|
521
|
+
false,
|
|
522
|
+
)
|
|
523
|
+
return this.#upload
|
|
524
|
+
}
|
|
525
|
+
|
|
526
|
+
/** Hands over what the writer holds, a line cut short included, and ends the upload. */
|
|
527
|
+
async end(): Promise<void> {
|
|
528
|
+
this.#ended = true
|
|
529
|
+
await this.#pumping
|
|
530
|
+
const upload = this.#upload
|
|
531
|
+
if (upload === undefined || upload.finished) {
|
|
532
|
+
return
|
|
533
|
+
}
|
|
534
|
+
await upload.send(new Uint8Array(0), true)
|
|
535
|
+
await upload.next()
|
|
536
|
+
}
|
|
537
|
+
}
|