@tinyshed/tinystore 0.1.0 → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. package/dist/blobs.d.ts +1 -1
  2. package/dist/blobs.js +13 -5
  3. package/dist/cancel.d.ts +16 -2
  4. package/dist/cancel.js +36 -2
  5. package/dist/channels.js +20 -6
  6. package/dist/config.d.ts +49 -1
  7. package/dist/config.js +86 -7
  8. package/dist/connection.d.ts +52 -10
  9. package/dist/connection.js +305 -73
  10. package/dist/handles.d.ts +10 -1
  11. package/dist/handles.js +27 -4
  12. package/dist/index.d.ts +4 -4
  13. package/dist/jobs.d.ts +33 -1
  14. package/dist/jobs.js +45 -0
  15. package/dist/logger.d.ts +22 -10
  16. package/dist/logger.js +118 -28
  17. package/dist/metrics.d.ts +19 -0
  18. package/dist/metrics.js +22 -1
  19. package/dist/query.js +33 -9
  20. package/dist/records.d.ts +30 -2
  21. package/dist/records.js +8 -0
  22. package/dist/runtime/addon.js +3 -2
  23. package/dist/runtime/bun.js +33 -12
  24. package/dist/runtime/node.js +30 -8
  25. package/dist/runtime/pipe.d.ts +2 -0
  26. package/dist/runtime/pipe.js +45 -20
  27. package/dist/runtime/shared.d.ts +28 -4
  28. package/dist/runtime/shared.js +125 -13
  29. package/dist/runtime.d.ts +28 -6
  30. package/dist/session.d.ts +9 -5
  31. package/dist/session.js +113 -31
  32. package/dist/sql.d.ts +18 -5
  33. package/dist/sql.js +44 -11
  34. package/dist/store.d.ts +25 -11
  35. package/dist/store.js +67 -9
  36. package/dist/wire/codec.d.ts +5 -0
  37. package/dist/wire/codec.js +10 -0
  38. package/dist/wire/frame.d.ts +2 -0
  39. package/dist/wire/frame.js +4 -0
  40. package/dist/wire/msgpack.d.ts +7 -3
  41. package/dist/wire/msgpack.js +11 -4
  42. package/dist/wire/protocol.d.ts +246 -36
  43. package/dist/wire/protocol.js +387 -101
  44. package/package.json +9 -8
  45. package/src/blobs.ts +13 -5
  46. package/src/cancel.ts +44 -2
  47. package/src/channels.ts +21 -6
  48. package/src/config.ts +138 -12
  49. package/src/connection.ts +377 -77
  50. package/src/handles.ts +32 -4
  51. package/src/index.ts +5 -0
  52. package/src/jobs.ts +63 -0
  53. package/src/logger.ts +158 -40
  54. package/src/metrics.ts +46 -1
  55. package/src/query.ts +33 -13
  56. package/src/records.ts +40 -2
  57. package/src/runtime/addon.ts +3 -2
  58. package/src/runtime/bun.ts +44 -9
  59. package/src/runtime/node.ts +30 -5
  60. package/src/runtime/pipe.ts +49 -19
  61. package/src/runtime/shared.ts +137 -16
  62. package/src/runtime.ts +31 -6
  63. package/src/session.ts +129 -30
  64. package/src/sql.ts +48 -14
  65. package/src/store.ts +81 -14
  66. package/src/wire/codec.ts +11 -0
  67. package/src/wire/frame.ts +5 -0
  68. package/src/wire/msgpack.ts +11 -4
  69. package/src/wire/protocol.ts +404 -103
package/src/handles.ts CHANGED
@@ -1,5 +1,6 @@
1
1
  import { type Connection, goesAgain, type Idempotence, type Link } from './connection.ts'
2
2
  import { InvalidError } from './errors.ts'
3
+ import { ConnectionLostError } from './session.ts'
3
4
  import type { Key } from './wire/codec.ts'
4
5
  import { Reader, Writer } from './wire/msgpack.ts'
5
6
  import { Handle } from './wire/protocol.ts'
@@ -158,7 +159,12 @@ export function handleOn(connection: Connection, method: number, open: Open): Pr
158
159
  const key = `${method}:${encoded}`
159
160
  let opening = connection.handles.get(key)
160
161
  if (opening === undefined) {
161
- opening = connection.session.call(method, open).then(body => Handle.decode(body).handle ?? 0)
162
+ opening = connection.session.call(method, open).then(
163
+ body => Handle.decode(body).handle ?? 0,
164
+ (err: unknown) => {
165
+ throw unsent(err)
166
+ },
167
+ )
162
168
  connection.handles.set(key, opening)
163
169
  opening.catch(() => connection.handles.delete(key))
164
170
  }
@@ -172,11 +178,33 @@ export function handleOn(connection: Connection, method: number, open: Open): Pr
172
178
  return opening
173
179
  }
174
180
 
175
- /** A bucket's or a queue's name, as every engine takes it: [a-z0-9][a-z0-9_-]{0,63}. */
181
+ /**
182
+ * A connection lost while a handle opened, as the call that needed the handle
183
+ * meets it: the open's REQUEST had left, the call's own had not, so the call
184
+ * goes again on the next connection, where the handle opens again.
185
+ */
186
+ export function unsent(err: unknown): unknown {
187
+ return err instanceof ConnectionLostError && err.sent
188
+ ? new ConnectionLostError(err.message, false)
189
+ : err
190
+ }
191
+
192
+ const named = /^[a-z0-9][a-z0-9_-]{0,63}$/
193
+
194
+ /**
195
+ * A bucket's or a queue's name, as every engine takes it: [a-z0-9][a-z0-9_-]{0,63},
196
+ * with the nearest name that is when there is one: `focusroom.rooms` as `focusroom_rooms`.
197
+ */
176
198
  export function checkName(name: string, of: string): void {
177
- if (!/^[a-z0-9][a-z0-9_-]{0,63}$/.test(name)) {
199
+ if (!named.test(name)) {
200
+ const nearest = name
201
+ .toLowerCase()
202
+ .replace(/[^a-z0-9_-]/g, '_')
203
+ .replace(/^[_-]+/, '')
204
+ .slice(0, 64)
205
+ const offered = named.test(nearest) ? `, as ${JSON.stringify(nearest)}` : ''
178
206
  throw new InvalidError(
179
- `the ${of} name ${JSON.stringify(name)}: a name is [a-z0-9][a-z0-9_-]{0,63}`,
207
+ `the ${of} name ${JSON.stringify(name)}: a name is [a-z0-9][a-z0-9_-]{0,63}${offered}`,
180
208
  )
181
209
  }
182
210
  }
package/src/index.ts CHANGED
@@ -33,7 +33,9 @@ export {
33
33
  type EnvLayer,
34
34
  type EnvOptions,
35
35
  type Explained,
36
+ type ExplainGiven,
36
37
  type ExplainLine,
38
+ type ExplainOrigin,
37
39
  env,
38
40
  type FileLayer,
39
41
  file,
@@ -56,6 +58,7 @@ export { type ConsoleFormat, type ConsoleTime, secrets } from './console.ts'
56
58
  export * from './errors.ts'
57
59
  export type { Page } from './handles.ts'
58
60
  export type {
61
+ Adding,
59
62
  AddOptions,
60
63
  Answer,
61
64
  Backoff,
@@ -127,6 +130,7 @@ export {
127
130
  type Percentile,
128
131
  type PercentileBucket,
129
132
  type Plan,
133
+ type PrometheusOptions,
130
134
  type Query as MetricsQuery,
131
135
  type Series,
132
136
  type SeriesInput,
@@ -151,6 +155,7 @@ export type {
151
155
  EventsOptions,
152
156
  Fields,
153
157
  Level,
158
+ Levels,
154
159
  Lines,
155
160
  NewEvent,
156
161
  Page as EventsPage,
package/src/jobs.ts CHANGED
@@ -513,6 +513,7 @@ async function* watchJob<T>(source: Source<T>, id: string): AsyncGenerator<Job<T
513
513
  let last: Read<typeof JobsJob.fields> | undefined
514
514
  for (;;) {
515
515
  const stream = await source.link.run('read', connection => openWatch(connection, source, id))
516
+ const unfollow = source.link.follow(stream)
516
517
  try {
517
518
  for (;;) {
518
519
  const event = await stream.next()
@@ -532,6 +533,7 @@ async function* watchJob<T>(source: Source<T>, id: string): AsyncGenerator<Job<T
532
533
  }
533
534
  } finally {
534
535
  stream.cancel()
536
+ unfollow()
535
537
  }
536
538
  }
537
539
  }
@@ -602,6 +604,21 @@ export class Queue<T> {
602
604
  return (await this.#changed(methods['jobs.add'], fields)) === true
603
605
  }
604
606
 
607
+ /**
608
+ * A job's add for a batch of the database the queue was opened from:
609
+ * `db.batch` writes it with the batch's statements, so the job commits
610
+ * with the rows or not at all, and shares the commit with every other
611
+ * write of the file, where `db.tx` holds the writer and pays a sync of
612
+ * its own. It adds as `add` does, unless its id is taken: its `Done` in
613
+ * the batch's answer changed 1 job, or 0.
614
+ *
615
+ * await db.batch([sql`update notes set body = ${body} where id = ${id}`, indexing.adding({ noteId: id })])
616
+ */
617
+ adding(value: T, options: AddOptions = {}): Adding {
618
+ const call = { ...jobFields(options), id: options.id, value: this.#source.values.encode(value) }
619
+ return new Adding({ name: this.name, source: this.#source as Source<unknown>, call })
620
+ }
621
+
605
622
  /**
606
623
  * Makes the id's job this value at this time, whatever it was: one waiting
607
624
  * is made new, a failed one starts again, and a running one runs once more
@@ -767,6 +784,52 @@ export class TxQueue<T> {
767
784
  }
768
785
  }
769
786
 
787
+ /** A queue's add as a database's batch carries it: the queue, and the add's call but its handle. */
788
+ interface AddingParts {
789
+ name: string
790
+ source: Source<unknown>
791
+ call: Omit<Written<typeof JobsCall.fields>, 'handle'>
792
+ }
793
+
794
+ // reads an add's parts, which only this module sees
795
+ let partsOf: (adding: Adding) => AddingParts
796
+
797
+ /** A job's add that a database's batch carries, from `queue.adding`: it commits with the batch's statements, or not at all. */
798
+ export class Adding {
799
+ readonly #parts: AddingParts
800
+
801
+ /** Comes from `queue.adding`. */
802
+ constructor(parts: AddingParts) {
803
+ this.#parts = parts
804
+ }
805
+
806
+ static {
807
+ partsOf = adding => adding.#parts
808
+ }
809
+ }
810
+
811
+ /**
812
+ * What a batch of `home` sends of an add, on the connection it goes on: the
813
+ * add's call, its queue's handle opened there first. A queue kept outside
814
+ * `home` is refused, since its job would not commit with the rows.
815
+ */
816
+ export function addingIn(
817
+ adding: Adding,
818
+ home: Home,
819
+ ): (connection: Connection) => Promise<{ method: number; body: Uint8Array }> {
820
+ const { name, source, call } = partsOf(adding)
821
+ if (source.home !== home) {
822
+ const kept = source.home === undefined ? 'jobs.db' : source.home.describe
823
+ throw new InvalidError(
824
+ `${home.describe}: jobs queue ${name}: kept in ${kept}, outside this batch of ${home.describe}`,
825
+ )
826
+ }
827
+ return async connection => {
828
+ const handle = await handleOn(connection, source.openMethod, source.open)
829
+ return { method: methods['jobs.add'], body: JobsCall.encode({ ...call, handle }) }
830
+ }
831
+ }
832
+
770
833
  /** A write the store's transaction keeps for its commit: its method, and its call or the id it cancels. */
771
834
  export interface JobOp {
772
835
  source: Source<unknown>
package/src/logger.ts CHANGED
@@ -26,13 +26,13 @@ import {
26
26
  } from './console.ts'
27
27
  import { parseDotenv } from './dotenv.ts'
28
28
  import { InvalidError } from './errors.ts'
29
- import type { Fields, Level } from './records.ts'
29
+ import type { Fields, Level, Levels } from './records.ts'
30
30
  import { currentTrace, type Trace } from './trace.ts'
31
31
 
32
32
  export interface LoggerOptions {
33
33
  /** the lines held between writes; past it a line is dropped and counted: 1024 */
34
34
  buffer?: number
35
- /** the least level a line is kept at; every level when absent */
35
+ /** the least level a line is printed at, every level when absent */
36
36
  level?: Level
37
37
  /** how a line is written as it is logged: pretty on a terminal and JSON otherwise when absent */
38
38
  console?: ConsoleFormat
@@ -51,12 +51,16 @@ export interface LoggerOptions {
51
51
  redact?: readonly string[]
52
52
  /** leaves a URL's password in a value as it is, which is otherwise hidden */
53
53
  keepUrlPasswords?: boolean
54
- /** changes each field's value before it is hidden, kept and shown */
54
+ /**
55
+ * changes each field's value before it is hidden, kept and shown, and
56
+ * undefined leaves the field out: return the value it was given for a
57
+ * field it leaves as it is
58
+ */
55
59
  replace?: (key: string, value: unknown) => unknown
56
60
  /**
57
- * where LOG_LEVEL, LOG_FORMAT and LOG_TIME are read, which win over the
58
- * options: `env('APP')` reads APP_LOG_LEVEL and the rest, and false
59
- * reads nothing; the bare names when absent
61
+ * where LOG_LEVEL, LOG_CONSOLE_LEVEL, LOG_FORMAT and LOG_TIME are read,
62
+ * each of which wins over its option: `env('APP')` reads APP_LOG_LEVEL
63
+ * and the rest, and false reads nothing; the bare names when absent
60
64
  */
61
65
  env?: EnvLayer | false
62
66
  /** adds where each line was logged: source={"function":…,"file":…,"line":…} */
@@ -82,7 +86,10 @@ interface Shared {
82
86
  append: ((lines: ConsoleLine[]) => Promise<void>) | undefined
83
87
  stream: string
84
88
  most: number
85
- least: number
89
+ /** the least level kept, which the config log changes while the logger runs */
90
+ kept: number
91
+ /** the least level printed, LOG_CONSOLE_LEVEL's or level's console part; undefined prints what is kept */
92
+ printed: number | undefined
86
93
  redact: Redactor | undefined
87
94
  replace: ((key: string, value: unknown) => unknown) | undefined
88
95
  source: boolean
@@ -136,14 +143,41 @@ type LogDeclaration = {
136
143
  time: Setting<string>
137
144
  }
138
145
 
146
+ /** What a logger is told: a logger's options, a store's logger's level in its two parts. */
147
+ type Told = Omit<LoggerOptions, 'level'> & { level?: Level | Levels | undefined }
148
+
149
+ /** A logger's options as the environment leaves them: its level read apart, the least kept and the least printed. */
150
+ type Read = Omit<LoggerOptions, 'level'> & { kept: Level | undefined; printed: Level | undefined }
151
+
152
+ /**
153
+ * A level's two parts, the least kept and the least printed: one level is
154
+ * the kept alone, which the console then follows.
155
+ */
156
+ function partsOf(
157
+ level: Level | Levels | undefined,
158
+ ): [kept: Level | undefined, printed: Level | undefined] {
159
+ if (typeof level !== 'object') {
160
+ return [level, undefined]
161
+ }
162
+ for (const part of Object.keys(level)) {
163
+ if (part !== 'store' && part !== 'console') {
164
+ throw new InvalidError(
165
+ `a logger's level of ${JSON.stringify(part)}: its parts are store and console`,
166
+ )
167
+ }
168
+ }
169
+ return [level.store, level.console]
170
+ }
171
+
139
172
  /** A store's logger of one stream, and what it declares its config `log` with. */
140
173
  export function newStoreLogger(
141
174
  append: (lines: ConsoleLine[]) => Promise<void>,
142
175
  stream: string,
143
- options: LoggerOptions = {},
176
+ options: Told = {},
144
177
  ): [Logger, LogConfig] {
145
178
  const { shared, spelled } = sharedOf(append, stream, options, undefined)
146
- const level = options.level === undefined ? 'trace' : nameOf(levelOf(options.level))
179
+ const [kept] = partsOf(options.level)
180
+ const level = kept === undefined ? 'trace' : nameOf(levelOf(kept))
147
181
  const format = options.console ?? (prettyUntold(options, undefined) ? 'pretty' : 'json')
148
182
  const time = options.time ?? 'clock'
149
183
  const others = (all: readonly string[], one: string) => all.filter(name => name !== one)
@@ -153,9 +187,10 @@ export function newStoreLogger(
153
187
  time: oneOf(time, others(times, time)),
154
188
  }
155
189
  const env = options.env ?? environmentOf('')
190
+ // the level kept, and printed too while the level has no console part
156
191
  const take = (name: string) => {
157
192
  if (Object.hasOwn(levels, name)) {
158
- shared.least = levels[name as keyof typeof levels]
193
+ shared.kept = levels[name as keyof typeof levels]
159
194
  }
160
195
  }
161
196
  if (env === false) {
@@ -179,7 +214,7 @@ function nameOf(least: number): string {
179
214
  function sharedOf(
180
215
  append: ((lines: ConsoleLine[]) => Promise<void>) | undefined,
181
216
  stream: string,
182
- options: LoggerOptions,
217
+ options: Told,
183
218
  out: (ConsoleOut & { utc?: boolean }) | undefined,
184
219
  ): { shared: Shared; spelled: Record<string, string> } {
185
220
  if (stream === '') {
@@ -194,7 +229,8 @@ function sharedOf(
194
229
  append,
195
230
  stream,
196
231
  most,
197
- least: read.level === undefined ? Number.NEGATIVE_INFINITY : levelOf(read.level),
232
+ kept: read.kept === undefined ? Number.NEGATIVE_INFINITY : levelOf(read.kept),
233
+ printed: read.printed === undefined ? undefined : levelOf(read.printed),
198
234
  redact: redactor(read.redact, read.keepUrlPasswords),
199
235
  replace: read.replace,
200
236
  source: read.source === true,
@@ -213,14 +249,16 @@ function sharedOf(
213
249
  type Unread = [name: string, value: string, expected: string]
214
250
 
215
251
  /**
216
- * LOG_LEVEL, LOG_FORMAT and LOG_TIME over the options, after the prefix env
217
- * names, as Go and Python read them; a console turned off stays off. Each
218
- * variable read is spelled as the config `log` reads it, by its name.
252
+ * LOG_LEVEL, LOG_CONSOLE_LEVEL, LOG_FORMAT and LOG_TIME over the options,
253
+ * each over its own, after the prefix env names, as Go and Python read them;
254
+ * a console turned off stays off. Each variable read is spelled as the
255
+ * config `log` reads it, by its name, but LOG_CONSOLE_LEVEL, which no
256
+ * setting of the config holds.
219
257
  */
220
- function fromEnvironment(
221
- options: LoggerOptions,
222
- ): [LoggerOptions, Unread[], Record<string, string>] {
223
- const read = { ...options }
258
+ function fromEnvironment(options: Told): [Read, Unread[], Record<string, string>] {
259
+ const { level: given, ...rest } = options
260
+ const [kept, printed] = partsOf(given)
261
+ const read: Read = { ...rest, kept, printed }
224
262
  const ignored: Unread[] = []
225
263
  const spelled: Record<string, string> = {}
226
264
  const env = options.env ?? environmentOf('')
@@ -229,15 +267,14 @@ function fromEnvironment(
229
267
  }
230
268
  const variables = environment(env)
231
269
  const name = (variable: string) => (env.prefix === '' ? variable : `${env.prefix}_${variable}`)
232
- const level = variables(name('LOG_LEVEL'))
270
+ const level = levelIn(variables, name('LOG_LEVEL'), ignored)
233
271
  if (level !== undefined) {
234
- const known = level.toLowerCase() === 'warning' ? 'warn' : level.toLowerCase()
235
- if (Object.hasOwn(levels, known)) {
236
- read.level = levels[known as keyof typeof levels]
237
- spelled[name('LOG_LEVEL')] = known
238
- } else {
239
- ignored.push([name('LOG_LEVEL'), level, 'trace, debug, info, warn or error'])
240
- }
272
+ read.kept = levels[level]
273
+ spelled[name('LOG_LEVEL')] = level
274
+ }
275
+ const shown = levelIn(variables, name('LOG_CONSOLE_LEVEL'), ignored)
276
+ if (shown !== undefined) {
277
+ read.printed = levels[shown]
241
278
  }
242
279
  const format = variables(name('LOG_FORMAT'))
243
280
  if (format !== undefined) {
@@ -262,6 +299,24 @@ function fromEnvironment(
262
299
  return [read, ignored, spelled]
263
300
  }
264
301
 
302
+ /** The level a variable names, the case ignored and warning read as warn; a value it cannot mean is ignored. */
303
+ function levelIn(
304
+ variables: (name: string) => string | undefined,
305
+ name: string,
306
+ ignored: Unread[],
307
+ ): keyof typeof levels | undefined {
308
+ const text = variables(name)
309
+ if (text === undefined) {
310
+ return undefined
311
+ }
312
+ const known = text.toLowerCase() === 'warning' ? 'warn' : text.toLowerCase()
313
+ if (Object.hasOwn(levels, known)) {
314
+ return known as keyof typeof levels
315
+ }
316
+ ignored.push([name, text, 'trace, debug, info, warn or error'])
317
+ return undefined
318
+ }
319
+
265
320
  /** The variables of env's files, the process's or `from`'s over them, a missing file skipped; an empty one is none. */
266
321
  function environment(env: EnvLayer): (name: string) => string | undefined {
267
322
  const fromFiles: Record<string, string> = {}
@@ -314,14 +369,17 @@ function forceColor(): boolean {
314
369
  }
315
370
 
316
371
  /** Whether a console told no format prints pretty lines: on a terminal, or where FORCE_COLOR asks. */
317
- function prettyUntold(options: LoggerOptions, out: ConsoleOut | undefined): boolean {
372
+ function prettyUntold(options: Pick<LoggerOptions, 'to'>, out: ConsoleOut | undefined): boolean {
318
373
  const stream: ConsoleOut = out ?? options.to ?? process.stderr
319
374
  // FORCE_COLOR other than 0 says a person reads a pipe, as an IDE's run console is one
320
375
  return stream.isTTY === true || (out === undefined && forceColor())
321
376
  }
322
377
 
323
378
  /** How a logger prints its lines, or undefined when it prints none. */
324
- function printer(options: LoggerOptions, out?: ConsoleOut & { utc?: boolean }): Shared['print'] {
379
+ function printer(
380
+ options: Pick<LoggerOptions, 'console' | 'to' | 'time' | 'hideStream'>,
381
+ out?: ConsoleOut & { utc?: boolean },
382
+ ): Shared['print'] {
325
383
  if (options.console === 'off') {
326
384
  return undefined
327
385
  }
@@ -436,12 +494,14 @@ export class Logger {
436
494
  }
437
495
 
438
496
  #log(level: number, message: string, attrs: Fields | undefined): void {
439
- if (level < this.#shared.least) {
497
+ // a line that no place of the logger takes is never encoded
498
+ if (!prints(this.#shared, level) && !keeps(this.#shared, level)) {
440
499
  return
441
500
  }
442
501
  this.#emit({ level, msg: message }, attrs)
443
502
  }
444
503
 
504
+ /** Prints a line from the level printed and keeps it from the level kept; an event, which has none, both. */
445
505
  #emit(what: Pick<ConsoleLine, 'level' | 'event' | 'msg'>, attrs: Fields | undefined): void {
446
506
  const trace = traceOf(currentTrace())
447
507
  const shared = this.#shared
@@ -454,8 +514,12 @@ export class Logger {
454
514
  attrs: shared.source ? [...encoded, sourceKey, sourceOfCaller()] : encoded,
455
515
  ...trace,
456
516
  }
457
- this.#shared.print?.(line)
458
- this.#queue(line)
517
+ if (what.level === undefined || prints(shared, what.level)) {
518
+ shared.print?.(line)
519
+ }
520
+ if (what.level === undefined || keeps(shared, what.level)) {
521
+ this.#queue(line)
522
+ }
459
523
  }
460
524
 
461
525
  #queue(line: ConsoleLine): void {
@@ -479,6 +543,16 @@ export class Logger {
479
543
  }
480
544
  }
481
545
 
546
+ /** Whether a logger prints a line of level: it has a console, and the level is the level printed or above. */
547
+ function prints(shared: Shared, level: number): boolean {
548
+ return shared.print !== undefined && level >= (shared.printed ?? shared.kept)
549
+ }
550
+
551
+ /** Whether a logger keeps a line of level: it is a store's, and the level is its level or above. */
552
+ function keeps(shared: Shared, level: number): boolean {
553
+ return shared.append !== undefined && level >= shared.kept
554
+ }
555
+
482
556
  /**
483
557
  * Writes what the buffer holds, and again while half of it filled during the
484
558
  * write before. A full buffer drops a line before the line could ask for a
@@ -491,21 +565,61 @@ async function writeWaiting(
491
565
  ): Promise<void> {
492
566
  let batch = shared.waiting.splice(0)
493
567
  while (batch.length > 0) {
494
- try {
495
- await append(batch)
496
- shared.failures?.succeeded()
497
- } catch (err) {
498
- shared.dropped += batch.length
499
- shared.failures?.failed(err)
568
+ for (const piece of pieces(batch)) {
569
+ try {
570
+ await append(piece)
571
+ shared.failures?.succeeded()
572
+ } catch (err) {
573
+ shared.dropped += piece.length
574
+ shared.failures?.failed(err)
575
+ }
500
576
  }
501
577
  batch = shared.waiting.length >= shared.most / 2 ? shared.waiting.splice(0) : []
502
578
  }
503
579
  }
504
580
 
581
+ /**
582
+ * The text a write carries at most. A request's body holds 2 MiB, and a
583
+ * buffer of long lines, or one made larger, held more: the whole write was
584
+ * refused and every line in it dropped. Counted in UTF-16 units, three bytes
585
+ * each at worst, so a piece stays within the body however its text is spelt.
586
+ */
587
+ const PIECE = 512 * 1024
588
+
589
+ /** The lines in writes of at most PIECE of text each, in their order. */
590
+ function pieces(lines: ConsoleLine[]): ConsoleLine[][] {
591
+ const out: ConsoleLine[][] = []
592
+ let piece: ConsoleLine[] = []
593
+ let size = 0
594
+ for (const line of lines) {
595
+ const weight = textOf(line)
596
+ if (piece.length > 0 && size + weight > PIECE) {
597
+ out.push(piece)
598
+ piece = []
599
+ size = 0
600
+ }
601
+ piece.push(line)
602
+ size += weight
603
+ }
604
+ if (piece.length > 0) {
605
+ out.push(piece)
606
+ }
607
+ return out
608
+ }
609
+
610
+ /** A line's text, with a little for its time, level and framing. */
611
+ function textOf(line: ConsoleLine): number {
612
+ let size = 64 + (line.msg?.length ?? 0) + (line.event?.length ?? 0)
613
+ for (const part of line.attrs) size += part.length
614
+ for (const part of line.context) size += part.length
615
+ return size
616
+ }
617
+
505
618
  /**
506
619
  * Fields as key, JSON pairs, as a console writes them and the store keeps
507
- * them: an Error's value its stack, which JSON would write as {}, a secret's
508
- * value hidden at any depth, and a URL's password unless kept.
620
+ * them: a field whose value is undefined left out, as JSON.stringify leaves
621
+ * it out of an object, an Error's value its stack, which JSON would write as
622
+ * {}, a secret's value hidden at any depth, and a URL's password unless kept.
509
623
  */
510
624
  export function encodeFields(
511
625
  fields: Fields,
@@ -518,6 +632,10 @@ export function encodeFields(
518
632
  const flat: string[] = []
519
633
  for (const [key, given] of pairs) {
520
634
  const value = replace === undefined ? given : replace(key, given)
635
+ // before it is hidden, as Rust's replace drops a field: a secret given no value is no field
636
+ if (value === undefined) {
637
+ continue
638
+ }
521
639
  const shown = value instanceof Error ? (value.stack ?? String(value)) : value
522
640
  if (redact?.hides(key)) {
523
641
  flat.push(key, redacted)
package/src/metrics.ts CHANGED
@@ -28,6 +28,7 @@ import {
28
28
  MetricsDamages,
29
29
  MetricsDeleted,
30
30
  MetricsDescription,
31
+ MetricsExposition,
31
32
  MetricsIngest,
32
33
  MetricsLatest,
33
34
  MetricsNameAsked,
@@ -102,6 +103,16 @@ export interface SeriesInput {
102
103
  keep?: Duration | undefined
103
104
  }
104
105
 
106
+ /** What an ingest of an exporter's text gives every series of it. */
107
+ export interface PrometheusOptions {
108
+ /** labels every series gets beside its own, such as the instance scraped; one the text gives too is refused, naming its line */
109
+ labels?: Record<string, LabelValue> | undefined
110
+ /** the time of every sample the text gives none; the store's clock unless given */
111
+ at?: Time | undefined
112
+ /** how long every name of the text keeps its samples */
113
+ keep?: Duration | undefined
114
+ }
115
+
105
116
  /** A series as a read gives it: its samples as two columns. */
106
117
  export interface Series {
107
118
  name: string
@@ -206,7 +217,14 @@ export interface Plan {
206
217
  series: number
207
218
  bytes: number
208
219
  decoded: number
209
- exceeds: { limit: string; wanted: number; bound: number } | undefined
220
+ exceeds:
221
+ | {
222
+ /** series, bytes read, samples decoded or answer */
223
+ limit: string
224
+ wanted: number
225
+ bound: number
226
+ }
227
+ | undefined
210
228
  }
211
229
 
212
230
  /** A page's options: a prefix, how many, and where the page before ended. */
@@ -285,6 +303,33 @@ export class Metrics {
285
303
  )
286
304
  }
287
305
 
306
+ /**
307
+ * Reads the text an exporter serves, in Prometheus' format or OpenMetrics', and writes its
308
+ * series, every one or none: a counter as a counter, a histogram's _bucket series as
309
+ * histogram. A line it cannot read is refused, naming the line.
310
+ *
311
+ * const text = await (await fetch('http://web-1:9100/metrics')).text()
312
+ * await store.metrics.ingestPrometheus(text, { labels: { instance: 'web-1' } })
313
+ */
314
+ async ingestPrometheus(
315
+ text: string | Uint8Array,
316
+ options: PrometheusOptions = {},
317
+ ): Promise<void> {
318
+ const labels: Labels = {}
319
+ for (const [label, value] of Object.entries(options.labels ?? {})) {
320
+ labels[label] = String(value)
321
+ }
322
+ const body = MetricsExposition.encode({
323
+ text: typeof text === 'string' ? new TextEncoder().encode(text) : text,
324
+ labels,
325
+ at: options.at === undefined ? undefined : unixMs(options.at),
326
+ keep: options.keep === undefined ? undefined : ms(options.keep),
327
+ })
328
+ await this.#link.run('write', connection =>
329
+ connection.session.call(methods['metrics.ingestPrometheus'], body),
330
+ )
331
+ }
332
+
288
333
  /** The samples of each series the query finds, in its range: whole, or a limit error naming its bound. */
289
334
  async read(query: Query): Promise<Series[]> {
290
335
  const parts = await this.#download('metrics.read', MetricsQuery.encode(queryOf(query)))
package/src/query.ts CHANGED
@@ -868,18 +868,31 @@ function render(parts: Parts): Sql {
868
868
  return new Sql(text, values)
869
869
  }
870
870
 
871
- /** The condition that starts a page after a row's order values. */
871
+ /**
872
+ * The condition that starts a page after a row's order values. SQLite orders
873
+ * a null before every value, first in ascending order and last in descending,
874
+ * and a comparison with a null is never true: equal is `is`, and past a null
875
+ * is every value ascending and nothing descending.
876
+ */
872
877
  function keyset(order: Parts['order'], after: readonly SqlValue[]): Sql {
873
878
  const alternatives: string[] = []
874
879
  const values: SqlArg[] = []
875
880
  order.forEach((each, at) => {
876
- const equal = order.slice(0, at).map(earlier => `${earlier.piece.text} = ?`)
881
+ const column = each.piece.text
882
+ const value = after[at] ?? null
883
+ if (value === null && each.direction === 'desc') {
884
+ return
885
+ }
886
+ const equal = order.slice(0, at).map(earlier => `${earlier.piece.text} is ?`)
877
887
  values.push(...after.slice(0, at))
878
- const past = `${each.piece.text} ${each.direction === 'asc' ? '>' : '<'} ?`
879
- values.push(after[at] ?? null)
888
+ let past = `${column} is not null`
889
+ if (value !== null) {
890
+ past = each.direction === 'asc' ? `${column} > ?` : `(${column} < ? or ${column} is null)`
891
+ values.push(value)
892
+ }
880
893
  alternatives.push(equal.length === 0 ? past : `(${[...equal, past].join(' and ')})`)
881
894
  })
882
- return new Sql(`(${alternatives.join(' or ')})`, values)
895
+ return new Sql(alternatives.length === 0 ? '0' : `(${alternatives.join(' or ')})`, values)
883
896
  }
884
897
 
885
898
  /** A short digest of a query, which a cursor carries so that another query refuses it. */
@@ -893,10 +906,14 @@ function digestOf(statement: Sql): string {
893
906
  return hash.toString(36)
894
907
  }
895
908
 
909
+ /** A cursor of order values: a bigint, and an infinity JSON has no number for, as their text. */
896
910
  function cursorText(values: readonly SqlValue[], digest: string): string {
897
- const json = JSON.stringify({ q: digest, v: values }, (_, v) =>
898
- typeof v === 'bigint' ? { n: String(v) } : v,
899
- )
911
+ const json = JSON.stringify({ q: digest, v: values }, (_, v) => {
912
+ if (typeof v === 'bigint') {
913
+ return { n: String(v) }
914
+ }
915
+ return typeof v === 'number' && !Number.isFinite(v) ? { r: String(v) } : v
916
+ })
900
917
  return Buffer.from(json).toString('base64url')
901
918
  }
902
919
 
@@ -912,11 +929,14 @@ function cursorOf(cursor: string, digest: string): SqlValue[] {
912
929
  "a cursor of another query: a page's next reads only the query that made it",
913
930
  )
914
931
  }
915
- return read.v.map(v =>
916
- typeof v === 'object' && v !== null && 'n' in v
917
- ? BigInt((v as { n: string }).n)
918
- : (v as SqlValue),
919
- )
932
+ return read.v.map(v => {
933
+ if (typeof v === 'object' && v !== null && 'n' in v) {
934
+ return BigInt((v as { n: string }).n)
935
+ }
936
+ return typeof v === 'object' && v !== null && 'r' in v
937
+ ? Number((v as { r: string }).r)
938
+ : (v as SqlValue)
939
+ })
920
940
  }
921
941
 
922
942
  /** An insert of rows of the same columns, the first row's. */