@tinyshed/tinystore 0.1.0 → 0.2.0-rc.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. package/dist/blobs.d.ts +1 -1
  2. package/dist/blobs.js +13 -5
  3. package/dist/cancel.d.ts +16 -2
  4. package/dist/cancel.js +36 -2
  5. package/dist/channels.js +20 -6
  6. package/dist/config.d.ts +49 -1
  7. package/dist/config.js +86 -7
  8. package/dist/connection.d.ts +52 -10
  9. package/dist/connection.js +305 -73
  10. package/dist/handles.d.ts +10 -1
  11. package/dist/handles.js +27 -4
  12. package/dist/index.d.ts +5 -5
  13. package/dist/jobs.d.ts +35 -1
  14. package/dist/jobs.js +46 -0
  15. package/dist/logger.d.ts +22 -10
  16. package/dist/logger.js +118 -28
  17. package/dist/metrics.d.ts +24 -1
  18. package/dist/metrics.js +24 -1
  19. package/dist/query.js +33 -9
  20. package/dist/records.d.ts +48 -7
  21. package/dist/records.js +22 -3
  22. package/dist/runtime/addon.js +3 -2
  23. package/dist/runtime/bun.js +33 -12
  24. package/dist/runtime/node.js +30 -8
  25. package/dist/runtime/pipe.d.ts +2 -0
  26. package/dist/runtime/pipe.js +45 -20
  27. package/dist/runtime/shared.d.ts +28 -4
  28. package/dist/runtime/shared.js +125 -13
  29. package/dist/runtime.d.ts +28 -6
  30. package/dist/session.d.ts +9 -5
  31. package/dist/session.js +113 -31
  32. package/dist/sql.d.ts +18 -5
  33. package/dist/sql.js +44 -11
  34. package/dist/store.d.ts +25 -11
  35. package/dist/store.js +67 -9
  36. package/dist/time.d.ts +2 -0
  37. package/dist/wire/codec.d.ts +5 -0
  38. package/dist/wire/codec.js +10 -0
  39. package/dist/wire/frame.d.ts +2 -0
  40. package/dist/wire/frame.js +4 -0
  41. package/dist/wire/msgpack.d.ts +7 -3
  42. package/dist/wire/msgpack.js +11 -4
  43. package/dist/wire/protocol.d.ts +280 -36
  44. package/dist/wire/protocol.js +510 -102
  45. package/package.json +9 -8
  46. package/src/blobs.ts +13 -5
  47. package/src/cancel.ts +44 -2
  48. package/src/channels.ts +21 -6
  49. package/src/config.ts +138 -12
  50. package/src/connection.ts +377 -77
  51. package/src/handles.ts +32 -4
  52. package/src/index.ts +6 -1
  53. package/src/jobs.ts +66 -0
  54. package/src/logger.ts +158 -40
  55. package/src/metrics.ts +53 -2
  56. package/src/query.ts +33 -13
  57. package/src/records.ts +73 -10
  58. package/src/runtime/addon.ts +3 -2
  59. package/src/runtime/bun.ts +44 -9
  60. package/src/runtime/node.ts +30 -5
  61. package/src/runtime/pipe.ts +49 -19
  62. package/src/runtime/shared.ts +137 -16
  63. package/src/runtime.ts +31 -6
  64. package/src/session.ts +129 -30
  65. package/src/sql.ts +48 -14
  66. package/src/store.ts +81 -14
  67. package/src/time.ts +3 -0
  68. package/src/wire/codec.ts +11 -0
  69. package/src/wire/frame.ts +5 -0
  70. package/src/wire/msgpack.ts +11 -4
  71. package/src/wire/protocol.ts +532 -104
package/src/handles.ts CHANGED
@@ -1,5 +1,6 @@
1
1
  import { type Connection, goesAgain, type Idempotence, type Link } from './connection.ts'
2
2
  import { InvalidError } from './errors.ts'
3
+ import { ConnectionLostError } from './session.ts'
3
4
  import type { Key } from './wire/codec.ts'
4
5
  import { Reader, Writer } from './wire/msgpack.ts'
5
6
  import { Handle } from './wire/protocol.ts'
@@ -158,7 +159,12 @@ export function handleOn(connection: Connection, method: number, open: Open): Pr
158
159
  const key = `${method}:${encoded}`
159
160
  let opening = connection.handles.get(key)
160
161
  if (opening === undefined) {
161
- opening = connection.session.call(method, open).then(body => Handle.decode(body).handle ?? 0)
162
+ opening = connection.session.call(method, open).then(
163
+ body => Handle.decode(body).handle ?? 0,
164
+ (err: unknown) => {
165
+ throw unsent(err)
166
+ },
167
+ )
162
168
  connection.handles.set(key, opening)
163
169
  opening.catch(() => connection.handles.delete(key))
164
170
  }
@@ -172,11 +178,33 @@ export function handleOn(connection: Connection, method: number, open: Open): Pr
172
178
  return opening
173
179
  }
174
180
 
175
- /** A bucket's or a queue's name, as every engine takes it: [a-z0-9][a-z0-9_-]{0,63}. */
181
+ /**
182
+ * A connection lost while a handle opened, as the call that needed the handle
183
+ * meets it: the open's REQUEST had left, the call's own had not, so the call
184
+ * goes again on the next connection, where the handle opens again.
185
+ */
186
+ export function unsent(err: unknown): unknown {
187
+ return err instanceof ConnectionLostError && err.sent
188
+ ? new ConnectionLostError(err.message, false)
189
+ : err
190
+ }
191
+
192
+ const named = /^[a-z0-9][a-z0-9_-]{0,63}$/
193
+
194
+ /**
195
+ * A bucket's or a queue's name, as every engine takes it: [a-z0-9][a-z0-9_-]{0,63},
196
+ * with the nearest name that is when there is one: `focusroom.rooms` as `focusroom_rooms`.
197
+ */
176
198
  export function checkName(name: string, of: string): void {
177
- if (!/^[a-z0-9][a-z0-9_-]{0,63}$/.test(name)) {
199
+ if (!named.test(name)) {
200
+ const nearest = name
201
+ .toLowerCase()
202
+ .replace(/[^a-z0-9_-]/g, '_')
203
+ .replace(/^[_-]+/, '')
204
+ .slice(0, 64)
205
+ const offered = named.test(nearest) ? `, as ${JSON.stringify(nearest)}` : ''
178
206
  throw new InvalidError(
179
- `the ${of} name ${JSON.stringify(name)}: a name is [a-z0-9][a-z0-9_-]{0,63}`,
207
+ `the ${of} name ${JSON.stringify(name)}: a name is [a-z0-9][a-z0-9_-]{0,63}${offered}`,
180
208
  )
181
209
  }
182
210
  }
package/src/index.ts CHANGED
@@ -33,7 +33,9 @@ export {
33
33
  type EnvLayer,
34
34
  type EnvOptions,
35
35
  type Explained,
36
+ type ExplainGiven,
36
37
  type ExplainLine,
38
+ type ExplainOrigin,
37
39
  env,
38
40
  type FileLayer,
39
41
  file,
@@ -56,6 +58,7 @@ export { type ConsoleFormat, type ConsoleTime, secrets } from './console.ts'
56
58
  export * from './errors.ts'
57
59
  export type { Page } from './handles.ts'
58
60
  export type {
61
+ Adding,
59
62
  AddOptions,
60
63
  Answer,
61
64
  Backoff,
@@ -127,6 +130,7 @@ export {
127
130
  type Percentile,
128
131
  type PercentileBucket,
129
132
  type Plan,
133
+ type PrometheusOptions,
130
134
  type Query as MetricsQuery,
131
135
  type Series,
132
136
  type SeriesInput,
@@ -151,6 +155,7 @@ export type {
151
155
  EventsOptions,
152
156
  Fields,
153
157
  Level,
158
+ Levels,
154
159
  Lines,
155
160
  NewEvent,
156
161
  Page as EventsPage,
@@ -205,7 +210,7 @@ export {
205
210
  type Status,
206
211
  Store,
207
212
  } from './store.ts'
208
- export type { Duration, DurationText, Time } from './time.ts'
213
+ export type { Calendar, Duration, DurationText, Time } from './time.ts'
209
214
  export { type Trace, withTrace } from './trace.ts'
210
215
  export type { Tx, TxBucket, TxCounters } from './tx.ts'
211
216
  export type { Key } from './wire/codec.ts'
package/src/jobs.ts CHANGED
@@ -57,6 +57,8 @@ export interface QueueOptions {
57
57
  concurrency?: number | Concurrency | undefined
58
58
  /** jobs started in any span: '30/s' */
59
59
  rate?: Rate | undefined
60
+ /** jobs of each group started in any span, beside rate: '2/s', each account's own limit; it lives in memory */
61
+ groupRate?: Rate | undefined
60
62
  /** how long a done job's id stays taken, so that an add of it adds nothing */
61
63
  dedupe?: Duration | undefined
62
64
  /** how long a failed job stays, to be found and started again: a week */
@@ -513,6 +515,7 @@ async function* watchJob<T>(source: Source<T>, id: string): AsyncGenerator<Job<T
513
515
  let last: Read<typeof JobsJob.fields> | undefined
514
516
  for (;;) {
515
517
  const stream = await source.link.run('read', connection => openWatch(connection, source, id))
518
+ const unfollow = source.link.follow(stream)
516
519
  try {
517
520
  for (;;) {
518
521
  const event = await stream.next()
@@ -532,6 +535,7 @@ async function* watchJob<T>(source: Source<T>, id: string): AsyncGenerator<Job<T
532
535
  }
533
536
  } finally {
534
537
  stream.cancel()
538
+ unfollow()
535
539
  }
536
540
  }
537
541
  }
@@ -602,6 +606,21 @@ export class Queue<T> {
602
606
  return (await this.#changed(methods['jobs.add'], fields)) === true
603
607
  }
604
608
 
609
+ /**
610
+ * A job's add for a batch of the database the queue was opened from:
611
+ * `db.batch` writes it with the batch's statements, so the job commits
612
+ * with the rows or not at all, and shares the commit with every other
613
+ * write of the file, where `db.tx` holds the writer and pays a sync of
614
+ * its own. It adds as `add` does, unless its id is taken: its `Done` in
615
+ * the batch's answer changed 1 job, or 0.
616
+ *
617
+ * await db.batch([sql`update notes set body = ${body} where id = ${id}`, indexing.adding({ noteId: id })])
618
+ */
619
+ adding(value: T, options: AddOptions = {}): Adding {
620
+ const call = { ...jobFields(options), id: options.id, value: this.#source.values.encode(value) }
621
+ return new Adding({ name: this.name, source: this.#source as Source<unknown>, call })
622
+ }
623
+
605
624
  /**
606
625
  * Makes the id's job this value at this time, whatever it was: one waiting
607
626
  * is made new, a failed one starts again, and a running one runs once more
@@ -767,6 +786,52 @@ export class TxQueue<T> {
767
786
  }
768
787
  }
769
788
 
789
+ /** A queue's add as a database's batch carries it: the queue, and the add's call but its handle. */
790
+ interface AddingParts {
791
+ name: string
792
+ source: Source<unknown>
793
+ call: Omit<Written<typeof JobsCall.fields>, 'handle'>
794
+ }
795
+
796
+ // reads an add's parts, which only this module sees
797
+ let partsOf: (adding: Adding) => AddingParts
798
+
799
+ /** A job's add that a database's batch carries, from `queue.adding`: it commits with the batch's statements, or not at all. */
800
+ export class Adding {
801
+ readonly #parts: AddingParts
802
+
803
+ /** Comes from `queue.adding`. */
804
+ constructor(parts: AddingParts) {
805
+ this.#parts = parts
806
+ }
807
+
808
+ static {
809
+ partsOf = adding => adding.#parts
810
+ }
811
+ }
812
+
813
+ /**
814
+ * What a batch of `home` sends of an add, on the connection it goes on: the
815
+ * add's call, its queue's handle opened there first. A queue kept outside
816
+ * `home` is refused, since its job would not commit with the rows.
817
+ */
818
+ export function addingIn(
819
+ adding: Adding,
820
+ home: Home,
821
+ ): (connection: Connection) => Promise<{ method: number; body: Uint8Array }> {
822
+ const { name, source, call } = partsOf(adding)
823
+ if (source.home !== home) {
824
+ const kept = source.home === undefined ? 'jobs.db' : source.home.describe
825
+ throw new InvalidError(
826
+ `${home.describe}: jobs queue ${name}: kept in ${kept}, outside this batch of ${home.describe}`,
827
+ )
828
+ }
829
+ return async connection => {
830
+ const handle = await handleOn(connection, source.openMethod, source.open)
831
+ return { method: methods['jobs.add'], body: JobsCall.encode({ ...call, handle }) }
832
+ }
833
+ }
834
+
770
835
  /** A write the store's transaction keeps for its commit: its method, and its call or the id it cancels. */
771
836
  export interface JobOp {
772
837
  source: Source<unknown>
@@ -920,6 +985,7 @@ export function openQueue<T>(
920
985
  group: whole(concurrency.group, describe, "a group's concurrency"),
921
986
  },
922
987
  rate: options.rate === undefined ? undefined : rateOf(options.rate),
988
+ groupRate: options.groupRate === undefined ? undefined : rateOf(options.groupRate),
923
989
  dedupe: msOf(options.dedupe),
924
990
  keep: msOf(options.keep),
925
991
  maxWaiting: whole(options.maxWaiting, describe, 'a maxWaiting'),
package/src/logger.ts CHANGED
@@ -26,13 +26,13 @@ import {
26
26
  } from './console.ts'
27
27
  import { parseDotenv } from './dotenv.ts'
28
28
  import { InvalidError } from './errors.ts'
29
- import type { Fields, Level } from './records.ts'
29
+ import type { Fields, Level, Levels } from './records.ts'
30
30
  import { currentTrace, type Trace } from './trace.ts'
31
31
 
32
32
  export interface LoggerOptions {
33
33
  /** the lines held between writes; past it a line is dropped and counted: 1024 */
34
34
  buffer?: number
35
- /** the least level a line is kept at; every level when absent */
35
+ /** the least level a line is printed at, every level when absent */
36
36
  level?: Level
37
37
  /** how a line is written as it is logged: pretty on a terminal and JSON otherwise when absent */
38
38
  console?: ConsoleFormat
@@ -51,12 +51,16 @@ export interface LoggerOptions {
51
51
  redact?: readonly string[]
52
52
  /** leaves a URL's password in a value as it is, which is otherwise hidden */
53
53
  keepUrlPasswords?: boolean
54
- /** changes each field's value before it is hidden, kept and shown */
54
+ /**
55
+ * changes each field's value before it is hidden, kept and shown, and
56
+ * undefined leaves the field out: return the value it was given for a
57
+ * field it leaves as it is
58
+ */
55
59
  replace?: (key: string, value: unknown) => unknown
56
60
  /**
57
- * where LOG_LEVEL, LOG_FORMAT and LOG_TIME are read, which win over the
58
- * options: `env('APP')` reads APP_LOG_LEVEL and the rest, and false
59
- * reads nothing; the bare names when absent
61
+ * where LOG_LEVEL, LOG_CONSOLE_LEVEL, LOG_FORMAT and LOG_TIME are read,
62
+ * each of which wins over its option: `env('APP')` reads APP_LOG_LEVEL
63
+ * and the rest, and false reads nothing; the bare names when absent
60
64
  */
61
65
  env?: EnvLayer | false
62
66
  /** adds where each line was logged: source={"function":…,"file":…,"line":…} */
@@ -82,7 +86,10 @@ interface Shared {
82
86
  append: ((lines: ConsoleLine[]) => Promise<void>) | undefined
83
87
  stream: string
84
88
  most: number
85
- least: number
89
+ /** the least level kept, which the config log changes while the logger runs */
90
+ kept: number
91
+ /** the least level printed, LOG_CONSOLE_LEVEL's or level's console part; undefined prints what is kept */
92
+ printed: number | undefined
86
93
  redact: Redactor | undefined
87
94
  replace: ((key: string, value: unknown) => unknown) | undefined
88
95
  source: boolean
@@ -136,14 +143,41 @@ type LogDeclaration = {
136
143
  time: Setting<string>
137
144
  }
138
145
 
146
+ /** What a logger is told: a logger's options, a store's logger's level in its two parts. */
147
+ type Told = Omit<LoggerOptions, 'level'> & { level?: Level | Levels | undefined }
148
+
149
+ /** A logger's options as the environment leaves them: its level read apart, the least kept and the least printed. */
150
+ type Read = Omit<LoggerOptions, 'level'> & { kept: Level | undefined; printed: Level | undefined }
151
+
152
+ /**
153
+ * A level's two parts, the least kept and the least printed: one level is
154
+ * the kept alone, which the console then follows.
155
+ */
156
+ function partsOf(
157
+ level: Level | Levels | undefined,
158
+ ): [kept: Level | undefined, printed: Level | undefined] {
159
+ if (typeof level !== 'object') {
160
+ return [level, undefined]
161
+ }
162
+ for (const part of Object.keys(level)) {
163
+ if (part !== 'store' && part !== 'console') {
164
+ throw new InvalidError(
165
+ `a logger's level of ${JSON.stringify(part)}: its parts are store and console`,
166
+ )
167
+ }
168
+ }
169
+ return [level.store, level.console]
170
+ }
171
+
139
172
  /** A store's logger of one stream, and what it declares its config `log` with. */
140
173
  export function newStoreLogger(
141
174
  append: (lines: ConsoleLine[]) => Promise<void>,
142
175
  stream: string,
143
- options: LoggerOptions = {},
176
+ options: Told = {},
144
177
  ): [Logger, LogConfig] {
145
178
  const { shared, spelled } = sharedOf(append, stream, options, undefined)
146
- const level = options.level === undefined ? 'trace' : nameOf(levelOf(options.level))
179
+ const [kept] = partsOf(options.level)
180
+ const level = kept === undefined ? 'trace' : nameOf(levelOf(kept))
147
181
  const format = options.console ?? (prettyUntold(options, undefined) ? 'pretty' : 'json')
148
182
  const time = options.time ?? 'clock'
149
183
  const others = (all: readonly string[], one: string) => all.filter(name => name !== one)
@@ -153,9 +187,10 @@ export function newStoreLogger(
153
187
  time: oneOf(time, others(times, time)),
154
188
  }
155
189
  const env = options.env ?? environmentOf('')
190
+ // the level kept, and printed too while the level has no console part
156
191
  const take = (name: string) => {
157
192
  if (Object.hasOwn(levels, name)) {
158
- shared.least = levels[name as keyof typeof levels]
193
+ shared.kept = levels[name as keyof typeof levels]
159
194
  }
160
195
  }
161
196
  if (env === false) {
@@ -179,7 +214,7 @@ function nameOf(least: number): string {
179
214
  function sharedOf(
180
215
  append: ((lines: ConsoleLine[]) => Promise<void>) | undefined,
181
216
  stream: string,
182
- options: LoggerOptions,
217
+ options: Told,
183
218
  out: (ConsoleOut & { utc?: boolean }) | undefined,
184
219
  ): { shared: Shared; spelled: Record<string, string> } {
185
220
  if (stream === '') {
@@ -194,7 +229,8 @@ function sharedOf(
194
229
  append,
195
230
  stream,
196
231
  most,
197
- least: read.level === undefined ? Number.NEGATIVE_INFINITY : levelOf(read.level),
232
+ kept: read.kept === undefined ? Number.NEGATIVE_INFINITY : levelOf(read.kept),
233
+ printed: read.printed === undefined ? undefined : levelOf(read.printed),
198
234
  redact: redactor(read.redact, read.keepUrlPasswords),
199
235
  replace: read.replace,
200
236
  source: read.source === true,
@@ -213,14 +249,16 @@ function sharedOf(
213
249
  type Unread = [name: string, value: string, expected: string]
214
250
 
215
251
  /**
216
- * LOG_LEVEL, LOG_FORMAT and LOG_TIME over the options, after the prefix env
217
- * names, as Go and Python read them; a console turned off stays off. Each
218
- * variable read is spelled as the config `log` reads it, by its name.
252
+ * LOG_LEVEL, LOG_CONSOLE_LEVEL, LOG_FORMAT and LOG_TIME over the options,
253
+ * each over its own, after the prefix env names, as Go and Python read them;
254
+ * a console turned off stays off. Each variable read is spelled as the
255
+ * config `log` reads it, by its name, but LOG_CONSOLE_LEVEL, which no
256
+ * setting of the config holds.
219
257
  */
220
- function fromEnvironment(
221
- options: LoggerOptions,
222
- ): [LoggerOptions, Unread[], Record<string, string>] {
223
- const read = { ...options }
258
+ function fromEnvironment(options: Told): [Read, Unread[], Record<string, string>] {
259
+ const { level: given, ...rest } = options
260
+ const [kept, printed] = partsOf(given)
261
+ const read: Read = { ...rest, kept, printed }
224
262
  const ignored: Unread[] = []
225
263
  const spelled: Record<string, string> = {}
226
264
  const env = options.env ?? environmentOf('')
@@ -229,15 +267,14 @@ function fromEnvironment(
229
267
  }
230
268
  const variables = environment(env)
231
269
  const name = (variable: string) => (env.prefix === '' ? variable : `${env.prefix}_${variable}`)
232
- const level = variables(name('LOG_LEVEL'))
270
+ const level = levelIn(variables, name('LOG_LEVEL'), ignored)
233
271
  if (level !== undefined) {
234
- const known = level.toLowerCase() === 'warning' ? 'warn' : level.toLowerCase()
235
- if (Object.hasOwn(levels, known)) {
236
- read.level = levels[known as keyof typeof levels]
237
- spelled[name('LOG_LEVEL')] = known
238
- } else {
239
- ignored.push([name('LOG_LEVEL'), level, 'trace, debug, info, warn or error'])
240
- }
272
+ read.kept = levels[level]
273
+ spelled[name('LOG_LEVEL')] = level
274
+ }
275
+ const shown = levelIn(variables, name('LOG_CONSOLE_LEVEL'), ignored)
276
+ if (shown !== undefined) {
277
+ read.printed = levels[shown]
241
278
  }
242
279
  const format = variables(name('LOG_FORMAT'))
243
280
  if (format !== undefined) {
@@ -262,6 +299,24 @@ function fromEnvironment(
262
299
  return [read, ignored, spelled]
263
300
  }
264
301
 
302
+ /** The level a variable names, the case ignored and warning read as warn; a value it cannot mean is ignored. */
303
+ function levelIn(
304
+ variables: (name: string) => string | undefined,
305
+ name: string,
306
+ ignored: Unread[],
307
+ ): keyof typeof levels | undefined {
308
+ const text = variables(name)
309
+ if (text === undefined) {
310
+ return undefined
311
+ }
312
+ const known = text.toLowerCase() === 'warning' ? 'warn' : text.toLowerCase()
313
+ if (Object.hasOwn(levels, known)) {
314
+ return known as keyof typeof levels
315
+ }
316
+ ignored.push([name, text, 'trace, debug, info, warn or error'])
317
+ return undefined
318
+ }
319
+
265
320
  /** The variables of env's files, the process's or `from`'s over them, a missing file skipped; an empty one is none. */
266
321
  function environment(env: EnvLayer): (name: string) => string | undefined {
267
322
  const fromFiles: Record<string, string> = {}
@@ -314,14 +369,17 @@ function forceColor(): boolean {
314
369
  }
315
370
 
316
371
  /** Whether a console told no format prints pretty lines: on a terminal, or where FORCE_COLOR asks. */
317
- function prettyUntold(options: LoggerOptions, out: ConsoleOut | undefined): boolean {
372
+ function prettyUntold(options: Pick<LoggerOptions, 'to'>, out: ConsoleOut | undefined): boolean {
318
373
  const stream: ConsoleOut = out ?? options.to ?? process.stderr
319
374
  // FORCE_COLOR other than 0 says a person reads a pipe, as an IDE's run console is one
320
375
  return stream.isTTY === true || (out === undefined && forceColor())
321
376
  }
322
377
 
323
378
  /** How a logger prints its lines, or undefined when it prints none. */
324
- function printer(options: LoggerOptions, out?: ConsoleOut & { utc?: boolean }): Shared['print'] {
379
+ function printer(
380
+ options: Pick<LoggerOptions, 'console' | 'to' | 'time' | 'hideStream'>,
381
+ out?: ConsoleOut & { utc?: boolean },
382
+ ): Shared['print'] {
325
383
  if (options.console === 'off') {
326
384
  return undefined
327
385
  }
@@ -436,12 +494,14 @@ export class Logger {
436
494
  }
437
495
 
438
496
  #log(level: number, message: string, attrs: Fields | undefined): void {
439
- if (level < this.#shared.least) {
497
+ // a line that no place of the logger takes is never encoded
498
+ if (!prints(this.#shared, level) && !keeps(this.#shared, level)) {
440
499
  return
441
500
  }
442
501
  this.#emit({ level, msg: message }, attrs)
443
502
  }
444
503
 
504
+ /** Prints a line from the level printed and keeps it from the level kept; an event, which has none, both. */
445
505
  #emit(what: Pick<ConsoleLine, 'level' | 'event' | 'msg'>, attrs: Fields | undefined): void {
446
506
  const trace = traceOf(currentTrace())
447
507
  const shared = this.#shared
@@ -454,8 +514,12 @@ export class Logger {
454
514
  attrs: shared.source ? [...encoded, sourceKey, sourceOfCaller()] : encoded,
455
515
  ...trace,
456
516
  }
457
- this.#shared.print?.(line)
458
- this.#queue(line)
517
+ if (what.level === undefined || prints(shared, what.level)) {
518
+ shared.print?.(line)
519
+ }
520
+ if (what.level === undefined || keeps(shared, what.level)) {
521
+ this.#queue(line)
522
+ }
459
523
  }
460
524
 
461
525
  #queue(line: ConsoleLine): void {
@@ -479,6 +543,16 @@ export class Logger {
479
543
  }
480
544
  }
481
545
 
546
+ /** Whether a logger prints a line of level: it has a console, and the level is the level printed or above. */
547
+ function prints(shared: Shared, level: number): boolean {
548
+ return shared.print !== undefined && level >= (shared.printed ?? shared.kept)
549
+ }
550
+
551
+ /** Whether a logger keeps a line of level: it is a store's, and the level is its level or above. */
552
+ function keeps(shared: Shared, level: number): boolean {
553
+ return shared.append !== undefined && level >= shared.kept
554
+ }
555
+
482
556
  /**
483
557
  * Writes what the buffer holds, and again while half of it filled during the
484
558
  * write before. A full buffer drops a line before the line could ask for a
@@ -491,21 +565,61 @@ async function writeWaiting(
491
565
  ): Promise<void> {
492
566
  let batch = shared.waiting.splice(0)
493
567
  while (batch.length > 0) {
494
- try {
495
- await append(batch)
496
- shared.failures?.succeeded()
497
- } catch (err) {
498
- shared.dropped += batch.length
499
- shared.failures?.failed(err)
568
+ for (const piece of pieces(batch)) {
569
+ try {
570
+ await append(piece)
571
+ shared.failures?.succeeded()
572
+ } catch (err) {
573
+ shared.dropped += piece.length
574
+ shared.failures?.failed(err)
575
+ }
500
576
  }
501
577
  batch = shared.waiting.length >= shared.most / 2 ? shared.waiting.splice(0) : []
502
578
  }
503
579
  }
504
580
 
581
+ /**
582
+ * The text a write carries at most. A request's body holds 2 MiB, and a
583
+ * buffer of long lines, or one made larger, held more: the whole write was
584
+ * refused and every line in it dropped. Counted in UTF-16 units, three bytes
585
+ * each at worst, so a piece stays within the body however its text is spelt.
586
+ */
587
+ const PIECE = 512 * 1024
588
+
589
+ /** The lines in writes of at most PIECE of text each, in their order. */
590
+ function pieces(lines: ConsoleLine[]): ConsoleLine[][] {
591
+ const out: ConsoleLine[][] = []
592
+ let piece: ConsoleLine[] = []
593
+ let size = 0
594
+ for (const line of lines) {
595
+ const weight = textOf(line)
596
+ if (piece.length > 0 && size + weight > PIECE) {
597
+ out.push(piece)
598
+ piece = []
599
+ size = 0
600
+ }
601
+ piece.push(line)
602
+ size += weight
603
+ }
604
+ if (piece.length > 0) {
605
+ out.push(piece)
606
+ }
607
+ return out
608
+ }
609
+
610
+ /** A line's text, with a little for its time, level and framing. */
611
+ function textOf(line: ConsoleLine): number {
612
+ let size = 64 + (line.msg?.length ?? 0) + (line.event?.length ?? 0)
613
+ for (const part of line.attrs) size += part.length
614
+ for (const part of line.context) size += part.length
615
+ return size
616
+ }
617
+
505
618
  /**
506
619
  * Fields as key, JSON pairs, as a console writes them and the store keeps
507
- * them: an Error's value its stack, which JSON would write as {}, a secret's
508
- * value hidden at any depth, and a URL's password unless kept.
620
+ * them: a field whose value is undefined left out, as JSON.stringify leaves
621
+ * it out of an object, an Error's value its stack, which JSON would write as
622
+ * {}, a secret's value hidden at any depth, and a URL's password unless kept.
509
623
  */
510
624
  export function encodeFields(
511
625
  fields: Fields,
@@ -518,6 +632,10 @@ export function encodeFields(
518
632
  const flat: string[] = []
519
633
  for (const [key, given] of pairs) {
520
634
  const value = replace === undefined ? given : replace(key, given)
635
+ // before it is hidden, as Rust's replace drops a field: a secret given no value is no field
636
+ if (value === undefined) {
637
+ continue
638
+ }
521
639
  const shown = value instanceof Error ? (value.stack ?? String(value)) : value
522
640
  if (redact?.hides(key)) {
523
641
  flat.push(key, redacted)
package/src/metrics.ts CHANGED
@@ -20,7 +20,7 @@ import {
20
20
  Timer,
21
21
  type TimerOptions,
22
22
  } from './instruments.ts'
23
- import { type Duration, type DurationText, ms, type Time, unixMs } from './time.ts'
23
+ import { type Calendar, type Duration, type DurationText, ms, type Time, unixMs } from './time.ts'
24
24
  import {
25
25
  Empty,
26
26
  MetricsAggregateAsked,
@@ -28,6 +28,7 @@ import {
28
28
  MetricsDamages,
29
29
  MetricsDeleted,
30
30
  MetricsDescription,
31
+ MetricsExposition,
31
32
  MetricsIngest,
32
33
  MetricsLatest,
33
34
  MetricsNameAsked,
@@ -102,6 +103,16 @@ export interface SeriesInput {
102
103
  keep?: Duration | undefined
103
104
  }
104
105
 
106
+ /** What an ingest of an exporter's text gives every series of it. */
107
+ export interface PrometheusOptions {
108
+ /** labels every series gets beside its own, such as the instance scraped; one the text gives too is refused, naming its line */
109
+ labels?: Record<string, LabelValue> | undefined
110
+ /** the time of every sample the text gives none; the store's clock unless given */
111
+ at?: Time | undefined
112
+ /** how long every name of the text keeps its samples */
113
+ keep?: Duration | undefined
114
+ }
115
+
105
116
  /** A series as a read gives it: its samples as two columns. */
106
117
  export interface Series {
107
118
  name: string
@@ -142,6 +153,10 @@ export type Op =
142
153
  export interface AggregateQuery<O extends Op = Op> extends Query {
143
154
  /** the buckets' span, counted from 1970 in UTC; the range is one bucket without it */
144
155
  every?: Duration | undefined
156
+ /** buckets of a calendar in timeZone, in place of every: 'day' begins each at a local midnight */
157
+ calendar?: Calendar | undefined
158
+ /** the IANA zone a calendar's buckets are in: 'Europe/Moscow', 'UTC' */
159
+ timeZone?: string | undefined
145
160
  op: O
146
161
  /** the labels a group keeps; without by and without, every series of a name is one group */
147
162
  by?: readonly string[] | undefined
@@ -206,7 +221,14 @@ export interface Plan {
206
221
  series: number
207
222
  bytes: number
208
223
  decoded: number
209
- exceeds: { limit: string; wanted: number; bound: number } | undefined
224
+ exceeds:
225
+ | {
226
+ /** series, bytes read, samples decoded or answer */
227
+ limit: string
228
+ wanted: number
229
+ bound: number
230
+ }
231
+ | undefined
210
232
  }
211
233
 
212
234
  /** A page's options: a prefix, how many, and where the page before ended. */
@@ -285,6 +307,33 @@ export class Metrics {
285
307
  )
286
308
  }
287
309
 
310
+ /**
311
+ * Reads the text an exporter serves, in Prometheus' format or OpenMetrics', and writes its
312
+ * series, every one or none: a counter as a counter, a histogram's _bucket series as
313
+ * histogram. A line it cannot read is refused, naming the line.
314
+ *
315
+ * const text = await (await fetch('http://web-1:9100/metrics')).text()
316
+ * await store.metrics.ingestPrometheus(text, { labels: { instance: 'web-1' } })
317
+ */
318
+ async ingestPrometheus(
319
+ text: string | Uint8Array,
320
+ options: PrometheusOptions = {},
321
+ ): Promise<void> {
322
+ const labels: Labels = {}
323
+ for (const [label, value] of Object.entries(options.labels ?? {})) {
324
+ labels[label] = String(value)
325
+ }
326
+ const body = MetricsExposition.encode({
327
+ text: typeof text === 'string' ? new TextEncoder().encode(text) : text,
328
+ labels,
329
+ at: options.at === undefined ? undefined : unixMs(options.at),
330
+ keep: options.keep === undefined ? undefined : ms(options.keep),
331
+ })
332
+ await this.#link.run('write', connection =>
333
+ connection.session.call(methods['metrics.ingestPrometheus'], body),
334
+ )
335
+ }
336
+
288
337
  /** The samples of each series the query finds, in its range: whole, or a limit error naming its bound. */
289
338
  async read(query: Query): Promise<Series[]> {
290
339
  const parts = await this.#download('metrics.read', MetricsQuery.encode(queryOf(query)))
@@ -572,6 +621,8 @@ function aggregateOf(query: AggregateQuery): Parameters<typeof MetricsAggregateA
572
621
  return {
573
622
  query: queryOf(query),
574
623
  every: query.every === undefined ? undefined : ms(query.every),
624
+ calendar: query.calendar,
625
+ timeZone: query.timeZone,
575
626
  op: query.op,
576
627
  by: query.by === undefined ? undefined : [...query.by],
577
628
  without: query.without === undefined ? undefined : [...query.without],