@tinyshed/tinystore 0.1.0-rc.5 → 0.1.0-rc.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (122) hide show
  1. package/README.md +37 -37
  2. package/bin/tinystore.js +12 -1
  3. package/dist/backup.d.ts +33 -0
  4. package/dist/backup.js +49 -0
  5. package/dist/blobs.d.ts +160 -79
  6. package/dist/blobs.js +510 -217
  7. package/dist/channels.d.ts +90 -0
  8. package/dist/channels.js +291 -0
  9. package/dist/clock.js +3 -3
  10. package/dist/config.d.ts +209 -86
  11. package/dist/config.js +581 -538
  12. package/dist/connection.d.ts +64 -2
  13. package/dist/connection.js +201 -16
  14. package/dist/console.d.ts +27 -3
  15. package/dist/console.js +165 -12
  16. package/dist/dotenv.d.ts +7 -0
  17. package/dist/dotenv.js +79 -0
  18. package/dist/errors.d.ts +61 -1
  19. package/dist/errors.js +81 -1
  20. package/dist/handles.d.ts +44 -2
  21. package/dist/handles.js +81 -2
  22. package/dist/index.d.ts +17 -11
  23. package/dist/index.js +5 -3
  24. package/dist/instruments.d.ts +177 -0
  25. package/dist/instruments.js +819 -0
  26. package/dist/jobs.d.ts +335 -239
  27. package/dist/jobs.js +716 -536
  28. package/dist/json.d.ts +2 -0
  29. package/dist/json.js +122 -0
  30. package/dist/kv.d.ts +130 -243
  31. package/dist/kv.js +273 -640
  32. package/dist/limits.d.ts +80 -0
  33. package/dist/limits.js +139 -0
  34. package/dist/locks.d.ts +62 -0
  35. package/dist/locks.js +237 -0
  36. package/dist/logger.d.ts +45 -9
  37. package/dist/logger.js +150 -38
  38. package/dist/metrics.d.ts +210 -209
  39. package/dist/metrics.js +319 -556
  40. package/dist/once.d.ts +19 -18
  41. package/dist/once.js +74 -63
  42. package/dist/query.d.ts +181 -0
  43. package/dist/query.js +788 -0
  44. package/dist/records.d.ts +241 -150
  45. package/dist/records.js +507 -291
  46. package/dist/runtime/addon.d.ts +14 -0
  47. package/dist/runtime/addon.js +69 -0
  48. package/dist/runtime/bun.js +5 -7
  49. package/dist/runtime/node.js +5 -7
  50. package/dist/runtime/pipe.d.ts +17 -0
  51. package/dist/runtime/pipe.js +133 -0
  52. package/dist/runtime/shared.d.ts +6 -0
  53. package/dist/runtime/shared.js +19 -0
  54. package/dist/runtime.d.ts +5 -2
  55. package/dist/search.d.ts +189 -0
  56. package/dist/search.js +281 -0
  57. package/dist/session.d.ts +31 -3
  58. package/dist/session.js +153 -41
  59. package/dist/sql.d.ts +228 -78
  60. package/dist/sql.js +745 -240
  61. package/dist/store.d.ts +244 -24
  62. package/dist/store.js +270 -67
  63. package/dist/time.d.ts +5 -1
  64. package/dist/time.js +22 -2
  65. package/dist/tx.d.ts +61 -0
  66. package/dist/tx.js +314 -0
  67. package/dist/wire/codec.d.ts +23 -7
  68. package/dist/wire/codec.js +56 -35
  69. package/dist/wire/frame.d.ts +2 -0
  70. package/dist/wire/frame.js +31 -14
  71. package/dist/wire/messages.d.ts +259 -0
  72. package/dist/wire/messages.js +22 -0
  73. package/dist/wire/msgpack.d.ts +49 -1
  74. package/dist/wire/msgpack.js +293 -56
  75. package/dist/wire/protocol.d.ts +2683 -0
  76. package/dist/wire/protocol.js +10846 -0
  77. package/package.json +39 -38
  78. package/src/backup.ts +81 -0
  79. package/src/blobs.ts +678 -301
  80. package/src/channels.ts +377 -0
  81. package/src/clock.ts +4 -4
  82. package/src/config.ts +902 -612
  83. package/src/connection.ts +279 -21
  84. package/src/console.ts +180 -12
  85. package/src/dotenv.ts +82 -0
  86. package/src/errors.ts +105 -0
  87. package/src/handles.ts +157 -8
  88. package/src/index.ts +158 -52
  89. package/src/instruments.ts +1042 -0
  90. package/src/jobs.ts +1017 -733
  91. package/src/json.ts +133 -0
  92. package/src/kv.ts +424 -847
  93. package/src/limits.ts +202 -0
  94. package/src/locks.ts +320 -0
  95. package/src/logger.ts +209 -40
  96. package/src/metrics.ts +557 -752
  97. package/src/once.ts +86 -83
  98. package/src/query.ts +1005 -0
  99. package/src/records.ts +795 -418
  100. package/src/runtime/addon.ts +99 -0
  101. package/src/runtime/bun.ts +5 -7
  102. package/src/runtime/node.ts +5 -7
  103. package/src/runtime/pipe.ts +178 -0
  104. package/src/runtime/shared.ts +20 -0
  105. package/src/runtime.ts +5 -2
  106. package/src/search.ts +480 -0
  107. package/src/session.ts +187 -41
  108. package/src/sql.ts +993 -306
  109. package/src/store.ts +473 -78
  110. package/src/time.ts +48 -3
  111. package/src/tx.ts +426 -0
  112. package/src/wire/codec.ts +78 -37
  113. package/src/wire/frame.ts +44 -14
  114. package/src/wire/messages.ts +25 -0
  115. package/src/wire/msgpack.ts +327 -56
  116. package/src/wire/protocol.ts +11709 -0
  117. package/dist/limiter.d.ts +0 -45
  118. package/dist/limiter.js +0 -68
  119. package/dist/quota.d.ts +0 -44
  120. package/dist/quota.js +0 -98
  121. package/src/limiter.ts +0 -99
  122. package/src/quota.ts +0 -137
package/src/connection.ts CHANGED
@@ -4,14 +4,71 @@
4
4
  // remote server.
5
5
 
6
6
  import { randomBytes } from 'node:crypto'
7
+ import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'
7
8
  import { join, resolve } from 'node:path'
8
9
 
9
10
  import manifest from '../package.json' with { type: 'json' }
10
11
  import { currentSignal } from './cancel.ts'
11
- import { ClosedError, OutcomeUnknownError, TinystoreError, UnavailableError } from './errors.ts'
12
+ import {
13
+ ClosedError,
14
+ InvalidError,
15
+ OutcomeUnknownError,
16
+ TinystoreError,
17
+ UnavailableError,
18
+ } from './errors.ts'
12
19
  import type { PrivateChild, Runtime, TlsOptions, Transport } from './runtime.ts'
13
- import { LostError, Session, type SessionOptions, type Stream, watch } from './session.ts'
14
- import { Empty, methods } from './wire/messages.ts'
20
+ import {
21
+ ConnectionLostError,
22
+ protocol,
23
+ Session,
24
+ type SessionOptions,
25
+ type Stream,
26
+ watch,
27
+ } from './session.ts'
28
+ import type { NodeOptions } from './store.ts'
29
+ import { Empty, methods, StoreOptions } from './wire/protocol.ts'
30
+
31
+ /**
32
+ * How far a commit goes before it returns, for the files of a store or of
33
+ * one database:
34
+ *
35
+ * 'full' synced to the disk: it survives a crash of the operating
36
+ * system and a power loss. What every file is unless told.
37
+ * 'os' written to the operating system, synced a little later: it
38
+ * survives the program's crash, and a power loss may take the
39
+ * last commits, never the file.
40
+ */
41
+ export type Durability = 'full' | 'os'
42
+
43
+ /** Refuses a durability that is neither word, before anything leaves. */
44
+ export function checkDurability(durability: Durability | undefined, of: string): void {
45
+ if (durability !== undefined && durability !== 'full' && durability !== 'os') {
46
+ throw new InvalidError(`${of}: durability ${JSON.stringify(durability)}: it is 'full' or 'os'`)
47
+ }
48
+ }
49
+
50
+ /**
51
+ * Dials as `dial` does, and refuses a store whose files commit another way
52
+ * than this program asked for: a store has one durability, its first
53
+ * opener's, and a program that needs another must not be told it has it.
54
+ */
55
+ export function asking(dial: Dialer, dir: string, durability: Durability | undefined): Dialer {
56
+ if (durability === undefined) {
57
+ return dial
58
+ }
59
+ return async () => {
60
+ const connection = await dial()
61
+ const opened = (await connection.session.welcomed).durability
62
+ if (opened !== durability) {
63
+ await connection.close()
64
+ const as = opened === undefined ? 'with none said' : `with durability '${opened}'`
65
+ throw new InvalidError(
66
+ `the store in ${dir} is open ${as}, and this program asked for '${durability}'`,
67
+ )
68
+ }
69
+ return connection
70
+ }
71
+ }
15
72
 
16
73
  /** How long a server may take to answer HELLO. */
17
74
  const handshakeTime = 5000
@@ -23,6 +80,8 @@ export class Connection {
23
80
  readonly #closing: () => Promise<unknown>
24
81
  /** the handles this connection opened, by what they are: a bucket's name and options */
25
82
  readonly handles = new Map<string, Promise<number>>()
83
+ /** the handle each open was answered, once it was: what a call made in place reads without waiting */
84
+ readonly opened = new Map<object, number>()
26
85
 
27
86
  constructor(
28
87
  session: Session,
@@ -64,6 +123,13 @@ export class Connection {
64
123
  return connection
65
124
  }
66
125
 
126
+ /** Shakes hands over a pipe to the core in this process. */
127
+ static async overPipe(session: Session, transport: Transport): Promise<Connection> {
128
+ const connection = new Connection(session, transport)
129
+ await connection.#handshake()
130
+ return connection
131
+ }
132
+
67
133
  /**
68
134
  * Shakes hands over a private child's stdin and stdout. Closing waits for
69
135
  * the child's exit, since until then it holds the directory: its LOCK, and
@@ -132,13 +198,13 @@ export function sidecar(
132
198
  runtime: Runtime,
133
199
  dir: string,
134
200
  binary: () => string,
135
- idle?: number,
201
+ started: Started = {},
136
202
  older: (server: string) => boolean = server => olderRelease(server, manifest.version),
137
203
  ): Dialer {
138
204
  const absolute = resolve(dir)
139
205
  const serve = join(absolute, 'server', 'SERVE')
140
206
  const log = join(absolute, 'server', 'serve.log')
141
- const idling = idle === undefined ? [] : ['--idle', `${idle}ms`]
207
+ const idling = started.idle === undefined ? [] : ['--idle', `${started.idle}ms`]
142
208
  let told = false
143
209
  return async () => {
144
210
  const found = await reachServe(runtime, serve)
@@ -159,11 +225,103 @@ export function sidecar(
159
225
  return found.connection
160
226
  }
161
227
  }
162
- const command = [binary(), 'serve', '--dir', absolute, '--local', '--log', log, ...idling]
228
+ const flags = [...idling, ...storeFlags(started, absolute)]
229
+ const command = [binary(), 'serve', '--dir', absolute, '--local', '--log-file', log, ...flags]
163
230
  return startSidecar(runtime, serve, log, command, stoppedInstance)
164
231
  }
165
232
  }
166
233
 
234
+ /** What a server this process starts is started with; one found running keeps its own. */
235
+ export interface Started {
236
+ /** how long a sidecar stays once its last connection has gone, in milliseconds */
237
+ idle?: number | undefined
238
+ durability?: Durability | undefined
239
+ /** the file of the store's encryption key, when it is not the store's own */
240
+ encryptionKeyFile?: string | undefined
241
+ /** the node the store is, when it is one */
242
+ node?: NodeOptions | undefined
243
+ /** measures the store into its own metrics, the machine too unless it says system: false */
244
+ selfMetrics?: SelfMetrics | undefined
245
+ }
246
+
247
+ /**
248
+ * What a store measures of itself: `true` its commits, files and memory and
249
+ * the machine it runs on, `{ system: false }` the same without the machine.
250
+ */
251
+ export type SelfMetrics = boolean | { system?: boolean }
252
+
253
+ /** What selfMetrics asks for, none when it asks for nothing. */
254
+ function selfMetricsOf(given: SelfMetrics | undefined): { system: boolean } | undefined {
255
+ if (given === undefined || given === false) {
256
+ return undefined
257
+ }
258
+ return { system: given === true || given.system !== false }
259
+ }
260
+
261
+ /** The flags of `tinystore serve` that say how the store in `dir` opens. */
262
+ function storeFlags(started: Started, dir: string): string[] {
263
+ const committing = started.durability === undefined ? [] : ['--durability', started.durability]
264
+ const key = started.encryptionKeyFile
265
+ const keyed =
266
+ key === undefined ? committing : [...committing, '--encryption-key-file', resolve(key)]
267
+ const asked = selfMetricsOf(started.selfMetrics)
268
+ const measured =
269
+ asked === undefined
270
+ ? keyed
271
+ : [...keyed, '--self-metrics', ...(asked.system ? [] : ['--no-system-metrics'])]
272
+ return [...measured, ...nodeFlags(started.node, join(dir, 'server', 'node.toml'))]
273
+ }
274
+
275
+ /**
276
+ * A node's options as `tinystore serve` takes them: where it listens and the
277
+ * nodes it copies as flags, and the rest in a node config written to
278
+ * `config`, which the server reads at its start.
279
+ */
280
+ function nodeFlags(node: NodeOptions | undefined, config: string): string[] {
281
+ if (node === undefined) {
282
+ return []
283
+ }
284
+ if (node.token !== undefined) {
285
+ throw new InvalidError(
286
+ "a node's token reaches a server this process starts through a file: give tokenFile, or open embedded",
287
+ )
288
+ }
289
+ const flags: string[] = []
290
+ for (const from of node.from ?? []) {
291
+ flags.push('--from', from)
292
+ }
293
+ if (node.serve !== undefined) {
294
+ flags.push('--node-listen', node.serve)
295
+ }
296
+ // a JSON string or list of strings is TOML's too
297
+ const settings: string[] = []
298
+ if (node.files !== undefined) {
299
+ settings.push(`files = ${JSON.stringify(node.files)}`)
300
+ }
301
+ if (node.tokenFile !== undefined) {
302
+ settings.push(`token-file = ${JSON.stringify(resolve(node.tokenFile))}`)
303
+ }
304
+ if (node.synchronous === true) {
305
+ settings.push('synchronous = true')
306
+ }
307
+ if (node.synchronousWith !== undefined && node.synchronousWith.length > 0) {
308
+ settings.push(`synchronous-with = ${JSON.stringify(node.synchronousWith)}`)
309
+ }
310
+ if (node.insecure === true) {
311
+ settings.push('insecure = true')
312
+ }
313
+ if (node.catalog === true) {
314
+ settings.push('catalog = true')
315
+ }
316
+ if (node.advertise !== undefined) {
317
+ settings.push(`advertise = ${JSON.stringify(node.advertise)}`)
318
+ }
319
+ mkdirSync(join(config, '..'), { recursive: true })
320
+ writeFileSync(config, `${settings.join('\n')}\n`)
321
+ flags.push('--node-config', config)
322
+ return flags
323
+ }
324
+
167
325
  /**
168
326
  * Starts a sidecar and waits for SERVE to name one that answers, its own or
169
327
  * another client's that won the start. One that exits with 3 found the
@@ -313,7 +471,7 @@ async function reachServe(
313
471
  }
314
472
  const endpoint = published.endpoints?.[0]
315
473
  const secret = Buffer.from(published.secret ?? '', 'base64url')
316
- if (published.protocol !== 1 || endpoint === undefined || secret.length !== 32) {
474
+ if (published.protocol !== protocol || endpoint === undefined || secret.length !== 32) {
317
475
  return undefined
318
476
  }
319
477
  try {
@@ -347,12 +505,21 @@ export function privateChild(
347
505
  dir: string,
348
506
  binary: () => string,
349
507
  clock?: Date,
508
+ started: Started = {},
350
509
  ): Dialer {
351
510
  // a test's clock starts where the first child's did, and a child started again starts it there
352
511
  const clocked = clock === undefined ? [] : ['--clock', clock.toISOString()]
353
512
  return async () => {
354
513
  let session: Session | undefined
355
- const argv = [binary(), 'serve', '--dir', resolve(dir), '--stdio', ...clocked]
514
+ const argv = [
515
+ binary(),
516
+ 'serve',
517
+ '--dir',
518
+ resolve(dir),
519
+ '--stdio',
520
+ ...clocked,
521
+ ...storeFlags(started, resolve(dir)),
522
+ ]
356
523
  const child = runtime.spawnPrivate(argv, {
357
524
  data: bytes => session?.receive(bytes),
358
525
  end: err => session?.end(err),
@@ -373,6 +540,58 @@ export function privateChild(
373
540
  }
374
541
  }
375
542
 
543
+ /**
544
+ * The core in this process: no server and no socket, the frames handed to
545
+ * the core and back, through bun:ffi under Bun and its addon under Node.
546
+ */
547
+ export function embedded(
548
+ runtime: Runtime,
549
+ dir: string,
550
+ library: () => string,
551
+ started: Started = {},
552
+ ): Dialer {
553
+ const encryptionKeyFile =
554
+ started.encryptionKeyFile === undefined ? undefined : resolve(started.encryptionKeyFile)
555
+ const node = started.node
556
+ const token =
557
+ node?.token ??
558
+ (node?.tokenFile === undefined
559
+ ? undefined
560
+ : readFileSync(resolve(node.tokenFile), 'utf8').trim())
561
+ const options = StoreOptions.encode({
562
+ durability: started.durability,
563
+ encryptionKeyFile,
564
+ node: node !== undefined,
565
+ from: node?.from ?? [],
566
+ serve: node?.serve,
567
+ files: node?.files ?? [],
568
+ token,
569
+ synchronous: node?.synchronous === true,
570
+ catalog: node?.catalog === true,
571
+ insecure: node?.insecure === true,
572
+ advertise: node?.advertise,
573
+ synchronousWith: node?.synchronousWith ?? [],
574
+ selfMetrics: selfMetricsOf(started.selfMetrics) !== undefined,
575
+ withoutSystem: selfMetricsOf(started.selfMetrics)?.system === false,
576
+ })
577
+ return async () => {
578
+ let session: Session | undefined
579
+ const end = (err: Error) => session?.end(err)
580
+ const transport =
581
+ typeof Bun === 'undefined'
582
+ ? (await import('./runtime/addon.ts')).openAddon(library(), resolve(dir), options, {
583
+ data: bytes => session?.receive(bytes),
584
+ end,
585
+ })
586
+ : (await import('./runtime/pipe.ts')).openPipe(library(), resolve(dir), options, {
587
+ frames: (bytes, length) => session?.read(bytes, length) ?? length,
588
+ end,
589
+ })
590
+ session = new Session(bytes => transport.write(bytes), { client: runtime.client })
591
+ return await Connection.overPipe(session, transport)
592
+ }
593
+ }
594
+
376
595
  /** A remote server: tls:// checks its certificate before the token leaves; tcp:// does not. */
377
596
  export function remote(runtime: Runtime, url: string, token: string, tls?: TlsOptions): Dialer {
378
597
  return () => Connection.dial(runtime, url, { client: runtime.client, token }, tls)
@@ -390,6 +609,7 @@ export type Idempotence = 'read' | 'write'
390
609
  export class Link {
391
610
  readonly #dial: Dialer
392
611
  #current: Promise<Connection> | undefined
612
+ #dialled: Connection | undefined
393
613
  #closed = false
394
614
 
395
615
  constructor(dial: Dialer) {
@@ -411,6 +631,12 @@ export class Link {
411
631
  return this.#redial(undefined)
412
632
  }
413
633
 
634
+ /** The connection a call goes on, when one is there to take it without waiting. */
635
+ ready(): Connection | undefined {
636
+ const c = this.#dialled
637
+ return c !== undefined && c.session.ended === undefined && !c.session.goingAway ? c : undefined
638
+ }
639
+
414
640
  // one dial at a time: every caller that found the same dead connection
415
641
  // waits for the one new one
416
642
  #redial(dead: Promise<Connection> | undefined): Promise<Connection> {
@@ -419,11 +645,19 @@ export class Link {
419
645
  }
420
646
  const dialled = this.#dial()
421
647
  this.#current = dialled
422
- dialled.catch(() => {
423
- if (this.#current === dialled) {
424
- this.#current = undefined
425
- }
426
- })
648
+ this.#dialled = undefined
649
+ dialled.then(
650
+ connection => {
651
+ if (this.#current === dialled) {
652
+ this.#dialled = connection
653
+ }
654
+ },
655
+ () => {
656
+ if (this.#current === dialled) {
657
+ this.#current = undefined
658
+ }
659
+ },
660
+ )
427
661
  return dialled
428
662
  }
429
663
 
@@ -442,14 +676,8 @@ export class Link {
442
676
  try {
443
677
  return await attempt(connection)
444
678
  } catch (err) {
445
- if (tries < 2 && (err instanceof LostError || err instanceof UnavailableError)) {
446
- const sent = err instanceof LostError && err.sent
447
- if (!sent || idempotence === 'read') {
448
- continue
449
- }
450
- throw new OutcomeUnknownError(
451
- `the connection was lost with the write in flight; read what it wrote before writing again: ${err.message}`,
452
- )
679
+ if (tries < 2 && goesAgain(err, idempotence)) {
680
+ continue
453
681
  }
454
682
  throw err
455
683
  }
@@ -461,6 +689,7 @@ export class Link {
461
689
  this.#closed = true
462
690
  const current = this.#current
463
691
  this.#current = undefined
692
+ this.#dialled = undefined
464
693
  await current?.then(
465
694
  c => c.close(),
466
695
  () => {},
@@ -468,6 +697,24 @@ export class Link {
468
697
  }
469
698
  }
470
699
 
700
+ /**
701
+ * Whether a call that failed goes on another connection: one whose REQUEST
702
+ * had not left, and a read that had. A write that had is OutcomeUnknownError,
703
+ * thrown here; any other failure is the call's own.
704
+ */
705
+ export function goesAgain(err: unknown, idempotence: Idempotence): boolean {
706
+ if (!(err instanceof ConnectionLostError || err instanceof UnavailableError)) {
707
+ return false
708
+ }
709
+ const sent = err instanceof ConnectionLostError && err.sent
710
+ if (!sent || idempotence === 'read') {
711
+ return true
712
+ }
713
+ throw new OutcomeUnknownError(
714
+ `the connection was lost with the write in flight; read what it wrote before writing again: ${err.message}`,
715
+ )
716
+ }
717
+
471
718
  /** What a download sent: its header, its items, and the trailer that ended it. */
472
719
  export interface Downloaded {
473
720
  header: Uint8Array
@@ -510,3 +757,14 @@ export async function download(
510
757
  }
511
758
 
512
759
  export type { Stream }
760
+
761
+ /**
762
+ * The bytes a piece carries: half what the server takes on a stream before
763
+ * it grants more, so that one piece goes while the last is written, less a
764
+ * piece's own few bytes.
765
+ */
766
+ export function pieceOf(connection: Connection): number {
767
+ const agreed = connection.session.agreed
768
+ const most = Math.min(agreed?.maxBody ?? 1 << 20, agreed?.streamCredit ?? 1 << 20)
769
+ return Math.max(1, Math.floor(most / 2) - 16)
770
+ }
package/src/console.ts CHANGED
@@ -1,5 +1,5 @@
1
- // A logger's console line, as Go's records.Handler and Python's handler write
2
- // it, byte for byte the lines of records/testdata/console.json: pretty for a
1
+ // A logger's console line, as Rust's console and Python's handler write it,
2
+ // byte for byte the lines of testdata/console/console.json: pretty for a
3
3
  // person at a terminal, one JSON object a line for a collector.
4
4
  //
5
5
  // 11:02:11.123 WARN api slow request requestId=7f3a ms=1200
@@ -108,7 +108,7 @@ export function prettyLine(line: ConsoleLine, look: Look): string {
108
108
  const below: [string, string][] = []
109
109
  for (const fields of [line.context, line.attrs]) {
110
110
  for (let i = 0; i < fields.length; i += 2) {
111
- const [shown, lines] = prettyValue(fields[i + 1] as string)
111
+ const [shown, lines] = prettyValue(fields[i] as string, fields[i + 1] as string)
112
112
  if (lines) {
113
113
  below.push([fields[i] as string, shown])
114
114
  } else {
@@ -166,7 +166,13 @@ function prettyMessage(line: ConsoleLine): string {
166
166
  }
167
167
 
168
168
  /** How a field's JSON shows, and whether it is text of several lines, which goes under the line. */
169
- function prettyValue(json: string): [string, boolean] {
169
+ function prettyValue(key: string, json: string): [string, boolean] {
170
+ if (key === sourceKey) {
171
+ const place = shortSource(json)
172
+ if (place !== undefined) {
173
+ return [place, false]
174
+ }
175
+ }
170
176
  if (!json.startsWith('"')) {
171
177
  return [json, false]
172
178
  }
@@ -223,21 +229,183 @@ function stamp(at: Date, look: Look): string {
223
229
  return `${String(year).padStart(4, '0')}-${two(month)}-${two(day)} ${clock} ${zone}`
224
230
  }
225
231
 
232
+ /** The field where a line says where it was logged, as slog's handlers spell it. */
233
+ export const sourceKey = 'source'
234
+
226
235
  /**
227
- * Says whether a field's key hides its value: the key, or the part of a dotted
228
- * key after its last dot, is one of the names, the case ignored.
236
+ * A source as a pretty line shows it, its file's directory and name and its
237
+ * line: {"function":"run","file":"/home/ann/app/server/main.ts","line":42} is
238
+ * server/main.ts:42.
239
+ */
240
+ function shortSource(json: string): string | undefined {
241
+ if (!json.startsWith('{')) {
242
+ return undefined
243
+ }
244
+ const source = JSON.parse(json) as { file?: unknown; line?: unknown }
245
+ if (typeof source.file !== 'string' || source.file === '') {
246
+ return undefined
247
+ }
248
+ const parts = source.file.split(/[\\/]+/).filter(part => part !== '')
249
+ const line = typeof source.line === 'number' ? source.line : 0
250
+ return `${parts.slice(-2).join('/')}:${line}`
251
+ }
252
+
253
+ /** The names `redact` takes to hide the usual secrets, as Go's console.Secrets and Python's SECRETS. */
254
+ export const secrets: readonly string[] = Object.freeze([
255
+ 'password',
256
+ 'passwd',
257
+ 'passphrase',
258
+ 'secret',
259
+ 'token',
260
+ 'credential',
261
+ 'credentials',
262
+ 'authorization',
263
+ 'cookie',
264
+ 'api key',
265
+ 'private key',
266
+ 'secret key',
267
+ 'access key',
268
+ 'signing key',
269
+ 'encryption key',
270
+ 'connection string',
271
+ 'dsn',
272
+ ])
273
+
274
+ /** What a logger hides: the keys its names name, and a URL's password unless it keeps them. */
275
+ export interface Redactor {
276
+ hides(key: string): boolean
277
+ urls: boolean
278
+ }
279
+
280
+ /**
281
+ * Hides the values of fields whose keys name a secret: some of the key's
282
+ * words in a row, written together, are a name's. DB_PASSWORD, PasswordHash,
283
+ * api-key and APIKey split into words at '_', '-', '.', spaces and capitals,
284
+ * the case ignored, so 'api key' hides api_key and apiKey, and 'token' hides
285
+ * bot_token and not tokens_used.
229
286
  */
230
287
  export function redactor(
231
288
  names: readonly string[] | undefined,
232
- ): ((key: string) => boolean) | undefined {
233
- if (names === undefined || names.length === 0) {
289
+ keepUrlPasswords = false,
290
+ ): Redactor | undefined {
291
+ const joined = (names ?? []).map(name => words(name).join('')).filter(name => name !== '')
292
+ if (joined.length === 0 && keepUrlPasswords) {
234
293
  return undefined
235
294
  }
236
- const hidden = new Set(names.map(name => name.toLowerCase()))
237
- return key => {
238
- const lower = key.toLowerCase()
239
- return hidden.has(lower) || hidden.has(lower.slice(lower.lastIndexOf('.') + 1))
295
+ return {
296
+ hides: key => joined.length > 0 && joined.some(name => joinsTo(words(key), name)),
297
+ urls: !keepUrlPasswords,
298
+ }
299
+ }
300
+
301
+ /**
302
+ * A key's words as a person reads them, in lower case: split at anything but
303
+ * a letter or a digit, and where a capital begins a word, as in passwordHash
304
+ * and APIKey.
305
+ */
306
+ export function words(key: string): string[] {
307
+ const out: string[] = []
308
+ const chars = [...key]
309
+ let start = -1
310
+ const upper = (c: string | undefined) => c !== undefined && /\p{Lu}/u.test(c)
311
+ const lower = (c: string | undefined) => c !== undefined && /\p{Ll}/u.test(c)
312
+ const digit = (c: string | undefined) => c !== undefined && /\p{N}/u.test(c)
313
+ for (let i = 0; i < chars.length; i++) {
314
+ const c = chars[i] as string
315
+ if (!/[\p{L}\p{N}]/u.test(c)) {
316
+ if (start >= 0) {
317
+ out.push(chars.slice(start, i).join('').toLowerCase())
318
+ start = -1
319
+ }
320
+ continue
321
+ }
322
+ const before = chars[i - 1]
323
+ const begins =
324
+ upper(c) && (lower(before) || digit(before) || (upper(before) && lower(chars[i + 1])))
325
+ if (start >= 0 && begins) {
326
+ out.push(chars.slice(start, i).join('').toLowerCase())
327
+ start = i
328
+ }
329
+ if (start < 0) {
330
+ start = i
331
+ }
332
+ }
333
+ if (start >= 0) {
334
+ out.push(chars.slice(start).join('').toLowerCase())
335
+ }
336
+ return out
337
+ }
338
+
339
+ function joinsTo(keyWords: string[], name: string): boolean {
340
+ for (let i = 0; i < keyWords.length; i++) {
341
+ let rest = name
342
+ for (const word of keyWords.slice(i)) {
343
+ if (!rest.startsWith(word)) {
344
+ break
345
+ }
346
+ rest = rest.slice(word.length)
347
+ if (rest === '') {
348
+ return true
349
+ }
350
+ }
351
+ }
352
+ return false
353
+ }
354
+
355
+ /**
356
+ * JSON text with the password of each URL inside it hidden, everything else
357
+ * as it was spelled: "postgres://ann:hunter2@db/app" is
358
+ * "postgres://ann:[redacted]@db/app".
359
+ */
360
+ export function hideUrlPasswords(text: string): string {
361
+ if (!text.includes('://') && !text.includes('\\')) {
362
+ return text
363
+ }
364
+ return text.replace(/"(?:[^"\\]|\\.)*"/g, raw => {
365
+ const value: string = JSON.parse(raw)
366
+ const hidden = hideUrlTextPasswords(value)
367
+ return hidden === value ? raw : JSON.stringify(hidden)
368
+ })
369
+ }
370
+
371
+ function hideUrlTextPasswords(text: string): string {
372
+ let out = ''
373
+ let written = 0
374
+ for (let from = 0; from < text.length; ) {
375
+ const schemeEnd = text.indexOf('://', from)
376
+ if (schemeEnd < 0) {
377
+ break
378
+ }
379
+ const [password, at, end] = findPassword(text, schemeEnd + 3)
380
+ from = end
381
+ if (password < 0 || !/[A-Za-z][A-Za-z0-9+.-]*$/.test(text.slice(0, schemeEnd))) {
382
+ continue
383
+ }
384
+ out += `${text.slice(written, password)}[redacted]`
385
+ written = at
386
+ }
387
+ return written === 0 ? text : out + text.slice(written)
388
+ }
389
+
390
+ /** Where an authority's password begins and the '@' after it, -1 when it has none, and where it ends. */
391
+ function findPassword(text: string, start: number): [number, number, number] {
392
+ let at = -1
393
+ let colon = -1
394
+ let i = start
395
+ while (i < text.length) {
396
+ const c = text[i] as string
397
+ if (c === '/' || c === '?' || c === '#' || c <= ' ') {
398
+ break
399
+ }
400
+ if (c === '@') {
401
+ at = i
402
+ } else if (c === ':' && colon < 0) {
403
+ colon = i
404
+ }
405
+ i++
240
406
  }
407
+ const end = Math.min(i, text.length)
408
+ return at < 0 || colon < 0 || colon + 1 >= at ? [-1, -1, end] : [colon + 1, at, end]
241
409
  }
242
410
 
243
411
  /** What a hidden value becomes, in the store and on the console. */