@tinyshed/tinystore 0.1.0-rc.5 → 0.1.0-rc.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (122) hide show
  1. package/README.md +37 -37
  2. package/bin/tinystore.js +12 -1
  3. package/dist/backup.d.ts +33 -0
  4. package/dist/backup.js +49 -0
  5. package/dist/blobs.d.ts +160 -79
  6. package/dist/blobs.js +510 -217
  7. package/dist/channels.d.ts +90 -0
  8. package/dist/channels.js +291 -0
  9. package/dist/clock.js +3 -3
  10. package/dist/config.d.ts +209 -86
  11. package/dist/config.js +581 -538
  12. package/dist/connection.d.ts +64 -2
  13. package/dist/connection.js +201 -16
  14. package/dist/console.d.ts +27 -3
  15. package/dist/console.js +165 -12
  16. package/dist/dotenv.d.ts +7 -0
  17. package/dist/dotenv.js +79 -0
  18. package/dist/errors.d.ts +61 -1
  19. package/dist/errors.js +81 -1
  20. package/dist/handles.d.ts +44 -2
  21. package/dist/handles.js +81 -2
  22. package/dist/index.d.ts +17 -11
  23. package/dist/index.js +5 -3
  24. package/dist/instruments.d.ts +177 -0
  25. package/dist/instruments.js +819 -0
  26. package/dist/jobs.d.ts +335 -239
  27. package/dist/jobs.js +716 -536
  28. package/dist/json.d.ts +2 -0
  29. package/dist/json.js +122 -0
  30. package/dist/kv.d.ts +130 -243
  31. package/dist/kv.js +273 -640
  32. package/dist/limits.d.ts +80 -0
  33. package/dist/limits.js +139 -0
  34. package/dist/locks.d.ts +62 -0
  35. package/dist/locks.js +237 -0
  36. package/dist/logger.d.ts +45 -9
  37. package/dist/logger.js +150 -38
  38. package/dist/metrics.d.ts +210 -209
  39. package/dist/metrics.js +319 -556
  40. package/dist/once.d.ts +19 -18
  41. package/dist/once.js +74 -63
  42. package/dist/query.d.ts +181 -0
  43. package/dist/query.js +788 -0
  44. package/dist/records.d.ts +241 -150
  45. package/dist/records.js +507 -291
  46. package/dist/runtime/addon.d.ts +14 -0
  47. package/dist/runtime/addon.js +69 -0
  48. package/dist/runtime/bun.js +5 -7
  49. package/dist/runtime/node.js +5 -7
  50. package/dist/runtime/pipe.d.ts +17 -0
  51. package/dist/runtime/pipe.js +133 -0
  52. package/dist/runtime/shared.d.ts +6 -0
  53. package/dist/runtime/shared.js +19 -0
  54. package/dist/runtime.d.ts +5 -2
  55. package/dist/search.d.ts +189 -0
  56. package/dist/search.js +281 -0
  57. package/dist/session.d.ts +31 -3
  58. package/dist/session.js +153 -41
  59. package/dist/sql.d.ts +228 -78
  60. package/dist/sql.js +745 -240
  61. package/dist/store.d.ts +244 -24
  62. package/dist/store.js +270 -67
  63. package/dist/time.d.ts +5 -1
  64. package/dist/time.js +22 -2
  65. package/dist/tx.d.ts +61 -0
  66. package/dist/tx.js +314 -0
  67. package/dist/wire/codec.d.ts +23 -7
  68. package/dist/wire/codec.js +56 -35
  69. package/dist/wire/frame.d.ts +2 -0
  70. package/dist/wire/frame.js +31 -14
  71. package/dist/wire/messages.d.ts +259 -0
  72. package/dist/wire/messages.js +22 -0
  73. package/dist/wire/msgpack.d.ts +49 -1
  74. package/dist/wire/msgpack.js +293 -56
  75. package/dist/wire/protocol.d.ts +2683 -0
  76. package/dist/wire/protocol.js +10846 -0
  77. package/package.json +39 -38
  78. package/src/backup.ts +81 -0
  79. package/src/blobs.ts +678 -301
  80. package/src/channels.ts +377 -0
  81. package/src/clock.ts +4 -4
  82. package/src/config.ts +902 -612
  83. package/src/connection.ts +279 -21
  84. package/src/console.ts +180 -12
  85. package/src/dotenv.ts +82 -0
  86. package/src/errors.ts +105 -0
  87. package/src/handles.ts +157 -8
  88. package/src/index.ts +158 -52
  89. package/src/instruments.ts +1042 -0
  90. package/src/jobs.ts +1017 -733
  91. package/src/json.ts +133 -0
  92. package/src/kv.ts +424 -847
  93. package/src/limits.ts +202 -0
  94. package/src/locks.ts +320 -0
  95. package/src/logger.ts +209 -40
  96. package/src/metrics.ts +557 -752
  97. package/src/once.ts +86 -83
  98. package/src/query.ts +1005 -0
  99. package/src/records.ts +795 -418
  100. package/src/runtime/addon.ts +99 -0
  101. package/src/runtime/bun.ts +5 -7
  102. package/src/runtime/node.ts +5 -7
  103. package/src/runtime/pipe.ts +178 -0
  104. package/src/runtime/shared.ts +20 -0
  105. package/src/runtime.ts +5 -2
  106. package/src/search.ts +480 -0
  107. package/src/session.ts +187 -41
  108. package/src/sql.ts +993 -306
  109. package/src/store.ts +473 -78
  110. package/src/time.ts +48 -3
  111. package/src/tx.ts +426 -0
  112. package/src/wire/codec.ts +78 -37
  113. package/src/wire/frame.ts +44 -14
  114. package/src/wire/messages.ts +25 -0
  115. package/src/wire/msgpack.ts +327 -56
  116. package/src/wire/protocol.ts +11709 -0
  117. package/dist/limiter.d.ts +0 -45
  118. package/dist/limiter.js +0 -68
  119. package/dist/quota.d.ts +0 -44
  120. package/dist/quota.js +0 -98
  121. package/src/limiter.ts +0 -99
  122. package/src/quota.ts +0 -137
@@ -1,13 +1,37 @@
1
1
  import type { PrivateChild, Runtime, TlsOptions, Transport } from './runtime.ts';
2
2
  import { Session, type SessionOptions, type Stream } from './session.ts';
3
+ import type { NodeOptions } from './store.ts';
4
+ /**
5
+ * How far a commit goes before it returns, for the files of a store or of
6
+ * one database:
7
+ *
8
+ * 'full' synced to the disk: it survives a crash of the operating
9
+ * system and a power loss. What every file is unless told.
10
+ * 'os' written to the operating system, synced a little later: it
11
+ * survives the program's crash, and a power loss may take the
12
+ * last commits, never the file.
13
+ */
14
+ export type Durability = 'full' | 'os';
15
+ /** Refuses a durability that is neither word, before anything leaves. */
16
+ export declare function checkDurability(durability: Durability | undefined, of: string): void;
17
+ /**
18
+ * Dials as `dial` does, and refuses a store whose files commit another way
19
+ * than this program asked for: a store has one durability, its first
20
+ * opener's, and a program that needs another must not be told it has it.
21
+ */
22
+ export declare function asking(dial: Dialer, dir: string, durability: Durability | undefined): Dialer;
3
23
  export declare class Connection {
4
24
  #private;
5
25
  readonly session: Session;
6
26
  /** the handles this connection opened, by what they are: a bucket's name and options */
7
27
  readonly handles: Map<string, Promise<number>>;
28
+ /** the handle each open was answered, once it was: what a call made in place reads without waiting */
29
+ readonly opened: Map<object, number>;
8
30
  constructor(session: Session, transport: Transport, closing?: () => Promise<unknown>);
9
31
  /** Connects, shakes hands, and gives up after the handshake's time. */
10
32
  static dial(runtime: Runtime, endpoint: string, options: SessionOptions, tls?: TlsOptions): Promise<Connection>;
33
+ /** Shakes hands over a pipe to the core in this process. */
34
+ static overPipe(session: Session, transport: Transport): Promise<Connection>;
11
35
  /**
12
36
  * Shakes hands over a private child's stdin and stdout. Closing waits for
13
37
  * the child's exit, since until then it holds the directory: its LOCK, and
@@ -27,7 +51,26 @@ export type Dialer = () => Promise<Connection>;
27
51
  * this SDK's is stopped and replaced with this SDK's binary; another server
28
52
  * of an older release, a person's or a program's own, is told of once.
29
53
  */
30
- export declare function sidecar(runtime: Runtime, dir: string, binary: () => string, idle?: number, older?: (server: string) => boolean): Dialer;
54
+ export declare function sidecar(runtime: Runtime, dir: string, binary: () => string, started?: Started, older?: (server: string) => boolean): Dialer;
55
+ /** What a server this process starts is started with; one found running keeps its own. */
56
+ export interface Started {
57
+ /** how long a sidecar stays once its last connection has gone, in milliseconds */
58
+ idle?: number | undefined;
59
+ durability?: Durability | undefined;
60
+ /** the file of the store's encryption key, when it is not the store's own */
61
+ encryptionKeyFile?: string | undefined;
62
+ /** the node the store is, when it is one */
63
+ node?: NodeOptions | undefined;
64
+ /** measures the store into its own metrics, the machine too unless it says system: false */
65
+ selfMetrics?: SelfMetrics | undefined;
66
+ }
67
+ /**
68
+ * What a store measures of itself: `true` its commits, files and memory and
69
+ * the machine it runs on, `{ system: false }` the same without the machine.
70
+ */
71
+ export type SelfMetrics = boolean | {
72
+ system?: boolean;
73
+ };
31
74
  /**
32
75
  * Whether a server's release is older than the SDK's own; a build that is no
33
76
  * release, a development copy's or a Go pseudo-version, is neither.
@@ -38,7 +81,12 @@ export declare function replacedSidecar(server: string, own: string, dir: string
38
81
  /** What a program whose SDK found a server of an older release that is no sidecar is told. */
39
82
  export declare function olderServer(server: string, own: string, dir: string): string;
40
83
  /** A private child: tinystore serve --stdio, living and dying with this process. */
41
- export declare function privateChild(runtime: Runtime, dir: string, binary: () => string, clock?: Date): Dialer;
84
+ export declare function privateChild(runtime: Runtime, dir: string, binary: () => string, clock?: Date, started?: Started): Dialer;
85
+ /**
86
+ * The core in this process: no server and no socket, the frames handed to
87
+ * the core and back, through bun:ffi under Bun and its addon under Node.
88
+ */
89
+ export declare function embedded(runtime: Runtime, dir: string, library: () => string, started?: Started): Dialer;
42
90
  /** A remote server: tls:// checks its certificate before the token leaves; tcp:// does not. */
43
91
  export declare function remote(runtime: Runtime, url: string, token: string, tls?: TlsOptions): Dialer;
44
92
  /** What a call may do when its connection is lost: go again, or say its outcome is unknown. */
@@ -54,6 +102,8 @@ export declare class Link {
54
102
  constructor(dial: Dialer);
55
103
  /** The connection a call goes on. */
56
104
  connection(): Promise<Connection>;
105
+ /** The connection a call goes on, when one is there to take it without waiting. */
106
+ ready(): Connection | undefined;
57
107
  /**
58
108
  * Runs a call on the link's connection. build makes its REQUEST's body for
59
109
  * that connection, whose handles it may open; run sends it.
@@ -62,6 +112,12 @@ export declare class Link {
62
112
  /** Closes the connection and every one to come. */
63
113
  close(): Promise<void>;
64
114
  }
115
+ /**
116
+ * Whether a call that failed goes on another connection: one whose REQUEST
117
+ * had not left, and a read that had. A write that had is OutcomeUnknownError,
118
+ * thrown here; any other failure is the call's own.
119
+ */
120
+ export declare function goesAgain(err: unknown, idempotence: Idempotence): boolean;
65
121
  /** What a download sent: its header, its items, and the trailer that ended it. */
66
122
  export interface Downloaded {
67
123
  header: Uint8Array;
@@ -76,3 +132,9 @@ export interface Downloaded {
76
132
  */
77
133
  export declare function download(connection: Connection, method: number, body: Uint8Array, signal?: AbortSignal): Promise<Downloaded>;
78
134
  export type { Stream };
135
+ /**
136
+ * The bytes a piece carries: half what the server takes on a stream before
137
+ * it grants more, so that one piece goes while the last is written, less a
138
+ * piece's own few bytes.
139
+ */
140
+ export declare function pieceOf(connection: Connection): number;
@@ -3,12 +3,39 @@
3
3
  // directory's sidecar, found through SERVE or started, a private child, or a
4
4
  // remote server.
5
5
  import { randomBytes } from 'node:crypto';
6
+ import { mkdirSync, readFileSync, writeFileSync } from 'node:fs';
6
7
  import { join, resolve } from 'node:path';
7
8
  import manifest from '../package.json' with { type: 'json' };
8
9
  import { currentSignal } from './cancel.js';
9
- import { ClosedError, OutcomeUnknownError, TinystoreError, UnavailableError } from './errors.js';
10
- import { LostError, Session, watch } from './session.js';
11
- import { Empty, methods } from './wire/messages.js';
10
+ import { ClosedError, InvalidError, OutcomeUnknownError, TinystoreError, UnavailableError, } from './errors.js';
11
+ import { ConnectionLostError, protocol, Session, watch, } from './session.js';
12
+ import { Empty, methods, StoreOptions } from './wire/protocol.js';
13
+ /** Refuses a durability that is neither word, before anything leaves. */
14
+ export function checkDurability(durability, of) {
15
+ if (durability !== undefined && durability !== 'full' && durability !== 'os') {
16
+ throw new InvalidError(`${of}: durability ${JSON.stringify(durability)}: it is 'full' or 'os'`);
17
+ }
18
+ }
19
+ /**
20
+ * Dials as `dial` does, and refuses a store whose files commit another way
21
+ * than this program asked for: a store has one durability, its first
22
+ * opener's, and a program that needs another must not be told it has it.
23
+ */
24
+ export function asking(dial, dir, durability) {
25
+ if (durability === undefined) {
26
+ return dial;
27
+ }
28
+ return async () => {
29
+ const connection = await dial();
30
+ const opened = (await connection.session.welcomed).durability;
31
+ if (opened !== durability) {
32
+ await connection.close();
33
+ const as = opened === undefined ? 'with none said' : `with durability '${opened}'`;
34
+ throw new InvalidError(`the store in ${dir} is open ${as}, and this program asked for '${durability}'`);
35
+ }
36
+ return connection;
37
+ };
38
+ }
12
39
  /** How long a server may take to answer HELLO. */
13
40
  const handshakeTime = 5000;
14
41
  export class Connection {
@@ -18,6 +45,8 @@ export class Connection {
18
45
  #closing;
19
46
  /** the handles this connection opened, by what they are: a bucket's name and options */
20
47
  handles = new Map();
48
+ /** the handle each open was answered, once it was: what a call made in place reads without waiting */
49
+ opened = new Map();
21
50
  constructor(session, transport, closing = () => Promise.resolve()) {
22
51
  this.session = session;
23
52
  this.#transport = transport;
@@ -47,6 +76,12 @@ export class Connection {
47
76
  await connection.#handshake();
48
77
  return connection;
49
78
  }
79
+ /** Shakes hands over a pipe to the core in this process. */
80
+ static async overPipe(session, transport) {
81
+ const connection = new Connection(session, transport);
82
+ await connection.#handshake();
83
+ return connection;
84
+ }
50
85
  /**
51
86
  * Shakes hands over a private child's stdin and stdout. Closing waits for
52
87
  * the child's exit, since until then it holds the directory: its LOCK, and
@@ -90,11 +125,11 @@ const heldPause = 100;
90
125
  * this SDK's is stopped and replaced with this SDK's binary; another server
91
126
  * of an older release, a person's or a program's own, is told of once.
92
127
  */
93
- export function sidecar(runtime, dir, binary, idle, older = server => olderRelease(server, manifest.version)) {
128
+ export function sidecar(runtime, dir, binary, started = {}, older = server => olderRelease(server, manifest.version)) {
94
129
  const absolute = resolve(dir);
95
130
  const serve = join(absolute, 'server', 'SERVE');
96
131
  const log = join(absolute, 'server', 'serve.log');
97
- const idling = idle === undefined ? [] : ['--idle', `${idle}ms`];
132
+ const idling = started.idle === undefined ? [] : ['--idle', `${started.idle}ms`];
98
133
  let told = false;
99
134
  return async () => {
100
135
  const found = await reachServe(runtime, serve);
@@ -116,10 +151,76 @@ export function sidecar(runtime, dir, binary, idle, older = server => olderRelea
116
151
  return found.connection;
117
152
  }
118
153
  }
119
- const command = [binary(), 'serve', '--dir', absolute, '--local', '--log', log, ...idling];
154
+ const flags = [...idling, ...storeFlags(started, absolute)];
155
+ const command = [binary(), 'serve', '--dir', absolute, '--local', '--log-file', log, ...flags];
120
156
  return startSidecar(runtime, serve, log, command, stoppedInstance);
121
157
  };
122
158
  }
159
+ /** What selfMetrics asks for, none when it asks for nothing. */
160
+ function selfMetricsOf(given) {
161
+ if (given === undefined || given === false) {
162
+ return undefined;
163
+ }
164
+ return { system: given === true || given.system !== false };
165
+ }
166
+ /** The flags of `tinystore serve` that say how the store in `dir` opens. */
167
+ function storeFlags(started, dir) {
168
+ const committing = started.durability === undefined ? [] : ['--durability', started.durability];
169
+ const key = started.encryptionKeyFile;
170
+ const keyed = key === undefined ? committing : [...committing, '--encryption-key-file', resolve(key)];
171
+ const asked = selfMetricsOf(started.selfMetrics);
172
+ const measured = asked === undefined
173
+ ? keyed
174
+ : [...keyed, '--self-metrics', ...(asked.system ? [] : ['--no-system-metrics'])];
175
+ return [...measured, ...nodeFlags(started.node, join(dir, 'server', 'node.toml'))];
176
+ }
177
+ /**
178
+ * A node's options as `tinystore serve` takes them: where it listens and the
179
+ * nodes it copies as flags, and the rest in a node config written to
180
+ * `config`, which the server reads at its start.
181
+ */
182
+ function nodeFlags(node, config) {
183
+ if (node === undefined) {
184
+ return [];
185
+ }
186
+ if (node.token !== undefined) {
187
+ throw new InvalidError("a node's token reaches a server this process starts through a file: give tokenFile, or open embedded");
188
+ }
189
+ const flags = [];
190
+ for (const from of node.from ?? []) {
191
+ flags.push('--from', from);
192
+ }
193
+ if (node.serve !== undefined) {
194
+ flags.push('--node-listen', node.serve);
195
+ }
196
+ // a JSON string or list of strings is TOML's too
197
+ const settings = [];
198
+ if (node.files !== undefined) {
199
+ settings.push(`files = ${JSON.stringify(node.files)}`);
200
+ }
201
+ if (node.tokenFile !== undefined) {
202
+ settings.push(`token-file = ${JSON.stringify(resolve(node.tokenFile))}`);
203
+ }
204
+ if (node.synchronous === true) {
205
+ settings.push('synchronous = true');
206
+ }
207
+ if (node.synchronousWith !== undefined && node.synchronousWith.length > 0) {
208
+ settings.push(`synchronous-with = ${JSON.stringify(node.synchronousWith)}`);
209
+ }
210
+ if (node.insecure === true) {
211
+ settings.push('insecure = true');
212
+ }
213
+ if (node.catalog === true) {
214
+ settings.push('catalog = true');
215
+ }
216
+ if (node.advertise !== undefined) {
217
+ settings.push(`advertise = ${JSON.stringify(node.advertise)}`);
218
+ }
219
+ mkdirSync(join(config, '..'), { recursive: true });
220
+ writeFileSync(config, `${settings.join('\n')}\n`);
221
+ flags.push('--node-config', config);
222
+ return flags;
223
+ }
123
224
  /**
124
225
  * Starts a sidecar and waits for SERVE to name one that answers, its own or
125
226
  * another client's that won the start. One that exits with 3 found the
@@ -246,7 +347,7 @@ async function reachServe(runtime, serve) {
246
347
  }
247
348
  const endpoint = published.endpoints?.[0];
248
349
  const secret = Buffer.from(published.secret ?? '', 'base64url');
249
- if (published.protocol !== 1 || endpoint === undefined || secret.length !== 32) {
350
+ if (published.protocol !== protocol || endpoint === undefined || secret.length !== 32) {
250
351
  return undefined;
251
352
  }
252
353
  try {
@@ -273,12 +374,20 @@ async function tail(runtime, log) {
273
374
  return lines.slice(-5).join('\n');
274
375
  }
275
376
  /** A private child: tinystore serve --stdio, living and dying with this process. */
276
- export function privateChild(runtime, dir, binary, clock) {
377
+ export function privateChild(runtime, dir, binary, clock, started = {}) {
277
378
  // a test's clock starts where the first child's did, and a child started again starts it there
278
379
  const clocked = clock === undefined ? [] : ['--clock', clock.toISOString()];
279
380
  return async () => {
280
381
  let session;
281
- const argv = [binary(), 'serve', '--dir', resolve(dir), '--stdio', ...clocked];
382
+ const argv = [
383
+ binary(),
384
+ 'serve',
385
+ '--dir',
386
+ resolve(dir),
387
+ '--stdio',
388
+ ...clocked,
389
+ ...storeFlags(started, resolve(dir)),
390
+ ];
282
391
  const child = runtime.spawnPrivate(argv, {
283
392
  data: bytes => session?.receive(bytes),
284
393
  end: err => session?.end(err),
@@ -299,6 +408,49 @@ export function privateChild(runtime, dir, binary, clock) {
299
408
  }
300
409
  };
301
410
  }
411
+ /**
412
+ * The core in this process: no server and no socket, the frames handed to
413
+ * the core and back, through bun:ffi under Bun and its addon under Node.
414
+ */
415
+ export function embedded(runtime, dir, library, started = {}) {
416
+ const encryptionKeyFile = started.encryptionKeyFile === undefined ? undefined : resolve(started.encryptionKeyFile);
417
+ const node = started.node;
418
+ const token = node?.token ??
419
+ (node?.tokenFile === undefined
420
+ ? undefined
421
+ : readFileSync(resolve(node.tokenFile), 'utf8').trim());
422
+ const options = StoreOptions.encode({
423
+ durability: started.durability,
424
+ encryptionKeyFile,
425
+ node: node !== undefined,
426
+ from: node?.from ?? [],
427
+ serve: node?.serve,
428
+ files: node?.files ?? [],
429
+ token,
430
+ synchronous: node?.synchronous === true,
431
+ catalog: node?.catalog === true,
432
+ insecure: node?.insecure === true,
433
+ advertise: node?.advertise,
434
+ synchronousWith: node?.synchronousWith ?? [],
435
+ selfMetrics: selfMetricsOf(started.selfMetrics) !== undefined,
436
+ withoutSystem: selfMetricsOf(started.selfMetrics)?.system === false,
437
+ });
438
+ return async () => {
439
+ let session;
440
+ const end = (err) => session?.end(err);
441
+ const transport = typeof Bun === 'undefined'
442
+ ? (await import('./runtime/addon.js')).openAddon(library(), resolve(dir), options, {
443
+ data: bytes => session?.receive(bytes),
444
+ end,
445
+ })
446
+ : (await import('./runtime/pipe.js')).openPipe(library(), resolve(dir), options, {
447
+ frames: (bytes, length) => session?.read(bytes, length) ?? length,
448
+ end,
449
+ });
450
+ session = new Session(bytes => transport.write(bytes), { client: runtime.client });
451
+ return await Connection.overPipe(session, transport);
452
+ };
453
+ }
302
454
  /** A remote server: tls:// checks its certificate before the token leaves; tcp:// does not. */
303
455
  export function remote(runtime, url, token, tls) {
304
456
  return () => Connection.dial(runtime, url, { client: runtime.client, token }, tls);
@@ -312,6 +464,7 @@ export function remote(runtime, url, token, tls) {
312
464
  export class Link {
313
465
  #dial;
314
466
  #current;
467
+ #dialled;
315
468
  #closed = false;
316
469
  constructor(dial) {
317
470
  this.#dial = dial;
@@ -327,6 +480,11 @@ export class Link {
327
480
  }
328
481
  return this.#redial(undefined);
329
482
  }
483
+ /** The connection a call goes on, when one is there to take it without waiting. */
484
+ ready() {
485
+ const c = this.#dialled;
486
+ return c !== undefined && c.session.ended === undefined && !c.session.goingAway ? c : undefined;
487
+ }
330
488
  // one dial at a time: every caller that found the same dead connection
331
489
  // waits for the one new one
332
490
  #redial(dead) {
@@ -335,7 +493,12 @@ export class Link {
335
493
  }
336
494
  const dialled = this.#dial();
337
495
  this.#current = dialled;
338
- dialled.catch(() => {
496
+ this.#dialled = undefined;
497
+ dialled.then(connection => {
498
+ if (this.#current === dialled) {
499
+ this.#dialled = connection;
500
+ }
501
+ }, () => {
339
502
  if (this.#current === dialled) {
340
503
  this.#current = undefined;
341
504
  }
@@ -355,12 +518,8 @@ export class Link {
355
518
  return await attempt(connection);
356
519
  }
357
520
  catch (err) {
358
- if (tries < 2 && (err instanceof LostError || err instanceof UnavailableError)) {
359
- const sent = err instanceof LostError && err.sent;
360
- if (!sent || idempotence === 'read') {
361
- continue;
362
- }
363
- throw new OutcomeUnknownError(`the connection was lost with the write in flight; read what it wrote before writing again: ${err.message}`);
521
+ if (tries < 2 && goesAgain(err, idempotence)) {
522
+ continue;
364
523
  }
365
524
  throw err;
366
525
  }
@@ -371,9 +530,25 @@ export class Link {
371
530
  this.#closed = true;
372
531
  const current = this.#current;
373
532
  this.#current = undefined;
533
+ this.#dialled = undefined;
374
534
  await current?.then(c => c.close(), () => { });
375
535
  }
376
536
  }
537
+ /**
538
+ * Whether a call that failed goes on another connection: one whose REQUEST
539
+ * had not left, and a read that had. A write that had is OutcomeUnknownError,
540
+ * thrown here; any other failure is the call's own.
541
+ */
542
+ export function goesAgain(err, idempotence) {
543
+ if (!(err instanceof ConnectionLostError || err instanceof UnavailableError)) {
544
+ return false;
545
+ }
546
+ const sent = err instanceof ConnectionLostError && err.sent;
547
+ if (!sent || idempotence === 'read') {
548
+ return true;
549
+ }
550
+ throw new OutcomeUnknownError(`the connection was lost with the write in flight; read what it wrote before writing again: ${err.message}`);
551
+ }
377
552
  /**
378
553
  * Runs a download to its end: the RESPONSE that heads it, an item a DATA, and
379
554
  * the DATA that ends it, giving each item's credit back as it arrives, since
@@ -403,3 +578,13 @@ export async function download(connection, method, body, signal) {
403
578
  unwatch();
404
579
  }
405
580
  }
581
+ /**
582
+ * The bytes a piece carries: half what the server takes on a stream before
583
+ * it grants more, so that one piece goes while the last is written, less a
584
+ * piece's own few bytes.
585
+ */
586
+ export function pieceOf(connection) {
587
+ const agreed = connection.session.agreed;
588
+ const most = Math.min(agreed?.maxBody ?? 1 << 20, agreed?.streamCredit ?? 1 << 20);
589
+ return Math.max(1, Math.floor(most / 2) - 16);
590
+ }
package/dist/console.d.ts CHANGED
@@ -42,11 +42,35 @@ export declare function jsonLine(line: ConsoleLine): string;
42
42
  export declare function prettyLine(line: ConsoleLine, look: Look): string;
43
43
  /** A level as slog spells it: INFO, WARN+2, DEBUG-4. */
44
44
  export declare function levelLabel(level: number): string;
45
+ /** The field where a line says where it was logged, as slog's handlers spell it. */
46
+ export declare const sourceKey = "source";
47
+ /** The names `redact` takes to hide the usual secrets, as Go's console.Secrets and Python's SECRETS. */
48
+ export declare const secrets: readonly string[];
49
+ /** What a logger hides: the keys its names name, and a URL's password unless it keeps them. */
50
+ export interface Redactor {
51
+ hides(key: string): boolean;
52
+ urls: boolean;
53
+ }
54
+ /**
55
+ * Hides the values of fields whose keys name a secret: some of the key's
56
+ * words in a row, written together, are a name's. DB_PASSWORD, PasswordHash,
57
+ * api-key and APIKey split into words at '_', '-', '.', spaces and capitals,
58
+ * the case ignored, so 'api key' hides api_key and apiKey, and 'token' hides
59
+ * bot_token and not tokens_used.
60
+ */
61
+ export declare function redactor(names: readonly string[] | undefined, keepUrlPasswords?: boolean): Redactor | undefined;
62
+ /**
63
+ * A key's words as a person reads them, in lower case: split at anything but
64
+ * a letter or a digit, and where a capital begins a word, as in passwordHash
65
+ * and APIKey.
66
+ */
67
+ export declare function words(key: string): string[];
45
68
  /**
46
- * Says whether a field's key hides its value: the key, or the part of a dotted
47
- * key after its last dot, is one of the names, the case ignored.
69
+ * JSON text with the password of each URL inside it hidden, everything else
70
+ * as it was spelled: "postgres://ann:hunter2@db/app" is
71
+ * "postgres://ann:[redacted]@db/app".
48
72
  */
49
- export declare function redactor(names: readonly string[] | undefined): ((key: string) => boolean) | undefined;
73
+ export declare function hideUrlPasswords(text: string): string;
50
74
  /** What a hidden value becomes, in the store and on the console. */
51
75
  export declare const redacted = "\"[redacted]\"";
52
76
  /**
package/dist/console.js CHANGED
@@ -1,5 +1,5 @@
1
- // A logger's console line, as Go's records.Handler and Python's handler write
2
- // it, byte for byte the lines of records/testdata/console.json: pretty for a
1
+ // A logger's console line, as Rust's console and Python's handler write it,
2
+ // byte for byte the lines of testdata/console/console.json: pretty for a
3
3
  // person at a terminal, one JSON object a line for a collector.
4
4
  //
5
5
  // 11:02:11.123 WARN api slow request requestId=7f3a ms=1200
@@ -73,7 +73,7 @@ export function prettyLine(line, look) {
73
73
  const below = [];
74
74
  for (const fields of [line.context, line.attrs]) {
75
75
  for (let i = 0; i < fields.length; i += 2) {
76
- const [shown, lines] = prettyValue(fields[i + 1]);
76
+ const [shown, lines] = prettyValue(fields[i], fields[i + 1]);
77
77
  if (lines) {
78
78
  below.push([fields[i], shown]);
79
79
  }
@@ -128,7 +128,13 @@ function prettyMessage(line) {
128
128
  return body === '' ? line.event : `${line.event} ${body}`;
129
129
  }
130
130
  /** How a field's JSON shows, and whether it is text of several lines, which goes under the line. */
131
- function prettyValue(json) {
131
+ function prettyValue(key, json) {
132
+ if (key === sourceKey) {
133
+ const place = shortSource(json);
134
+ if (place !== undefined) {
135
+ return [place, false];
136
+ }
137
+ }
132
138
  if (!json.startsWith('"')) {
133
139
  return [json, false];
134
140
  }
@@ -182,20 +188,167 @@ function stamp(at, look) {
182
188
  const zone = `${offset < 0 ? '-' : '+'}${two(Math.trunc(Math.abs(offset) / 60))}:${two(Math.abs(offset) % 60)}`;
183
189
  return `${String(year).padStart(4, '0')}-${two(month)}-${two(day)} ${clock} ${zone}`;
184
190
  }
191
+ /** The field where a line says where it was logged, as slog's handlers spell it. */
192
+ export const sourceKey = 'source';
193
+ /**
194
+ * A source as a pretty line shows it, its file's directory and name and its
195
+ * line: {"function":"run","file":"/home/ann/app/server/main.ts","line":42} is
196
+ * server/main.ts:42.
197
+ */
198
+ function shortSource(json) {
199
+ if (!json.startsWith('{')) {
200
+ return undefined;
201
+ }
202
+ const source = JSON.parse(json);
203
+ if (typeof source.file !== 'string' || source.file === '') {
204
+ return undefined;
205
+ }
206
+ const parts = source.file.split(/[\\/]+/).filter(part => part !== '');
207
+ const line = typeof source.line === 'number' ? source.line : 0;
208
+ return `${parts.slice(-2).join('/')}:${line}`;
209
+ }
210
+ /** The names `redact` takes to hide the usual secrets, as Go's console.Secrets and Python's SECRETS. */
211
+ export const secrets = Object.freeze([
212
+ 'password',
213
+ 'passwd',
214
+ 'passphrase',
215
+ 'secret',
216
+ 'token',
217
+ 'credential',
218
+ 'credentials',
219
+ 'authorization',
220
+ 'cookie',
221
+ 'api key',
222
+ 'private key',
223
+ 'secret key',
224
+ 'access key',
225
+ 'signing key',
226
+ 'encryption key',
227
+ 'connection string',
228
+ 'dsn',
229
+ ]);
185
230
  /**
186
- * Says whether a field's key hides its value: the key, or the part of a dotted
187
- * key after its last dot, is one of the names, the case ignored.
231
+ * Hides the values of fields whose keys name a secret: some of the key's
232
+ * words in a row, written together, are a name's. DB_PASSWORD, PasswordHash,
233
+ * api-key and APIKey split into words at '_', '-', '.', spaces and capitals,
234
+ * the case ignored, so 'api key' hides api_key and apiKey, and 'token' hides
235
+ * bot_token and not tokens_used.
188
236
  */
189
- export function redactor(names) {
190
- if (names === undefined || names.length === 0) {
237
+ export function redactor(names, keepUrlPasswords = false) {
238
+ const joined = (names ?? []).map(name => words(name).join('')).filter(name => name !== '');
239
+ if (joined.length === 0 && keepUrlPasswords) {
191
240
  return undefined;
192
241
  }
193
- const hidden = new Set(names.map(name => name.toLowerCase()));
194
- return key => {
195
- const lower = key.toLowerCase();
196
- return hidden.has(lower) || hidden.has(lower.slice(lower.lastIndexOf('.') + 1));
242
+ return {
243
+ hides: key => joined.length > 0 && joined.some(name => joinsTo(words(key), name)),
244
+ urls: !keepUrlPasswords,
197
245
  };
198
246
  }
247
+ /**
248
+ * A key's words as a person reads them, in lower case: split at anything but
249
+ * a letter or a digit, and where a capital begins a word, as in passwordHash
250
+ * and APIKey.
251
+ */
252
+ export function words(key) {
253
+ const out = [];
254
+ const chars = [...key];
255
+ let start = -1;
256
+ const upper = (c) => c !== undefined && /\p{Lu}/u.test(c);
257
+ const lower = (c) => c !== undefined && /\p{Ll}/u.test(c);
258
+ const digit = (c) => c !== undefined && /\p{N}/u.test(c);
259
+ for (let i = 0; i < chars.length; i++) {
260
+ const c = chars[i];
261
+ if (!/[\p{L}\p{N}]/u.test(c)) {
262
+ if (start >= 0) {
263
+ out.push(chars.slice(start, i).join('').toLowerCase());
264
+ start = -1;
265
+ }
266
+ continue;
267
+ }
268
+ const before = chars[i - 1];
269
+ const begins = upper(c) && (lower(before) || digit(before) || (upper(before) && lower(chars[i + 1])));
270
+ if (start >= 0 && begins) {
271
+ out.push(chars.slice(start, i).join('').toLowerCase());
272
+ start = i;
273
+ }
274
+ if (start < 0) {
275
+ start = i;
276
+ }
277
+ }
278
+ if (start >= 0) {
279
+ out.push(chars.slice(start).join('').toLowerCase());
280
+ }
281
+ return out;
282
+ }
283
+ function joinsTo(keyWords, name) {
284
+ for (let i = 0; i < keyWords.length; i++) {
285
+ let rest = name;
286
+ for (const word of keyWords.slice(i)) {
287
+ if (!rest.startsWith(word)) {
288
+ break;
289
+ }
290
+ rest = rest.slice(word.length);
291
+ if (rest === '') {
292
+ return true;
293
+ }
294
+ }
295
+ }
296
+ return false;
297
+ }
298
+ /**
299
+ * JSON text with the password of each URL inside it hidden, everything else
300
+ * as it was spelled: "postgres://ann:hunter2@db/app" is
301
+ * "postgres://ann:[redacted]@db/app".
302
+ */
303
+ export function hideUrlPasswords(text) {
304
+ if (!text.includes('://') && !text.includes('\\')) {
305
+ return text;
306
+ }
307
+ return text.replace(/"(?:[^"\\]|\\.)*"/g, raw => {
308
+ const value = JSON.parse(raw);
309
+ const hidden = hideUrlTextPasswords(value);
310
+ return hidden === value ? raw : JSON.stringify(hidden);
311
+ });
312
+ }
313
+ function hideUrlTextPasswords(text) {
314
+ let out = '';
315
+ let written = 0;
316
+ for (let from = 0; from < text.length;) {
317
+ const schemeEnd = text.indexOf('://', from);
318
+ if (schemeEnd < 0) {
319
+ break;
320
+ }
321
+ const [password, at, end] = findPassword(text, schemeEnd + 3);
322
+ from = end;
323
+ if (password < 0 || !/[A-Za-z][A-Za-z0-9+.-]*$/.test(text.slice(0, schemeEnd))) {
324
+ continue;
325
+ }
326
+ out += `${text.slice(written, password)}[redacted]`;
327
+ written = at;
328
+ }
329
+ return written === 0 ? text : out + text.slice(written);
330
+ }
331
+ /** Where an authority's password begins and the '@' after it, -1 when it has none, and where it ends. */
332
+ function findPassword(text, start) {
333
+ let at = -1;
334
+ let colon = -1;
335
+ let i = start;
336
+ while (i < text.length) {
337
+ const c = text[i];
338
+ if (c === '/' || c === '?' || c === '#' || c <= ' ') {
339
+ break;
340
+ }
341
+ if (c === '@') {
342
+ at = i;
343
+ }
344
+ else if (c === ':' && colon < 0) {
345
+ colon = i;
346
+ }
347
+ i++;
348
+ }
349
+ const end = Math.min(i, text.length);
350
+ return at < 0 || colon < 0 || colon + 1 >= at ? [-1, -1, end] : [colon + 1, at, end];
351
+ }
199
352
  /** What a hidden value becomes, in the store and on the console. */
200
353
  export const redacted = '"[redacted]"';
201
354
  /**