@tinyshed/tinystore 0.1.0-rc.4 → 0.1.0-rc.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +37 -37
- package/bin/tinystore.js +12 -1
- package/dist/backup.d.ts +33 -0
- package/dist/backup.js +49 -0
- package/dist/blobs.d.ts +160 -79
- package/dist/blobs.js +510 -217
- package/dist/channels.d.ts +90 -0
- package/dist/channels.js +291 -0
- package/dist/clock.js +3 -3
- package/dist/config.d.ts +209 -86
- package/dist/config.js +581 -538
- package/dist/connection.d.ts +64 -2
- package/dist/connection.js +201 -16
- package/dist/console.d.ts +27 -3
- package/dist/console.js +165 -12
- package/dist/dotenv.d.ts +7 -0
- package/dist/dotenv.js +79 -0
- package/dist/errors.d.ts +61 -1
- package/dist/errors.js +81 -1
- package/dist/handles.d.ts +44 -2
- package/dist/handles.js +81 -2
- package/dist/index.d.ts +17 -11
- package/dist/index.js +5 -3
- package/dist/instruments.d.ts +177 -0
- package/dist/instruments.js +819 -0
- package/dist/jobs.d.ts +335 -239
- package/dist/jobs.js +716 -536
- package/dist/json.d.ts +2 -0
- package/dist/json.js +122 -0
- package/dist/kv.d.ts +130 -243
- package/dist/kv.js +273 -640
- package/dist/limits.d.ts +80 -0
- package/dist/limits.js +139 -0
- package/dist/locks.d.ts +62 -0
- package/dist/locks.js +237 -0
- package/dist/logger.d.ts +45 -9
- package/dist/logger.js +150 -38
- package/dist/metrics.d.ts +210 -209
- package/dist/metrics.js +319 -556
- package/dist/once.d.ts +19 -18
- package/dist/once.js +74 -63
- package/dist/query.d.ts +181 -0
- package/dist/query.js +788 -0
- package/dist/records.d.ts +241 -150
- package/dist/records.js +507 -291
- package/dist/runtime/addon.d.ts +14 -0
- package/dist/runtime/addon.js +69 -0
- package/dist/runtime/bun.js +5 -7
- package/dist/runtime/node.js +5 -7
- package/dist/runtime/pipe.d.ts +17 -0
- package/dist/runtime/pipe.js +133 -0
- package/dist/runtime/shared.d.ts +6 -0
- package/dist/runtime/shared.js +19 -0
- package/dist/runtime.d.ts +5 -2
- package/dist/search.d.ts +189 -0
- package/dist/search.js +281 -0
- package/dist/session.d.ts +31 -3
- package/dist/session.js +153 -41
- package/dist/sql.d.ts +228 -78
- package/dist/sql.js +745 -240
- package/dist/store.d.ts +244 -24
- package/dist/store.js +270 -67
- package/dist/time.d.ts +5 -1
- package/dist/time.js +22 -2
- package/dist/tx.d.ts +61 -0
- package/dist/tx.js +314 -0
- package/dist/wire/codec.d.ts +23 -7
- package/dist/wire/codec.js +56 -35
- package/dist/wire/frame.d.ts +2 -0
- package/dist/wire/frame.js +31 -14
- package/dist/wire/messages.d.ts +259 -0
- package/dist/wire/messages.js +22 -0
- package/dist/wire/msgpack.d.ts +49 -1
- package/dist/wire/msgpack.js +293 -56
- package/dist/wire/protocol.d.ts +2683 -0
- package/dist/wire/protocol.js +10846 -0
- package/package.json +39 -38
- package/src/backup.ts +81 -0
- package/src/blobs.ts +678 -301
- package/src/channels.ts +377 -0
- package/src/clock.ts +4 -4
- package/src/config.ts +902 -612
- package/src/connection.ts +279 -21
- package/src/console.ts +180 -12
- package/src/dotenv.ts +82 -0
- package/src/errors.ts +105 -0
- package/src/handles.ts +157 -8
- package/src/index.ts +158 -52
- package/src/instruments.ts +1042 -0
- package/src/jobs.ts +1017 -733
- package/src/json.ts +133 -0
- package/src/kv.ts +424 -847
- package/src/limits.ts +202 -0
- package/src/locks.ts +320 -0
- package/src/logger.ts +209 -40
- package/src/metrics.ts +557 -752
- package/src/once.ts +86 -83
- package/src/query.ts +1005 -0
- package/src/records.ts +795 -418
- package/src/runtime/addon.ts +99 -0
- package/src/runtime/bun.ts +5 -7
- package/src/runtime/node.ts +5 -7
- package/src/runtime/pipe.ts +178 -0
- package/src/runtime/shared.ts +20 -0
- package/src/runtime.ts +5 -2
- package/src/search.ts +480 -0
- package/src/session.ts +187 -41
- package/src/sql.ts +993 -306
- package/src/store.ts +473 -78
- package/src/time.ts +48 -3
- package/src/tx.ts +426 -0
- package/src/wire/codec.ts +78 -37
- package/src/wire/frame.ts +44 -14
- package/src/wire/messages.ts +25 -0
- package/src/wire/msgpack.ts +327 -56
- package/src/wire/protocol.ts +11709 -0
- package/dist/limiter.d.ts +0 -45
- package/dist/limiter.js +0 -68
- package/dist/quota.d.ts +0 -44
- package/dist/quota.js +0 -98
- package/src/limiter.ts +0 -99
- package/src/quota.ts +0 -137
package/src/connection.ts
CHANGED
|
@@ -4,14 +4,71 @@
|
|
|
4
4
|
// remote server.
|
|
5
5
|
|
|
6
6
|
import { randomBytes } from 'node:crypto'
|
|
7
|
+
import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'
|
|
7
8
|
import { join, resolve } from 'node:path'
|
|
8
9
|
|
|
9
10
|
import manifest from '../package.json' with { type: 'json' }
|
|
10
11
|
import { currentSignal } from './cancel.ts'
|
|
11
|
-
import {
|
|
12
|
+
import {
|
|
13
|
+
ClosedError,
|
|
14
|
+
InvalidError,
|
|
15
|
+
OutcomeUnknownError,
|
|
16
|
+
TinystoreError,
|
|
17
|
+
UnavailableError,
|
|
18
|
+
} from './errors.ts'
|
|
12
19
|
import type { PrivateChild, Runtime, TlsOptions, Transport } from './runtime.ts'
|
|
13
|
-
import {
|
|
14
|
-
|
|
20
|
+
import {
|
|
21
|
+
ConnectionLostError,
|
|
22
|
+
protocol,
|
|
23
|
+
Session,
|
|
24
|
+
type SessionOptions,
|
|
25
|
+
type Stream,
|
|
26
|
+
watch,
|
|
27
|
+
} from './session.ts'
|
|
28
|
+
import type { NodeOptions } from './store.ts'
|
|
29
|
+
import { Empty, methods, StoreOptions } from './wire/protocol.ts'
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* How far a commit goes before it returns, for the files of a store or of
|
|
33
|
+
* one database:
|
|
34
|
+
*
|
|
35
|
+
* 'full' synced to the disk: it survives a crash of the operating
|
|
36
|
+
* system and a power loss. What every file is unless told.
|
|
37
|
+
* 'os' written to the operating system, synced a little later: it
|
|
38
|
+
* survives the program's crash, and a power loss may take the
|
|
39
|
+
* last commits, never the file.
|
|
40
|
+
*/
|
|
41
|
+
export type Durability = 'full' | 'os'
|
|
42
|
+
|
|
43
|
+
/** Refuses a durability that is neither word, before anything leaves. */
|
|
44
|
+
export function checkDurability(durability: Durability | undefined, of: string): void {
|
|
45
|
+
if (durability !== undefined && durability !== 'full' && durability !== 'os') {
|
|
46
|
+
throw new InvalidError(`${of}: durability ${JSON.stringify(durability)}: it is 'full' or 'os'`)
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* Dials as `dial` does, and refuses a store whose files commit another way
|
|
52
|
+
* than this program asked for: a store has one durability, its first
|
|
53
|
+
* opener's, and a program that needs another must not be told it has it.
|
|
54
|
+
*/
|
|
55
|
+
export function asking(dial: Dialer, dir: string, durability: Durability | undefined): Dialer {
|
|
56
|
+
if (durability === undefined) {
|
|
57
|
+
return dial
|
|
58
|
+
}
|
|
59
|
+
return async () => {
|
|
60
|
+
const connection = await dial()
|
|
61
|
+
const opened = (await connection.session.welcomed).durability
|
|
62
|
+
if (opened !== durability) {
|
|
63
|
+
await connection.close()
|
|
64
|
+
const as = opened === undefined ? 'with none said' : `with durability '${opened}'`
|
|
65
|
+
throw new InvalidError(
|
|
66
|
+
`the store in ${dir} is open ${as}, and this program asked for '${durability}'`,
|
|
67
|
+
)
|
|
68
|
+
}
|
|
69
|
+
return connection
|
|
70
|
+
}
|
|
71
|
+
}
|
|
15
72
|
|
|
16
73
|
/** How long a server may take to answer HELLO. */
|
|
17
74
|
const handshakeTime = 5000
|
|
@@ -23,6 +80,8 @@ export class Connection {
|
|
|
23
80
|
readonly #closing: () => Promise<unknown>
|
|
24
81
|
/** the handles this connection opened, by what they are: a bucket's name and options */
|
|
25
82
|
readonly handles = new Map<string, Promise<number>>()
|
|
83
|
+
/** the handle each open was answered, once it was: what a call made in place reads without waiting */
|
|
84
|
+
readonly opened = new Map<object, number>()
|
|
26
85
|
|
|
27
86
|
constructor(
|
|
28
87
|
session: Session,
|
|
@@ -64,6 +123,13 @@ export class Connection {
|
|
|
64
123
|
return connection
|
|
65
124
|
}
|
|
66
125
|
|
|
126
|
+
/** Shakes hands over a pipe to the core in this process. */
|
|
127
|
+
static async overPipe(session: Session, transport: Transport): Promise<Connection> {
|
|
128
|
+
const connection = new Connection(session, transport)
|
|
129
|
+
await connection.#handshake()
|
|
130
|
+
return connection
|
|
131
|
+
}
|
|
132
|
+
|
|
67
133
|
/**
|
|
68
134
|
* Shakes hands over a private child's stdin and stdout. Closing waits for
|
|
69
135
|
* the child's exit, since until then it holds the directory: its LOCK, and
|
|
@@ -132,13 +198,13 @@ export function sidecar(
|
|
|
132
198
|
runtime: Runtime,
|
|
133
199
|
dir: string,
|
|
134
200
|
binary: () => string,
|
|
135
|
-
|
|
201
|
+
started: Started = {},
|
|
136
202
|
older: (server: string) => boolean = server => olderRelease(server, manifest.version),
|
|
137
203
|
): Dialer {
|
|
138
204
|
const absolute = resolve(dir)
|
|
139
205
|
const serve = join(absolute, 'server', 'SERVE')
|
|
140
206
|
const log = join(absolute, 'server', 'serve.log')
|
|
141
|
-
const idling = idle === undefined ? [] : ['--idle', `${idle}ms`]
|
|
207
|
+
const idling = started.idle === undefined ? [] : ['--idle', `${started.idle}ms`]
|
|
142
208
|
let told = false
|
|
143
209
|
return async () => {
|
|
144
210
|
const found = await reachServe(runtime, serve)
|
|
@@ -159,11 +225,103 @@ export function sidecar(
|
|
|
159
225
|
return found.connection
|
|
160
226
|
}
|
|
161
227
|
}
|
|
162
|
-
const
|
|
228
|
+
const flags = [...idling, ...storeFlags(started, absolute)]
|
|
229
|
+
const command = [binary(), 'serve', '--dir', absolute, '--local', '--log-file', log, ...flags]
|
|
163
230
|
return startSidecar(runtime, serve, log, command, stoppedInstance)
|
|
164
231
|
}
|
|
165
232
|
}
|
|
166
233
|
|
|
234
|
+
/** What a server this process starts is started with; one found running keeps its own. */
|
|
235
|
+
export interface Started {
|
|
236
|
+
/** how long a sidecar stays once its last connection has gone, in milliseconds */
|
|
237
|
+
idle?: number | undefined
|
|
238
|
+
durability?: Durability | undefined
|
|
239
|
+
/** the file of the store's encryption key, when it is not the store's own */
|
|
240
|
+
encryptionKeyFile?: string | undefined
|
|
241
|
+
/** the node the store is, when it is one */
|
|
242
|
+
node?: NodeOptions | undefined
|
|
243
|
+
/** measures the store into its own metrics, the machine too unless it says system: false */
|
|
244
|
+
selfMetrics?: SelfMetrics | undefined
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
/**
|
|
248
|
+
* What a store measures of itself: `true` its commits, files and memory and
|
|
249
|
+
* the machine it runs on, `{ system: false }` the same without the machine.
|
|
250
|
+
*/
|
|
251
|
+
export type SelfMetrics = boolean | { system?: boolean }
|
|
252
|
+
|
|
253
|
+
/** What selfMetrics asks for, none when it asks for nothing. */
|
|
254
|
+
function selfMetricsOf(given: SelfMetrics | undefined): { system: boolean } | undefined {
|
|
255
|
+
if (given === undefined || given === false) {
|
|
256
|
+
return undefined
|
|
257
|
+
}
|
|
258
|
+
return { system: given === true || given.system !== false }
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
/** The flags of `tinystore serve` that say how the store in `dir` opens. */
|
|
262
|
+
function storeFlags(started: Started, dir: string): string[] {
|
|
263
|
+
const committing = started.durability === undefined ? [] : ['--durability', started.durability]
|
|
264
|
+
const key = started.encryptionKeyFile
|
|
265
|
+
const keyed =
|
|
266
|
+
key === undefined ? committing : [...committing, '--encryption-key-file', resolve(key)]
|
|
267
|
+
const asked = selfMetricsOf(started.selfMetrics)
|
|
268
|
+
const measured =
|
|
269
|
+
asked === undefined
|
|
270
|
+
? keyed
|
|
271
|
+
: [...keyed, '--self-metrics', ...(asked.system ? [] : ['--no-system-metrics'])]
|
|
272
|
+
return [...measured, ...nodeFlags(started.node, join(dir, 'server', 'node.toml'))]
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
/**
|
|
276
|
+
* A node's options as `tinystore serve` takes them: where it listens and the
|
|
277
|
+
* nodes it copies as flags, and the rest in a node config written to
|
|
278
|
+
* `config`, which the server reads at its start.
|
|
279
|
+
*/
|
|
280
|
+
function nodeFlags(node: NodeOptions | undefined, config: string): string[] {
|
|
281
|
+
if (node === undefined) {
|
|
282
|
+
return []
|
|
283
|
+
}
|
|
284
|
+
if (node.token !== undefined) {
|
|
285
|
+
throw new InvalidError(
|
|
286
|
+
"a node's token reaches a server this process starts through a file: give tokenFile, or open embedded",
|
|
287
|
+
)
|
|
288
|
+
}
|
|
289
|
+
const flags: string[] = []
|
|
290
|
+
for (const from of node.from ?? []) {
|
|
291
|
+
flags.push('--from', from)
|
|
292
|
+
}
|
|
293
|
+
if (node.serve !== undefined) {
|
|
294
|
+
flags.push('--node-listen', node.serve)
|
|
295
|
+
}
|
|
296
|
+
// a JSON string or list of strings is TOML's too
|
|
297
|
+
const settings: string[] = []
|
|
298
|
+
if (node.files !== undefined) {
|
|
299
|
+
settings.push(`files = ${JSON.stringify(node.files)}`)
|
|
300
|
+
}
|
|
301
|
+
if (node.tokenFile !== undefined) {
|
|
302
|
+
settings.push(`token-file = ${JSON.stringify(resolve(node.tokenFile))}`)
|
|
303
|
+
}
|
|
304
|
+
if (node.synchronous === true) {
|
|
305
|
+
settings.push('synchronous = true')
|
|
306
|
+
}
|
|
307
|
+
if (node.synchronousWith !== undefined && node.synchronousWith.length > 0) {
|
|
308
|
+
settings.push(`synchronous-with = ${JSON.stringify(node.synchronousWith)}`)
|
|
309
|
+
}
|
|
310
|
+
if (node.insecure === true) {
|
|
311
|
+
settings.push('insecure = true')
|
|
312
|
+
}
|
|
313
|
+
if (node.catalog === true) {
|
|
314
|
+
settings.push('catalog = true')
|
|
315
|
+
}
|
|
316
|
+
if (node.advertise !== undefined) {
|
|
317
|
+
settings.push(`advertise = ${JSON.stringify(node.advertise)}`)
|
|
318
|
+
}
|
|
319
|
+
mkdirSync(join(config, '..'), { recursive: true })
|
|
320
|
+
writeFileSync(config, `${settings.join('\n')}\n`)
|
|
321
|
+
flags.push('--node-config', config)
|
|
322
|
+
return flags
|
|
323
|
+
}
|
|
324
|
+
|
|
167
325
|
/**
|
|
168
326
|
* Starts a sidecar and waits for SERVE to name one that answers, its own or
|
|
169
327
|
* another client's that won the start. One that exits with 3 found the
|
|
@@ -313,7 +471,7 @@ async function reachServe(
|
|
|
313
471
|
}
|
|
314
472
|
const endpoint = published.endpoints?.[0]
|
|
315
473
|
const secret = Buffer.from(published.secret ?? '', 'base64url')
|
|
316
|
-
if (published.protocol !==
|
|
474
|
+
if (published.protocol !== protocol || endpoint === undefined || secret.length !== 32) {
|
|
317
475
|
return undefined
|
|
318
476
|
}
|
|
319
477
|
try {
|
|
@@ -347,12 +505,21 @@ export function privateChild(
|
|
|
347
505
|
dir: string,
|
|
348
506
|
binary: () => string,
|
|
349
507
|
clock?: Date,
|
|
508
|
+
started: Started = {},
|
|
350
509
|
): Dialer {
|
|
351
510
|
// a test's clock starts where the first child's did, and a child started again starts it there
|
|
352
511
|
const clocked = clock === undefined ? [] : ['--clock', clock.toISOString()]
|
|
353
512
|
return async () => {
|
|
354
513
|
let session: Session | undefined
|
|
355
|
-
const argv = [
|
|
514
|
+
const argv = [
|
|
515
|
+
binary(),
|
|
516
|
+
'serve',
|
|
517
|
+
'--dir',
|
|
518
|
+
resolve(dir),
|
|
519
|
+
'--stdio',
|
|
520
|
+
...clocked,
|
|
521
|
+
...storeFlags(started, resolve(dir)),
|
|
522
|
+
]
|
|
356
523
|
const child = runtime.spawnPrivate(argv, {
|
|
357
524
|
data: bytes => session?.receive(bytes),
|
|
358
525
|
end: err => session?.end(err),
|
|
@@ -373,6 +540,58 @@ export function privateChild(
|
|
|
373
540
|
}
|
|
374
541
|
}
|
|
375
542
|
|
|
543
|
+
/**
|
|
544
|
+
* The core in this process: no server and no socket, the frames handed to
|
|
545
|
+
* the core and back, through bun:ffi under Bun and its addon under Node.
|
|
546
|
+
*/
|
|
547
|
+
export function embedded(
|
|
548
|
+
runtime: Runtime,
|
|
549
|
+
dir: string,
|
|
550
|
+
library: () => string,
|
|
551
|
+
started: Started = {},
|
|
552
|
+
): Dialer {
|
|
553
|
+
const encryptionKeyFile =
|
|
554
|
+
started.encryptionKeyFile === undefined ? undefined : resolve(started.encryptionKeyFile)
|
|
555
|
+
const node = started.node
|
|
556
|
+
const token =
|
|
557
|
+
node?.token ??
|
|
558
|
+
(node?.tokenFile === undefined
|
|
559
|
+
? undefined
|
|
560
|
+
: readFileSync(resolve(node.tokenFile), 'utf8').trim())
|
|
561
|
+
const options = StoreOptions.encode({
|
|
562
|
+
durability: started.durability,
|
|
563
|
+
encryptionKeyFile,
|
|
564
|
+
node: node !== undefined,
|
|
565
|
+
from: node?.from ?? [],
|
|
566
|
+
serve: node?.serve,
|
|
567
|
+
files: node?.files ?? [],
|
|
568
|
+
token,
|
|
569
|
+
synchronous: node?.synchronous === true,
|
|
570
|
+
catalog: node?.catalog === true,
|
|
571
|
+
insecure: node?.insecure === true,
|
|
572
|
+
advertise: node?.advertise,
|
|
573
|
+
synchronousWith: node?.synchronousWith ?? [],
|
|
574
|
+
selfMetrics: selfMetricsOf(started.selfMetrics) !== undefined,
|
|
575
|
+
withoutSystem: selfMetricsOf(started.selfMetrics)?.system === false,
|
|
576
|
+
})
|
|
577
|
+
return async () => {
|
|
578
|
+
let session: Session | undefined
|
|
579
|
+
const end = (err: Error) => session?.end(err)
|
|
580
|
+
const transport =
|
|
581
|
+
typeof Bun === 'undefined'
|
|
582
|
+
? (await import('./runtime/addon.ts')).openAddon(library(), resolve(dir), options, {
|
|
583
|
+
data: bytes => session?.receive(bytes),
|
|
584
|
+
end,
|
|
585
|
+
})
|
|
586
|
+
: (await import('./runtime/pipe.ts')).openPipe(library(), resolve(dir), options, {
|
|
587
|
+
frames: (bytes, length) => session?.read(bytes, length) ?? length,
|
|
588
|
+
end,
|
|
589
|
+
})
|
|
590
|
+
session = new Session(bytes => transport.write(bytes), { client: runtime.client })
|
|
591
|
+
return await Connection.overPipe(session, transport)
|
|
592
|
+
}
|
|
593
|
+
}
|
|
594
|
+
|
|
376
595
|
/** A remote server: tls:// checks its certificate before the token leaves; tcp:// does not. */
|
|
377
596
|
export function remote(runtime: Runtime, url: string, token: string, tls?: TlsOptions): Dialer {
|
|
378
597
|
return () => Connection.dial(runtime, url, { client: runtime.client, token }, tls)
|
|
@@ -390,6 +609,7 @@ export type Idempotence = 'read' | 'write'
|
|
|
390
609
|
export class Link {
|
|
391
610
|
readonly #dial: Dialer
|
|
392
611
|
#current: Promise<Connection> | undefined
|
|
612
|
+
#dialled: Connection | undefined
|
|
393
613
|
#closed = false
|
|
394
614
|
|
|
395
615
|
constructor(dial: Dialer) {
|
|
@@ -411,6 +631,12 @@ export class Link {
|
|
|
411
631
|
return this.#redial(undefined)
|
|
412
632
|
}
|
|
413
633
|
|
|
634
|
+
/** The connection a call goes on, when one is there to take it without waiting. */
|
|
635
|
+
ready(): Connection | undefined {
|
|
636
|
+
const c = this.#dialled
|
|
637
|
+
return c !== undefined && c.session.ended === undefined && !c.session.goingAway ? c : undefined
|
|
638
|
+
}
|
|
639
|
+
|
|
414
640
|
// one dial at a time: every caller that found the same dead connection
|
|
415
641
|
// waits for the one new one
|
|
416
642
|
#redial(dead: Promise<Connection> | undefined): Promise<Connection> {
|
|
@@ -419,11 +645,19 @@ export class Link {
|
|
|
419
645
|
}
|
|
420
646
|
const dialled = this.#dial()
|
|
421
647
|
this.#current = dialled
|
|
422
|
-
dialled
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
648
|
+
this.#dialled = undefined
|
|
649
|
+
dialled.then(
|
|
650
|
+
connection => {
|
|
651
|
+
if (this.#current === dialled) {
|
|
652
|
+
this.#dialled = connection
|
|
653
|
+
}
|
|
654
|
+
},
|
|
655
|
+
() => {
|
|
656
|
+
if (this.#current === dialled) {
|
|
657
|
+
this.#current = undefined
|
|
658
|
+
}
|
|
659
|
+
},
|
|
660
|
+
)
|
|
427
661
|
return dialled
|
|
428
662
|
}
|
|
429
663
|
|
|
@@ -442,14 +676,8 @@ export class Link {
|
|
|
442
676
|
try {
|
|
443
677
|
return await attempt(connection)
|
|
444
678
|
} catch (err) {
|
|
445
|
-
if (tries < 2 && (err
|
|
446
|
-
|
|
447
|
-
if (!sent || idempotence === 'read') {
|
|
448
|
-
continue
|
|
449
|
-
}
|
|
450
|
-
throw new OutcomeUnknownError(
|
|
451
|
-
`the connection was lost with the write in flight; read what it wrote before writing again: ${err.message}`,
|
|
452
|
-
)
|
|
679
|
+
if (tries < 2 && goesAgain(err, idempotence)) {
|
|
680
|
+
continue
|
|
453
681
|
}
|
|
454
682
|
throw err
|
|
455
683
|
}
|
|
@@ -461,6 +689,7 @@ export class Link {
|
|
|
461
689
|
this.#closed = true
|
|
462
690
|
const current = this.#current
|
|
463
691
|
this.#current = undefined
|
|
692
|
+
this.#dialled = undefined
|
|
464
693
|
await current?.then(
|
|
465
694
|
c => c.close(),
|
|
466
695
|
() => {},
|
|
@@ -468,6 +697,24 @@ export class Link {
|
|
|
468
697
|
}
|
|
469
698
|
}
|
|
470
699
|
|
|
700
|
+
/**
|
|
701
|
+
* Whether a call that failed goes on another connection: one whose REQUEST
|
|
702
|
+
* had not left, and a read that had. A write that had is OutcomeUnknownError,
|
|
703
|
+
* thrown here; any other failure is the call's own.
|
|
704
|
+
*/
|
|
705
|
+
export function goesAgain(err: unknown, idempotence: Idempotence): boolean {
|
|
706
|
+
if (!(err instanceof ConnectionLostError || err instanceof UnavailableError)) {
|
|
707
|
+
return false
|
|
708
|
+
}
|
|
709
|
+
const sent = err instanceof ConnectionLostError && err.sent
|
|
710
|
+
if (!sent || idempotence === 'read') {
|
|
711
|
+
return true
|
|
712
|
+
}
|
|
713
|
+
throw new OutcomeUnknownError(
|
|
714
|
+
`the connection was lost with the write in flight; read what it wrote before writing again: ${err.message}`,
|
|
715
|
+
)
|
|
716
|
+
}
|
|
717
|
+
|
|
471
718
|
/** What a download sent: its header, its items, and the trailer that ended it. */
|
|
472
719
|
export interface Downloaded {
|
|
473
720
|
header: Uint8Array
|
|
@@ -510,3 +757,14 @@ export async function download(
|
|
|
510
757
|
}
|
|
511
758
|
|
|
512
759
|
export type { Stream }
|
|
760
|
+
|
|
761
|
+
/**
|
|
762
|
+
* The bytes a piece carries: half what the server takes on a stream before
|
|
763
|
+
* it grants more, so that one piece goes while the last is written, less a
|
|
764
|
+
* piece's own few bytes.
|
|
765
|
+
*/
|
|
766
|
+
export function pieceOf(connection: Connection): number {
|
|
767
|
+
const agreed = connection.session.agreed
|
|
768
|
+
const most = Math.min(agreed?.maxBody ?? 1 << 20, agreed?.streamCredit ?? 1 << 20)
|
|
769
|
+
return Math.max(1, Math.floor(most / 2) - 16)
|
|
770
|
+
}
|
package/src/console.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
// A logger's console line, as
|
|
2
|
-
//
|
|
1
|
+
// A logger's console line, as Rust's console and Python's handler write it,
|
|
2
|
+
// byte for byte the lines of testdata/console/console.json: pretty for a
|
|
3
3
|
// person at a terminal, one JSON object a line for a collector.
|
|
4
4
|
//
|
|
5
5
|
// 11:02:11.123 WARN api slow request requestId=7f3a ms=1200
|
|
@@ -108,7 +108,7 @@ export function prettyLine(line: ConsoleLine, look: Look): string {
|
|
|
108
108
|
const below: [string, string][] = []
|
|
109
109
|
for (const fields of [line.context, line.attrs]) {
|
|
110
110
|
for (let i = 0; i < fields.length; i += 2) {
|
|
111
|
-
const [shown, lines] = prettyValue(fields[i + 1] as string)
|
|
111
|
+
const [shown, lines] = prettyValue(fields[i] as string, fields[i + 1] as string)
|
|
112
112
|
if (lines) {
|
|
113
113
|
below.push([fields[i] as string, shown])
|
|
114
114
|
} else {
|
|
@@ -166,7 +166,13 @@ function prettyMessage(line: ConsoleLine): string {
|
|
|
166
166
|
}
|
|
167
167
|
|
|
168
168
|
/** How a field's JSON shows, and whether it is text of several lines, which goes under the line. */
|
|
169
|
-
function prettyValue(json: string): [string, boolean] {
|
|
169
|
+
function prettyValue(key: string, json: string): [string, boolean] {
|
|
170
|
+
if (key === sourceKey) {
|
|
171
|
+
const place = shortSource(json)
|
|
172
|
+
if (place !== undefined) {
|
|
173
|
+
return [place, false]
|
|
174
|
+
}
|
|
175
|
+
}
|
|
170
176
|
if (!json.startsWith('"')) {
|
|
171
177
|
return [json, false]
|
|
172
178
|
}
|
|
@@ -223,21 +229,183 @@ function stamp(at: Date, look: Look): string {
|
|
|
223
229
|
return `${String(year).padStart(4, '0')}-${two(month)}-${two(day)} ${clock} ${zone}`
|
|
224
230
|
}
|
|
225
231
|
|
|
232
|
+
/** The field where a line says where it was logged, as slog's handlers spell it. */
|
|
233
|
+
export const sourceKey = 'source'
|
|
234
|
+
|
|
226
235
|
/**
|
|
227
|
-
*
|
|
228
|
-
*
|
|
236
|
+
* A source as a pretty line shows it, its file's directory and name and its
|
|
237
|
+
* line: {"function":"run","file":"/home/ann/app/server/main.ts","line":42} is
|
|
238
|
+
* server/main.ts:42.
|
|
239
|
+
*/
|
|
240
|
+
function shortSource(json: string): string | undefined {
|
|
241
|
+
if (!json.startsWith('{')) {
|
|
242
|
+
return undefined
|
|
243
|
+
}
|
|
244
|
+
const source = JSON.parse(json) as { file?: unknown; line?: unknown }
|
|
245
|
+
if (typeof source.file !== 'string' || source.file === '') {
|
|
246
|
+
return undefined
|
|
247
|
+
}
|
|
248
|
+
const parts = source.file.split(/[\\/]+/).filter(part => part !== '')
|
|
249
|
+
const line = typeof source.line === 'number' ? source.line : 0
|
|
250
|
+
return `${parts.slice(-2).join('/')}:${line}`
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
/** The names `redact` takes to hide the usual secrets, as Go's console.Secrets and Python's SECRETS. */
|
|
254
|
+
export const secrets: readonly string[] = Object.freeze([
|
|
255
|
+
'password',
|
|
256
|
+
'passwd',
|
|
257
|
+
'passphrase',
|
|
258
|
+
'secret',
|
|
259
|
+
'token',
|
|
260
|
+
'credential',
|
|
261
|
+
'credentials',
|
|
262
|
+
'authorization',
|
|
263
|
+
'cookie',
|
|
264
|
+
'api key',
|
|
265
|
+
'private key',
|
|
266
|
+
'secret key',
|
|
267
|
+
'access key',
|
|
268
|
+
'signing key',
|
|
269
|
+
'encryption key',
|
|
270
|
+
'connection string',
|
|
271
|
+
'dsn',
|
|
272
|
+
])
|
|
273
|
+
|
|
274
|
+
/** What a logger hides: the keys its names name, and a URL's password unless it keeps them. */
|
|
275
|
+
export interface Redactor {
|
|
276
|
+
hides(key: string): boolean
|
|
277
|
+
urls: boolean
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
/**
|
|
281
|
+
* Hides the values of fields whose keys name a secret: some of the key's
|
|
282
|
+
* words in a row, written together, are a name's. DB_PASSWORD, PasswordHash,
|
|
283
|
+
* api-key and APIKey split into words at '_', '-', '.', spaces and capitals,
|
|
284
|
+
* the case ignored, so 'api key' hides api_key and apiKey, and 'token' hides
|
|
285
|
+
* bot_token and not tokens_used.
|
|
229
286
|
*/
|
|
230
287
|
export function redactor(
|
|
231
288
|
names: readonly string[] | undefined,
|
|
232
|
-
|
|
233
|
-
|
|
289
|
+
keepUrlPasswords = false,
|
|
290
|
+
): Redactor | undefined {
|
|
291
|
+
const joined = (names ?? []).map(name => words(name).join('')).filter(name => name !== '')
|
|
292
|
+
if (joined.length === 0 && keepUrlPasswords) {
|
|
234
293
|
return undefined
|
|
235
294
|
}
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
295
|
+
return {
|
|
296
|
+
hides: key => joined.length > 0 && joined.some(name => joinsTo(words(key), name)),
|
|
297
|
+
urls: !keepUrlPasswords,
|
|
298
|
+
}
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
/**
|
|
302
|
+
* A key's words as a person reads them, in lower case: split at anything but
|
|
303
|
+
* a letter or a digit, and where a capital begins a word, as in passwordHash
|
|
304
|
+
* and APIKey.
|
|
305
|
+
*/
|
|
306
|
+
export function words(key: string): string[] {
|
|
307
|
+
const out: string[] = []
|
|
308
|
+
const chars = [...key]
|
|
309
|
+
let start = -1
|
|
310
|
+
const upper = (c: string | undefined) => c !== undefined && /\p{Lu}/u.test(c)
|
|
311
|
+
const lower = (c: string | undefined) => c !== undefined && /\p{Ll}/u.test(c)
|
|
312
|
+
const digit = (c: string | undefined) => c !== undefined && /\p{N}/u.test(c)
|
|
313
|
+
for (let i = 0; i < chars.length; i++) {
|
|
314
|
+
const c = chars[i] as string
|
|
315
|
+
if (!/[\p{L}\p{N}]/u.test(c)) {
|
|
316
|
+
if (start >= 0) {
|
|
317
|
+
out.push(chars.slice(start, i).join('').toLowerCase())
|
|
318
|
+
start = -1
|
|
319
|
+
}
|
|
320
|
+
continue
|
|
321
|
+
}
|
|
322
|
+
const before = chars[i - 1]
|
|
323
|
+
const begins =
|
|
324
|
+
upper(c) && (lower(before) || digit(before) || (upper(before) && lower(chars[i + 1])))
|
|
325
|
+
if (start >= 0 && begins) {
|
|
326
|
+
out.push(chars.slice(start, i).join('').toLowerCase())
|
|
327
|
+
start = i
|
|
328
|
+
}
|
|
329
|
+
if (start < 0) {
|
|
330
|
+
start = i
|
|
331
|
+
}
|
|
332
|
+
}
|
|
333
|
+
if (start >= 0) {
|
|
334
|
+
out.push(chars.slice(start).join('').toLowerCase())
|
|
335
|
+
}
|
|
336
|
+
return out
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
function joinsTo(keyWords: string[], name: string): boolean {
|
|
340
|
+
for (let i = 0; i < keyWords.length; i++) {
|
|
341
|
+
let rest = name
|
|
342
|
+
for (const word of keyWords.slice(i)) {
|
|
343
|
+
if (!rest.startsWith(word)) {
|
|
344
|
+
break
|
|
345
|
+
}
|
|
346
|
+
rest = rest.slice(word.length)
|
|
347
|
+
if (rest === '') {
|
|
348
|
+
return true
|
|
349
|
+
}
|
|
350
|
+
}
|
|
351
|
+
}
|
|
352
|
+
return false
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
/**
|
|
356
|
+
* JSON text with the password of each URL inside it hidden, everything else
|
|
357
|
+
* as it was spelled: "postgres://ann:hunter2@db/app" is
|
|
358
|
+
* "postgres://ann:[redacted]@db/app".
|
|
359
|
+
*/
|
|
360
|
+
export function hideUrlPasswords(text: string): string {
|
|
361
|
+
if (!text.includes('://') && !text.includes('\\')) {
|
|
362
|
+
return text
|
|
363
|
+
}
|
|
364
|
+
return text.replace(/"(?:[^"\\]|\\.)*"/g, raw => {
|
|
365
|
+
const value: string = JSON.parse(raw)
|
|
366
|
+
const hidden = hideUrlTextPasswords(value)
|
|
367
|
+
return hidden === value ? raw : JSON.stringify(hidden)
|
|
368
|
+
})
|
|
369
|
+
}
|
|
370
|
+
|
|
371
|
+
function hideUrlTextPasswords(text: string): string {
|
|
372
|
+
let out = ''
|
|
373
|
+
let written = 0
|
|
374
|
+
for (let from = 0; from < text.length; ) {
|
|
375
|
+
const schemeEnd = text.indexOf('://', from)
|
|
376
|
+
if (schemeEnd < 0) {
|
|
377
|
+
break
|
|
378
|
+
}
|
|
379
|
+
const [password, at, end] = findPassword(text, schemeEnd + 3)
|
|
380
|
+
from = end
|
|
381
|
+
if (password < 0 || !/[A-Za-z][A-Za-z0-9+.-]*$/.test(text.slice(0, schemeEnd))) {
|
|
382
|
+
continue
|
|
383
|
+
}
|
|
384
|
+
out += `${text.slice(written, password)}[redacted]`
|
|
385
|
+
written = at
|
|
386
|
+
}
|
|
387
|
+
return written === 0 ? text : out + text.slice(written)
|
|
388
|
+
}
|
|
389
|
+
|
|
390
|
+
/** Where an authority's password begins and the '@' after it, -1 when it has none, and where it ends. */
|
|
391
|
+
function findPassword(text: string, start: number): [number, number, number] {
|
|
392
|
+
let at = -1
|
|
393
|
+
let colon = -1
|
|
394
|
+
let i = start
|
|
395
|
+
while (i < text.length) {
|
|
396
|
+
const c = text[i] as string
|
|
397
|
+
if (c === '/' || c === '?' || c === '#' || c <= ' ') {
|
|
398
|
+
break
|
|
399
|
+
}
|
|
400
|
+
if (c === '@') {
|
|
401
|
+
at = i
|
|
402
|
+
} else if (c === ':' && colon < 0) {
|
|
403
|
+
colon = i
|
|
404
|
+
}
|
|
405
|
+
i++
|
|
240
406
|
}
|
|
407
|
+
const end = Math.min(i, text.length)
|
|
408
|
+
return at < 0 || colon < 0 || colon + 1 >= at ? [-1, -1, end] : [colon + 1, at, end]
|
|
241
409
|
}
|
|
242
410
|
|
|
243
411
|
/** What a hidden value becomes, in the store and on the console. */
|