@tinyshed/tinystore 0.1.0-rc.5 → 0.1.0-rc.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +37 -37
- package/bin/tinystore.js +12 -1
- package/dist/backup.d.ts +33 -0
- package/dist/backup.js +49 -0
- package/dist/blobs.d.ts +160 -79
- package/dist/blobs.js +510 -217
- package/dist/channels.d.ts +90 -0
- package/dist/channels.js +291 -0
- package/dist/clock.js +3 -3
- package/dist/config.d.ts +209 -86
- package/dist/config.js +581 -538
- package/dist/connection.d.ts +64 -2
- package/dist/connection.js +201 -16
- package/dist/console.d.ts +27 -3
- package/dist/console.js +165 -12
- package/dist/dotenv.d.ts +7 -0
- package/dist/dotenv.js +79 -0
- package/dist/errors.d.ts +61 -1
- package/dist/errors.js +81 -1
- package/dist/handles.d.ts +44 -2
- package/dist/handles.js +81 -2
- package/dist/index.d.ts +17 -11
- package/dist/index.js +5 -3
- package/dist/instruments.d.ts +177 -0
- package/dist/instruments.js +819 -0
- package/dist/jobs.d.ts +335 -239
- package/dist/jobs.js +716 -536
- package/dist/json.d.ts +2 -0
- package/dist/json.js +122 -0
- package/dist/kv.d.ts +130 -243
- package/dist/kv.js +273 -640
- package/dist/limits.d.ts +80 -0
- package/dist/limits.js +139 -0
- package/dist/locks.d.ts +62 -0
- package/dist/locks.js +237 -0
- package/dist/logger.d.ts +45 -9
- package/dist/logger.js +150 -38
- package/dist/metrics.d.ts +210 -209
- package/dist/metrics.js +319 -556
- package/dist/once.d.ts +19 -18
- package/dist/once.js +74 -63
- package/dist/query.d.ts +181 -0
- package/dist/query.js +788 -0
- package/dist/records.d.ts +241 -150
- package/dist/records.js +507 -291
- package/dist/runtime/addon.d.ts +14 -0
- package/dist/runtime/addon.js +69 -0
- package/dist/runtime/bun.js +5 -7
- package/dist/runtime/node.js +5 -7
- package/dist/runtime/pipe.d.ts +17 -0
- package/dist/runtime/pipe.js +133 -0
- package/dist/runtime/shared.d.ts +6 -0
- package/dist/runtime/shared.js +19 -0
- package/dist/runtime.d.ts +5 -2
- package/dist/search.d.ts +189 -0
- package/dist/search.js +281 -0
- package/dist/session.d.ts +31 -3
- package/dist/session.js +153 -41
- package/dist/sql.d.ts +228 -78
- package/dist/sql.js +745 -240
- package/dist/store.d.ts +244 -24
- package/dist/store.js +270 -67
- package/dist/time.d.ts +5 -1
- package/dist/time.js +22 -2
- package/dist/tx.d.ts +61 -0
- package/dist/tx.js +314 -0
- package/dist/wire/codec.d.ts +23 -7
- package/dist/wire/codec.js +56 -35
- package/dist/wire/frame.d.ts +2 -0
- package/dist/wire/frame.js +31 -14
- package/dist/wire/messages.d.ts +259 -0
- package/dist/wire/messages.js +22 -0
- package/dist/wire/msgpack.d.ts +49 -1
- package/dist/wire/msgpack.js +293 -56
- package/dist/wire/protocol.d.ts +2683 -0
- package/dist/wire/protocol.js +10846 -0
- package/package.json +39 -38
- package/src/backup.ts +81 -0
- package/src/blobs.ts +678 -301
- package/src/channels.ts +377 -0
- package/src/clock.ts +4 -4
- package/src/config.ts +902 -612
- package/src/connection.ts +279 -21
- package/src/console.ts +180 -12
- package/src/dotenv.ts +82 -0
- package/src/errors.ts +105 -0
- package/src/handles.ts +157 -8
- package/src/index.ts +158 -52
- package/src/instruments.ts +1042 -0
- package/src/jobs.ts +1017 -733
- package/src/json.ts +133 -0
- package/src/kv.ts +424 -847
- package/src/limits.ts +202 -0
- package/src/locks.ts +320 -0
- package/src/logger.ts +209 -40
- package/src/metrics.ts +557 -752
- package/src/once.ts +86 -83
- package/src/query.ts +1005 -0
- package/src/records.ts +795 -418
- package/src/runtime/addon.ts +99 -0
- package/src/runtime/bun.ts +5 -7
- package/src/runtime/node.ts +5 -7
- package/src/runtime/pipe.ts +178 -0
- package/src/runtime/shared.ts +20 -0
- package/src/runtime.ts +5 -2
- package/src/search.ts +480 -0
- package/src/session.ts +187 -41
- package/src/sql.ts +993 -306
- package/src/store.ts +473 -78
- package/src/time.ts +48 -3
- package/src/tx.ts +426 -0
- package/src/wire/codec.ts +78 -37
- package/src/wire/frame.ts +44 -14
- package/src/wire/messages.ts +25 -0
- package/src/wire/msgpack.ts +327 -56
- package/src/wire/protocol.ts +11709 -0
- package/dist/limiter.d.ts +0 -45
- package/dist/limiter.js +0 -68
- package/dist/quota.d.ts +0 -44
- package/dist/quota.js +0 -98
- package/src/limiter.ts +0 -99
- package/src/quota.ts +0 -137
package/dist/connection.d.ts
CHANGED
|
@@ -1,13 +1,37 @@
|
|
|
1
1
|
import type { PrivateChild, Runtime, TlsOptions, Transport } from './runtime.ts';
|
|
2
2
|
import { Session, type SessionOptions, type Stream } from './session.ts';
|
|
3
|
+
import type { NodeOptions } from './store.ts';
|
|
4
|
+
/**
|
|
5
|
+
* How far a commit goes before it returns, for the files of a store or of
|
|
6
|
+
* one database:
|
|
7
|
+
*
|
|
8
|
+
* 'full' synced to the disk: it survives a crash of the operating
|
|
9
|
+
* system and a power loss. What every file is unless told.
|
|
10
|
+
* 'os' written to the operating system, synced a little later: it
|
|
11
|
+
* survives the program's crash, and a power loss may take the
|
|
12
|
+
* last commits, never the file.
|
|
13
|
+
*/
|
|
14
|
+
export type Durability = 'full' | 'os';
|
|
15
|
+
/** Refuses a durability that is neither word, before anything leaves. */
|
|
16
|
+
export declare function checkDurability(durability: Durability | undefined, of: string): void;
|
|
17
|
+
/**
|
|
18
|
+
* Dials as `dial` does, and refuses a store whose files commit another way
|
|
19
|
+
* than this program asked for: a store has one durability, its first
|
|
20
|
+
* opener's, and a program that needs another must not be told it has it.
|
|
21
|
+
*/
|
|
22
|
+
export declare function asking(dial: Dialer, dir: string, durability: Durability | undefined): Dialer;
|
|
3
23
|
export declare class Connection {
|
|
4
24
|
#private;
|
|
5
25
|
readonly session: Session;
|
|
6
26
|
/** the handles this connection opened, by what they are: a bucket's name and options */
|
|
7
27
|
readonly handles: Map<string, Promise<number>>;
|
|
28
|
+
/** the handle each open was answered, once it was: what a call made in place reads without waiting */
|
|
29
|
+
readonly opened: Map<object, number>;
|
|
8
30
|
constructor(session: Session, transport: Transport, closing?: () => Promise<unknown>);
|
|
9
31
|
/** Connects, shakes hands, and gives up after the handshake's time. */
|
|
10
32
|
static dial(runtime: Runtime, endpoint: string, options: SessionOptions, tls?: TlsOptions): Promise<Connection>;
|
|
33
|
+
/** Shakes hands over a pipe to the core in this process. */
|
|
34
|
+
static overPipe(session: Session, transport: Transport): Promise<Connection>;
|
|
11
35
|
/**
|
|
12
36
|
* Shakes hands over a private child's stdin and stdout. Closing waits for
|
|
13
37
|
* the child's exit, since until then it holds the directory: its LOCK, and
|
|
@@ -27,7 +51,26 @@ export type Dialer = () => Promise<Connection>;
|
|
|
27
51
|
* this SDK's is stopped and replaced with this SDK's binary; another server
|
|
28
52
|
* of an older release, a person's or a program's own, is told of once.
|
|
29
53
|
*/
|
|
30
|
-
export declare function sidecar(runtime: Runtime, dir: string, binary: () => string,
|
|
54
|
+
export declare function sidecar(runtime: Runtime, dir: string, binary: () => string, started?: Started, older?: (server: string) => boolean): Dialer;
|
|
55
|
+
/** What a server this process starts is started with; one found running keeps its own. */
|
|
56
|
+
export interface Started {
|
|
57
|
+
/** how long a sidecar stays once its last connection has gone, in milliseconds */
|
|
58
|
+
idle?: number | undefined;
|
|
59
|
+
durability?: Durability | undefined;
|
|
60
|
+
/** the file of the store's encryption key, when it is not the store's own */
|
|
61
|
+
encryptionKeyFile?: string | undefined;
|
|
62
|
+
/** the node the store is, when it is one */
|
|
63
|
+
node?: NodeOptions | undefined;
|
|
64
|
+
/** measures the store into its own metrics, the machine too unless it says system: false */
|
|
65
|
+
selfMetrics?: SelfMetrics | undefined;
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* What a store measures of itself: `true` its commits, files and memory and
|
|
69
|
+
* the machine it runs on, `{ system: false }` the same without the machine.
|
|
70
|
+
*/
|
|
71
|
+
export type SelfMetrics = boolean | {
|
|
72
|
+
system?: boolean;
|
|
73
|
+
};
|
|
31
74
|
/**
|
|
32
75
|
* Whether a server's release is older than the SDK's own; a build that is no
|
|
33
76
|
* release, a development copy's or a Go pseudo-version, is neither.
|
|
@@ -38,7 +81,12 @@ export declare function replacedSidecar(server: string, own: string, dir: string
|
|
|
38
81
|
/** What a program whose SDK found a server of an older release that is no sidecar is told. */
|
|
39
82
|
export declare function olderServer(server: string, own: string, dir: string): string;
|
|
40
83
|
/** A private child: tinystore serve --stdio, living and dying with this process. */
|
|
41
|
-
export declare function privateChild(runtime: Runtime, dir: string, binary: () => string, clock?: Date): Dialer;
|
|
84
|
+
export declare function privateChild(runtime: Runtime, dir: string, binary: () => string, clock?: Date, started?: Started): Dialer;
|
|
85
|
+
/**
|
|
86
|
+
* The core in this process: no server and no socket, the frames handed to
|
|
87
|
+
* the core and back, through bun:ffi under Bun and its addon under Node.
|
|
88
|
+
*/
|
|
89
|
+
export declare function embedded(runtime: Runtime, dir: string, library: () => string, started?: Started): Dialer;
|
|
42
90
|
/** A remote server: tls:// checks its certificate before the token leaves; tcp:// does not. */
|
|
43
91
|
export declare function remote(runtime: Runtime, url: string, token: string, tls?: TlsOptions): Dialer;
|
|
44
92
|
/** What a call may do when its connection is lost: go again, or say its outcome is unknown. */
|
|
@@ -54,6 +102,8 @@ export declare class Link {
|
|
|
54
102
|
constructor(dial: Dialer);
|
|
55
103
|
/** The connection a call goes on. */
|
|
56
104
|
connection(): Promise<Connection>;
|
|
105
|
+
/** The connection a call goes on, when one is there to take it without waiting. */
|
|
106
|
+
ready(): Connection | undefined;
|
|
57
107
|
/**
|
|
58
108
|
* Runs a call on the link's connection. build makes its REQUEST's body for
|
|
59
109
|
* that connection, whose handles it may open; run sends it.
|
|
@@ -62,6 +112,12 @@ export declare class Link {
|
|
|
62
112
|
/** Closes the connection and every one to come. */
|
|
63
113
|
close(): Promise<void>;
|
|
64
114
|
}
|
|
115
|
+
/**
|
|
116
|
+
* Whether a call that failed goes on another connection: one whose REQUEST
|
|
117
|
+
* had not left, and a read that had. A write that had is OutcomeUnknownError,
|
|
118
|
+
* thrown here; any other failure is the call's own.
|
|
119
|
+
*/
|
|
120
|
+
export declare function goesAgain(err: unknown, idempotence: Idempotence): boolean;
|
|
65
121
|
/** What a download sent: its header, its items, and the trailer that ended it. */
|
|
66
122
|
export interface Downloaded {
|
|
67
123
|
header: Uint8Array;
|
|
@@ -76,3 +132,9 @@ export interface Downloaded {
|
|
|
76
132
|
*/
|
|
77
133
|
export declare function download(connection: Connection, method: number, body: Uint8Array, signal?: AbortSignal): Promise<Downloaded>;
|
|
78
134
|
export type { Stream };
|
|
135
|
+
/**
|
|
136
|
+
* The bytes a piece carries: half what the server takes on a stream before
|
|
137
|
+
* it grants more, so that one piece goes while the last is written, less a
|
|
138
|
+
* piece's own few bytes.
|
|
139
|
+
*/
|
|
140
|
+
export declare function pieceOf(connection: Connection): number;
|
package/dist/connection.js
CHANGED
|
@@ -3,12 +3,39 @@
|
|
|
3
3
|
// directory's sidecar, found through SERVE or started, a private child, or a
|
|
4
4
|
// remote server.
|
|
5
5
|
import { randomBytes } from 'node:crypto';
|
|
6
|
+
import { mkdirSync, readFileSync, writeFileSync } from 'node:fs';
|
|
6
7
|
import { join, resolve } from 'node:path';
|
|
7
8
|
import manifest from '../package.json' with { type: 'json' };
|
|
8
9
|
import { currentSignal } from './cancel.js';
|
|
9
|
-
import { ClosedError, OutcomeUnknownError, TinystoreError, UnavailableError } from './errors.js';
|
|
10
|
-
import {
|
|
11
|
-
import { Empty, methods } from './wire/
|
|
10
|
+
import { ClosedError, InvalidError, OutcomeUnknownError, TinystoreError, UnavailableError, } from './errors.js';
|
|
11
|
+
import { ConnectionLostError, protocol, Session, watch, } from './session.js';
|
|
12
|
+
import { Empty, methods, StoreOptions } from './wire/protocol.js';
|
|
13
|
+
/** Refuses a durability that is neither word, before anything leaves. */
|
|
14
|
+
export function checkDurability(durability, of) {
|
|
15
|
+
if (durability !== undefined && durability !== 'full' && durability !== 'os') {
|
|
16
|
+
throw new InvalidError(`${of}: durability ${JSON.stringify(durability)}: it is 'full' or 'os'`);
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* Dials as `dial` does, and refuses a store whose files commit another way
|
|
21
|
+
* than this program asked for: a store has one durability, its first
|
|
22
|
+
* opener's, and a program that needs another must not be told it has it.
|
|
23
|
+
*/
|
|
24
|
+
export function asking(dial, dir, durability) {
|
|
25
|
+
if (durability === undefined) {
|
|
26
|
+
return dial;
|
|
27
|
+
}
|
|
28
|
+
return async () => {
|
|
29
|
+
const connection = await dial();
|
|
30
|
+
const opened = (await connection.session.welcomed).durability;
|
|
31
|
+
if (opened !== durability) {
|
|
32
|
+
await connection.close();
|
|
33
|
+
const as = opened === undefined ? 'with none said' : `with durability '${opened}'`;
|
|
34
|
+
throw new InvalidError(`the store in ${dir} is open ${as}, and this program asked for '${durability}'`);
|
|
35
|
+
}
|
|
36
|
+
return connection;
|
|
37
|
+
};
|
|
38
|
+
}
|
|
12
39
|
/** How long a server may take to answer HELLO. */
|
|
13
40
|
const handshakeTime = 5000;
|
|
14
41
|
export class Connection {
|
|
@@ -18,6 +45,8 @@ export class Connection {
|
|
|
18
45
|
#closing;
|
|
19
46
|
/** the handles this connection opened, by what they are: a bucket's name and options */
|
|
20
47
|
handles = new Map();
|
|
48
|
+
/** the handle each open was answered, once it was: what a call made in place reads without waiting */
|
|
49
|
+
opened = new Map();
|
|
21
50
|
constructor(session, transport, closing = () => Promise.resolve()) {
|
|
22
51
|
this.session = session;
|
|
23
52
|
this.#transport = transport;
|
|
@@ -47,6 +76,12 @@ export class Connection {
|
|
|
47
76
|
await connection.#handshake();
|
|
48
77
|
return connection;
|
|
49
78
|
}
|
|
79
|
+
/** Shakes hands over a pipe to the core in this process. */
|
|
80
|
+
static async overPipe(session, transport) {
|
|
81
|
+
const connection = new Connection(session, transport);
|
|
82
|
+
await connection.#handshake();
|
|
83
|
+
return connection;
|
|
84
|
+
}
|
|
50
85
|
/**
|
|
51
86
|
* Shakes hands over a private child's stdin and stdout. Closing waits for
|
|
52
87
|
* the child's exit, since until then it holds the directory: its LOCK, and
|
|
@@ -90,11 +125,11 @@ const heldPause = 100;
|
|
|
90
125
|
* this SDK's is stopped and replaced with this SDK's binary; another server
|
|
91
126
|
* of an older release, a person's or a program's own, is told of once.
|
|
92
127
|
*/
|
|
93
|
-
export function sidecar(runtime, dir, binary,
|
|
128
|
+
export function sidecar(runtime, dir, binary, started = {}, older = server => olderRelease(server, manifest.version)) {
|
|
94
129
|
const absolute = resolve(dir);
|
|
95
130
|
const serve = join(absolute, 'server', 'SERVE');
|
|
96
131
|
const log = join(absolute, 'server', 'serve.log');
|
|
97
|
-
const idling = idle === undefined ? [] : ['--idle', `${idle}ms`];
|
|
132
|
+
const idling = started.idle === undefined ? [] : ['--idle', `${started.idle}ms`];
|
|
98
133
|
let told = false;
|
|
99
134
|
return async () => {
|
|
100
135
|
const found = await reachServe(runtime, serve);
|
|
@@ -116,10 +151,76 @@ export function sidecar(runtime, dir, binary, idle, older = server => olderRelea
|
|
|
116
151
|
return found.connection;
|
|
117
152
|
}
|
|
118
153
|
}
|
|
119
|
-
const
|
|
154
|
+
const flags = [...idling, ...storeFlags(started, absolute)];
|
|
155
|
+
const command = [binary(), 'serve', '--dir', absolute, '--local', '--log-file', log, ...flags];
|
|
120
156
|
return startSidecar(runtime, serve, log, command, stoppedInstance);
|
|
121
157
|
};
|
|
122
158
|
}
|
|
159
|
+
/** What selfMetrics asks for, none when it asks for nothing. */
|
|
160
|
+
function selfMetricsOf(given) {
|
|
161
|
+
if (given === undefined || given === false) {
|
|
162
|
+
return undefined;
|
|
163
|
+
}
|
|
164
|
+
return { system: given === true || given.system !== false };
|
|
165
|
+
}
|
|
166
|
+
/** The flags of `tinystore serve` that say how the store in `dir` opens. */
|
|
167
|
+
function storeFlags(started, dir) {
|
|
168
|
+
const committing = started.durability === undefined ? [] : ['--durability', started.durability];
|
|
169
|
+
const key = started.encryptionKeyFile;
|
|
170
|
+
const keyed = key === undefined ? committing : [...committing, '--encryption-key-file', resolve(key)];
|
|
171
|
+
const asked = selfMetricsOf(started.selfMetrics);
|
|
172
|
+
const measured = asked === undefined
|
|
173
|
+
? keyed
|
|
174
|
+
: [...keyed, '--self-metrics', ...(asked.system ? [] : ['--no-system-metrics'])];
|
|
175
|
+
return [...measured, ...nodeFlags(started.node, join(dir, 'server', 'node.toml'))];
|
|
176
|
+
}
|
|
177
|
+
/**
|
|
178
|
+
* A node's options as `tinystore serve` takes them: where it listens and the
|
|
179
|
+
* nodes it copies as flags, and the rest in a node config written to
|
|
180
|
+
* `config`, which the server reads at its start.
|
|
181
|
+
*/
|
|
182
|
+
function nodeFlags(node, config) {
|
|
183
|
+
if (node === undefined) {
|
|
184
|
+
return [];
|
|
185
|
+
}
|
|
186
|
+
if (node.token !== undefined) {
|
|
187
|
+
throw new InvalidError("a node's token reaches a server this process starts through a file: give tokenFile, or open embedded");
|
|
188
|
+
}
|
|
189
|
+
const flags = [];
|
|
190
|
+
for (const from of node.from ?? []) {
|
|
191
|
+
flags.push('--from', from);
|
|
192
|
+
}
|
|
193
|
+
if (node.serve !== undefined) {
|
|
194
|
+
flags.push('--node-listen', node.serve);
|
|
195
|
+
}
|
|
196
|
+
// a JSON string or list of strings is TOML's too
|
|
197
|
+
const settings = [];
|
|
198
|
+
if (node.files !== undefined) {
|
|
199
|
+
settings.push(`files = ${JSON.stringify(node.files)}`);
|
|
200
|
+
}
|
|
201
|
+
if (node.tokenFile !== undefined) {
|
|
202
|
+
settings.push(`token-file = ${JSON.stringify(resolve(node.tokenFile))}`);
|
|
203
|
+
}
|
|
204
|
+
if (node.synchronous === true) {
|
|
205
|
+
settings.push('synchronous = true');
|
|
206
|
+
}
|
|
207
|
+
if (node.synchronousWith !== undefined && node.synchronousWith.length > 0) {
|
|
208
|
+
settings.push(`synchronous-with = ${JSON.stringify(node.synchronousWith)}`);
|
|
209
|
+
}
|
|
210
|
+
if (node.insecure === true) {
|
|
211
|
+
settings.push('insecure = true');
|
|
212
|
+
}
|
|
213
|
+
if (node.catalog === true) {
|
|
214
|
+
settings.push('catalog = true');
|
|
215
|
+
}
|
|
216
|
+
if (node.advertise !== undefined) {
|
|
217
|
+
settings.push(`advertise = ${JSON.stringify(node.advertise)}`);
|
|
218
|
+
}
|
|
219
|
+
mkdirSync(join(config, '..'), { recursive: true });
|
|
220
|
+
writeFileSync(config, `${settings.join('\n')}\n`);
|
|
221
|
+
flags.push('--node-config', config);
|
|
222
|
+
return flags;
|
|
223
|
+
}
|
|
123
224
|
/**
|
|
124
225
|
* Starts a sidecar and waits for SERVE to name one that answers, its own or
|
|
125
226
|
* another client's that won the start. One that exits with 3 found the
|
|
@@ -246,7 +347,7 @@ async function reachServe(runtime, serve) {
|
|
|
246
347
|
}
|
|
247
348
|
const endpoint = published.endpoints?.[0];
|
|
248
349
|
const secret = Buffer.from(published.secret ?? '', 'base64url');
|
|
249
|
-
if (published.protocol !==
|
|
350
|
+
if (published.protocol !== protocol || endpoint === undefined || secret.length !== 32) {
|
|
250
351
|
return undefined;
|
|
251
352
|
}
|
|
252
353
|
try {
|
|
@@ -273,12 +374,20 @@ async function tail(runtime, log) {
|
|
|
273
374
|
return lines.slice(-5).join('\n');
|
|
274
375
|
}
|
|
275
376
|
/** A private child: tinystore serve --stdio, living and dying with this process. */
|
|
276
|
-
export function privateChild(runtime, dir, binary, clock) {
|
|
377
|
+
export function privateChild(runtime, dir, binary, clock, started = {}) {
|
|
277
378
|
// a test's clock starts where the first child's did, and a child started again starts it there
|
|
278
379
|
const clocked = clock === undefined ? [] : ['--clock', clock.toISOString()];
|
|
279
380
|
return async () => {
|
|
280
381
|
let session;
|
|
281
|
-
const argv = [
|
|
382
|
+
const argv = [
|
|
383
|
+
binary(),
|
|
384
|
+
'serve',
|
|
385
|
+
'--dir',
|
|
386
|
+
resolve(dir),
|
|
387
|
+
'--stdio',
|
|
388
|
+
...clocked,
|
|
389
|
+
...storeFlags(started, resolve(dir)),
|
|
390
|
+
];
|
|
282
391
|
const child = runtime.spawnPrivate(argv, {
|
|
283
392
|
data: bytes => session?.receive(bytes),
|
|
284
393
|
end: err => session?.end(err),
|
|
@@ -299,6 +408,49 @@ export function privateChild(runtime, dir, binary, clock) {
|
|
|
299
408
|
}
|
|
300
409
|
};
|
|
301
410
|
}
|
|
411
|
+
/**
|
|
412
|
+
* The core in this process: no server and no socket, the frames handed to
|
|
413
|
+
* the core and back, through bun:ffi under Bun and its addon under Node.
|
|
414
|
+
*/
|
|
415
|
+
export function embedded(runtime, dir, library, started = {}) {
|
|
416
|
+
const encryptionKeyFile = started.encryptionKeyFile === undefined ? undefined : resolve(started.encryptionKeyFile);
|
|
417
|
+
const node = started.node;
|
|
418
|
+
const token = node?.token ??
|
|
419
|
+
(node?.tokenFile === undefined
|
|
420
|
+
? undefined
|
|
421
|
+
: readFileSync(resolve(node.tokenFile), 'utf8').trim());
|
|
422
|
+
const options = StoreOptions.encode({
|
|
423
|
+
durability: started.durability,
|
|
424
|
+
encryptionKeyFile,
|
|
425
|
+
node: node !== undefined,
|
|
426
|
+
from: node?.from ?? [],
|
|
427
|
+
serve: node?.serve,
|
|
428
|
+
files: node?.files ?? [],
|
|
429
|
+
token,
|
|
430
|
+
synchronous: node?.synchronous === true,
|
|
431
|
+
catalog: node?.catalog === true,
|
|
432
|
+
insecure: node?.insecure === true,
|
|
433
|
+
advertise: node?.advertise,
|
|
434
|
+
synchronousWith: node?.synchronousWith ?? [],
|
|
435
|
+
selfMetrics: selfMetricsOf(started.selfMetrics) !== undefined,
|
|
436
|
+
withoutSystem: selfMetricsOf(started.selfMetrics)?.system === false,
|
|
437
|
+
});
|
|
438
|
+
return async () => {
|
|
439
|
+
let session;
|
|
440
|
+
const end = (err) => session?.end(err);
|
|
441
|
+
const transport = typeof Bun === 'undefined'
|
|
442
|
+
? (await import('./runtime/addon.js')).openAddon(library(), resolve(dir), options, {
|
|
443
|
+
data: bytes => session?.receive(bytes),
|
|
444
|
+
end,
|
|
445
|
+
})
|
|
446
|
+
: (await import('./runtime/pipe.js')).openPipe(library(), resolve(dir), options, {
|
|
447
|
+
frames: (bytes, length) => session?.read(bytes, length) ?? length,
|
|
448
|
+
end,
|
|
449
|
+
});
|
|
450
|
+
session = new Session(bytes => transport.write(bytes), { client: runtime.client });
|
|
451
|
+
return await Connection.overPipe(session, transport);
|
|
452
|
+
};
|
|
453
|
+
}
|
|
302
454
|
/** A remote server: tls:// checks its certificate before the token leaves; tcp:// does not. */
|
|
303
455
|
export function remote(runtime, url, token, tls) {
|
|
304
456
|
return () => Connection.dial(runtime, url, { client: runtime.client, token }, tls);
|
|
@@ -312,6 +464,7 @@ export function remote(runtime, url, token, tls) {
|
|
|
312
464
|
export class Link {
|
|
313
465
|
#dial;
|
|
314
466
|
#current;
|
|
467
|
+
#dialled;
|
|
315
468
|
#closed = false;
|
|
316
469
|
constructor(dial) {
|
|
317
470
|
this.#dial = dial;
|
|
@@ -327,6 +480,11 @@ export class Link {
|
|
|
327
480
|
}
|
|
328
481
|
return this.#redial(undefined);
|
|
329
482
|
}
|
|
483
|
+
/** The connection a call goes on, when one is there to take it without waiting. */
|
|
484
|
+
ready() {
|
|
485
|
+
const c = this.#dialled;
|
|
486
|
+
return c !== undefined && c.session.ended === undefined && !c.session.goingAway ? c : undefined;
|
|
487
|
+
}
|
|
330
488
|
// one dial at a time: every caller that found the same dead connection
|
|
331
489
|
// waits for the one new one
|
|
332
490
|
#redial(dead) {
|
|
@@ -335,7 +493,12 @@ export class Link {
|
|
|
335
493
|
}
|
|
336
494
|
const dialled = this.#dial();
|
|
337
495
|
this.#current = dialled;
|
|
338
|
-
dialled
|
|
496
|
+
this.#dialled = undefined;
|
|
497
|
+
dialled.then(connection => {
|
|
498
|
+
if (this.#current === dialled) {
|
|
499
|
+
this.#dialled = connection;
|
|
500
|
+
}
|
|
501
|
+
}, () => {
|
|
339
502
|
if (this.#current === dialled) {
|
|
340
503
|
this.#current = undefined;
|
|
341
504
|
}
|
|
@@ -355,12 +518,8 @@ export class Link {
|
|
|
355
518
|
return await attempt(connection);
|
|
356
519
|
}
|
|
357
520
|
catch (err) {
|
|
358
|
-
if (tries < 2 && (err
|
|
359
|
-
|
|
360
|
-
if (!sent || idempotence === 'read') {
|
|
361
|
-
continue;
|
|
362
|
-
}
|
|
363
|
-
throw new OutcomeUnknownError(`the connection was lost with the write in flight; read what it wrote before writing again: ${err.message}`);
|
|
521
|
+
if (tries < 2 && goesAgain(err, idempotence)) {
|
|
522
|
+
continue;
|
|
364
523
|
}
|
|
365
524
|
throw err;
|
|
366
525
|
}
|
|
@@ -371,9 +530,25 @@ export class Link {
|
|
|
371
530
|
this.#closed = true;
|
|
372
531
|
const current = this.#current;
|
|
373
532
|
this.#current = undefined;
|
|
533
|
+
this.#dialled = undefined;
|
|
374
534
|
await current?.then(c => c.close(), () => { });
|
|
375
535
|
}
|
|
376
536
|
}
|
|
537
|
+
/**
|
|
538
|
+
* Whether a call that failed goes on another connection: one whose REQUEST
|
|
539
|
+
* had not left, and a read that had. A write that had is OutcomeUnknownError,
|
|
540
|
+
* thrown here; any other failure is the call's own.
|
|
541
|
+
*/
|
|
542
|
+
export function goesAgain(err, idempotence) {
|
|
543
|
+
if (!(err instanceof ConnectionLostError || err instanceof UnavailableError)) {
|
|
544
|
+
return false;
|
|
545
|
+
}
|
|
546
|
+
const sent = err instanceof ConnectionLostError && err.sent;
|
|
547
|
+
if (!sent || idempotence === 'read') {
|
|
548
|
+
return true;
|
|
549
|
+
}
|
|
550
|
+
throw new OutcomeUnknownError(`the connection was lost with the write in flight; read what it wrote before writing again: ${err.message}`);
|
|
551
|
+
}
|
|
377
552
|
/**
|
|
378
553
|
* Runs a download to its end: the RESPONSE that heads it, an item a DATA, and
|
|
379
554
|
* the DATA that ends it, giving each item's credit back as it arrives, since
|
|
@@ -403,3 +578,13 @@ export async function download(connection, method, body, signal) {
|
|
|
403
578
|
unwatch();
|
|
404
579
|
}
|
|
405
580
|
}
|
|
581
|
+
/**
|
|
582
|
+
* The bytes a piece carries: half what the server takes on a stream before
|
|
583
|
+
* it grants more, so that one piece goes while the last is written, less a
|
|
584
|
+
* piece's own few bytes.
|
|
585
|
+
*/
|
|
586
|
+
export function pieceOf(connection) {
|
|
587
|
+
const agreed = connection.session.agreed;
|
|
588
|
+
const most = Math.min(agreed?.maxBody ?? 1 << 20, agreed?.streamCredit ?? 1 << 20);
|
|
589
|
+
return Math.max(1, Math.floor(most / 2) - 16);
|
|
590
|
+
}
|
package/dist/console.d.ts
CHANGED
|
@@ -42,11 +42,35 @@ export declare function jsonLine(line: ConsoleLine): string;
|
|
|
42
42
|
export declare function prettyLine(line: ConsoleLine, look: Look): string;
|
|
43
43
|
/** A level as slog spells it: INFO, WARN+2, DEBUG-4. */
|
|
44
44
|
export declare function levelLabel(level: number): string;
|
|
45
|
+
/** The field where a line says where it was logged, as slog's handlers spell it. */
|
|
46
|
+
export declare const sourceKey = "source";
|
|
47
|
+
/** The names `redact` takes to hide the usual secrets, as Go's console.Secrets and Python's SECRETS. */
|
|
48
|
+
export declare const secrets: readonly string[];
|
|
49
|
+
/** What a logger hides: the keys its names name, and a URL's password unless it keeps them. */
|
|
50
|
+
export interface Redactor {
|
|
51
|
+
hides(key: string): boolean;
|
|
52
|
+
urls: boolean;
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* Hides the values of fields whose keys name a secret: some of the key's
|
|
56
|
+
* words in a row, written together, are a name's. DB_PASSWORD, PasswordHash,
|
|
57
|
+
* api-key and APIKey split into words at '_', '-', '.', spaces and capitals,
|
|
58
|
+
* the case ignored, so 'api key' hides api_key and apiKey, and 'token' hides
|
|
59
|
+
* bot_token and not tokens_used.
|
|
60
|
+
*/
|
|
61
|
+
export declare function redactor(names: readonly string[] | undefined, keepUrlPasswords?: boolean): Redactor | undefined;
|
|
62
|
+
/**
|
|
63
|
+
* A key's words as a person reads them, in lower case: split at anything but
|
|
64
|
+
* a letter or a digit, and where a capital begins a word, as in passwordHash
|
|
65
|
+
* and APIKey.
|
|
66
|
+
*/
|
|
67
|
+
export declare function words(key: string): string[];
|
|
45
68
|
/**
|
|
46
|
-
*
|
|
47
|
-
*
|
|
69
|
+
* JSON text with the password of each URL inside it hidden, everything else
|
|
70
|
+
* as it was spelled: "postgres://ann:hunter2@db/app" is
|
|
71
|
+
* "postgres://ann:[redacted]@db/app".
|
|
48
72
|
*/
|
|
49
|
-
export declare function
|
|
73
|
+
export declare function hideUrlPasswords(text: string): string;
|
|
50
74
|
/** What a hidden value becomes, in the store and on the console. */
|
|
51
75
|
export declare const redacted = "\"[redacted]\"";
|
|
52
76
|
/**
|
package/dist/console.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
// A logger's console line, as
|
|
2
|
-
//
|
|
1
|
+
// A logger's console line, as Rust's console and Python's handler write it,
|
|
2
|
+
// byte for byte the lines of testdata/console/console.json: pretty for a
|
|
3
3
|
// person at a terminal, one JSON object a line for a collector.
|
|
4
4
|
//
|
|
5
5
|
// 11:02:11.123 WARN api slow request requestId=7f3a ms=1200
|
|
@@ -73,7 +73,7 @@ export function prettyLine(line, look) {
|
|
|
73
73
|
const below = [];
|
|
74
74
|
for (const fields of [line.context, line.attrs]) {
|
|
75
75
|
for (let i = 0; i < fields.length; i += 2) {
|
|
76
|
-
const [shown, lines] = prettyValue(fields[i + 1]);
|
|
76
|
+
const [shown, lines] = prettyValue(fields[i], fields[i + 1]);
|
|
77
77
|
if (lines) {
|
|
78
78
|
below.push([fields[i], shown]);
|
|
79
79
|
}
|
|
@@ -128,7 +128,13 @@ function prettyMessage(line) {
|
|
|
128
128
|
return body === '' ? line.event : `${line.event} ${body}`;
|
|
129
129
|
}
|
|
130
130
|
/** How a field's JSON shows, and whether it is text of several lines, which goes under the line. */
|
|
131
|
-
function prettyValue(json) {
|
|
131
|
+
function prettyValue(key, json) {
|
|
132
|
+
if (key === sourceKey) {
|
|
133
|
+
const place = shortSource(json);
|
|
134
|
+
if (place !== undefined) {
|
|
135
|
+
return [place, false];
|
|
136
|
+
}
|
|
137
|
+
}
|
|
132
138
|
if (!json.startsWith('"')) {
|
|
133
139
|
return [json, false];
|
|
134
140
|
}
|
|
@@ -182,20 +188,167 @@ function stamp(at, look) {
|
|
|
182
188
|
const zone = `${offset < 0 ? '-' : '+'}${two(Math.trunc(Math.abs(offset) / 60))}:${two(Math.abs(offset) % 60)}`;
|
|
183
189
|
return `${String(year).padStart(4, '0')}-${two(month)}-${two(day)} ${clock} ${zone}`;
|
|
184
190
|
}
|
|
191
|
+
/** The field where a line says where it was logged, as slog's handlers spell it. */
|
|
192
|
+
export const sourceKey = 'source';
|
|
193
|
+
/**
|
|
194
|
+
* A source as a pretty line shows it, its file's directory and name and its
|
|
195
|
+
* line: {"function":"run","file":"/home/ann/app/server/main.ts","line":42} is
|
|
196
|
+
* server/main.ts:42.
|
|
197
|
+
*/
|
|
198
|
+
function shortSource(json) {
|
|
199
|
+
if (!json.startsWith('{')) {
|
|
200
|
+
return undefined;
|
|
201
|
+
}
|
|
202
|
+
const source = JSON.parse(json);
|
|
203
|
+
if (typeof source.file !== 'string' || source.file === '') {
|
|
204
|
+
return undefined;
|
|
205
|
+
}
|
|
206
|
+
const parts = source.file.split(/[\\/]+/).filter(part => part !== '');
|
|
207
|
+
const line = typeof source.line === 'number' ? source.line : 0;
|
|
208
|
+
return `${parts.slice(-2).join('/')}:${line}`;
|
|
209
|
+
}
|
|
210
|
+
/** The names `redact` takes to hide the usual secrets, as Go's console.Secrets and Python's SECRETS. */
|
|
211
|
+
export const secrets = Object.freeze([
|
|
212
|
+
'password',
|
|
213
|
+
'passwd',
|
|
214
|
+
'passphrase',
|
|
215
|
+
'secret',
|
|
216
|
+
'token',
|
|
217
|
+
'credential',
|
|
218
|
+
'credentials',
|
|
219
|
+
'authorization',
|
|
220
|
+
'cookie',
|
|
221
|
+
'api key',
|
|
222
|
+
'private key',
|
|
223
|
+
'secret key',
|
|
224
|
+
'access key',
|
|
225
|
+
'signing key',
|
|
226
|
+
'encryption key',
|
|
227
|
+
'connection string',
|
|
228
|
+
'dsn',
|
|
229
|
+
]);
|
|
185
230
|
/**
|
|
186
|
-
*
|
|
187
|
-
*
|
|
231
|
+
* Hides the values of fields whose keys name a secret: some of the key's
|
|
232
|
+
* words in a row, written together, are a name's. DB_PASSWORD, PasswordHash,
|
|
233
|
+
* api-key and APIKey split into words at '_', '-', '.', spaces and capitals,
|
|
234
|
+
* the case ignored, so 'api key' hides api_key and apiKey, and 'token' hides
|
|
235
|
+
* bot_token and not tokens_used.
|
|
188
236
|
*/
|
|
189
|
-
export function redactor(names) {
|
|
190
|
-
|
|
237
|
+
export function redactor(names, keepUrlPasswords = false) {
|
|
238
|
+
const joined = (names ?? []).map(name => words(name).join('')).filter(name => name !== '');
|
|
239
|
+
if (joined.length === 0 && keepUrlPasswords) {
|
|
191
240
|
return undefined;
|
|
192
241
|
}
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
return hidden.has(lower) || hidden.has(lower.slice(lower.lastIndexOf('.') + 1));
|
|
242
|
+
return {
|
|
243
|
+
hides: key => joined.length > 0 && joined.some(name => joinsTo(words(key), name)),
|
|
244
|
+
urls: !keepUrlPasswords,
|
|
197
245
|
};
|
|
198
246
|
}
|
|
247
|
+
/**
|
|
248
|
+
* A key's words as a person reads them, in lower case: split at anything but
|
|
249
|
+
* a letter or a digit, and where a capital begins a word, as in passwordHash
|
|
250
|
+
* and APIKey.
|
|
251
|
+
*/
|
|
252
|
+
export function words(key) {
|
|
253
|
+
const out = [];
|
|
254
|
+
const chars = [...key];
|
|
255
|
+
let start = -1;
|
|
256
|
+
const upper = (c) => c !== undefined && /\p{Lu}/u.test(c);
|
|
257
|
+
const lower = (c) => c !== undefined && /\p{Ll}/u.test(c);
|
|
258
|
+
const digit = (c) => c !== undefined && /\p{N}/u.test(c);
|
|
259
|
+
for (let i = 0; i < chars.length; i++) {
|
|
260
|
+
const c = chars[i];
|
|
261
|
+
if (!/[\p{L}\p{N}]/u.test(c)) {
|
|
262
|
+
if (start >= 0) {
|
|
263
|
+
out.push(chars.slice(start, i).join('').toLowerCase());
|
|
264
|
+
start = -1;
|
|
265
|
+
}
|
|
266
|
+
continue;
|
|
267
|
+
}
|
|
268
|
+
const before = chars[i - 1];
|
|
269
|
+
const begins = upper(c) && (lower(before) || digit(before) || (upper(before) && lower(chars[i + 1])));
|
|
270
|
+
if (start >= 0 && begins) {
|
|
271
|
+
out.push(chars.slice(start, i).join('').toLowerCase());
|
|
272
|
+
start = i;
|
|
273
|
+
}
|
|
274
|
+
if (start < 0) {
|
|
275
|
+
start = i;
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
if (start >= 0) {
|
|
279
|
+
out.push(chars.slice(start).join('').toLowerCase());
|
|
280
|
+
}
|
|
281
|
+
return out;
|
|
282
|
+
}
|
|
283
|
+
function joinsTo(keyWords, name) {
|
|
284
|
+
for (let i = 0; i < keyWords.length; i++) {
|
|
285
|
+
let rest = name;
|
|
286
|
+
for (const word of keyWords.slice(i)) {
|
|
287
|
+
if (!rest.startsWith(word)) {
|
|
288
|
+
break;
|
|
289
|
+
}
|
|
290
|
+
rest = rest.slice(word.length);
|
|
291
|
+
if (rest === '') {
|
|
292
|
+
return true;
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
return false;
|
|
297
|
+
}
|
|
298
|
+
/**
|
|
299
|
+
* JSON text with the password of each URL inside it hidden, everything else
|
|
300
|
+
* as it was spelled: "postgres://ann:hunter2@db/app" is
|
|
301
|
+
* "postgres://ann:[redacted]@db/app".
|
|
302
|
+
*/
|
|
303
|
+
export function hideUrlPasswords(text) {
|
|
304
|
+
if (!text.includes('://') && !text.includes('\\')) {
|
|
305
|
+
return text;
|
|
306
|
+
}
|
|
307
|
+
return text.replace(/"(?:[^"\\]|\\.)*"/g, raw => {
|
|
308
|
+
const value = JSON.parse(raw);
|
|
309
|
+
const hidden = hideUrlTextPasswords(value);
|
|
310
|
+
return hidden === value ? raw : JSON.stringify(hidden);
|
|
311
|
+
});
|
|
312
|
+
}
|
|
313
|
+
function hideUrlTextPasswords(text) {
|
|
314
|
+
let out = '';
|
|
315
|
+
let written = 0;
|
|
316
|
+
for (let from = 0; from < text.length;) {
|
|
317
|
+
const schemeEnd = text.indexOf('://', from);
|
|
318
|
+
if (schemeEnd < 0) {
|
|
319
|
+
break;
|
|
320
|
+
}
|
|
321
|
+
const [password, at, end] = findPassword(text, schemeEnd + 3);
|
|
322
|
+
from = end;
|
|
323
|
+
if (password < 0 || !/[A-Za-z][A-Za-z0-9+.-]*$/.test(text.slice(0, schemeEnd))) {
|
|
324
|
+
continue;
|
|
325
|
+
}
|
|
326
|
+
out += `${text.slice(written, password)}[redacted]`;
|
|
327
|
+
written = at;
|
|
328
|
+
}
|
|
329
|
+
return written === 0 ? text : out + text.slice(written);
|
|
330
|
+
}
|
|
331
|
+
/** Where an authority's password begins and the '@' after it, -1 when it has none, and where it ends. */
|
|
332
|
+
function findPassword(text, start) {
|
|
333
|
+
let at = -1;
|
|
334
|
+
let colon = -1;
|
|
335
|
+
let i = start;
|
|
336
|
+
while (i < text.length) {
|
|
337
|
+
const c = text[i];
|
|
338
|
+
if (c === '/' || c === '?' || c === '#' || c <= ' ') {
|
|
339
|
+
break;
|
|
340
|
+
}
|
|
341
|
+
if (c === '@') {
|
|
342
|
+
at = i;
|
|
343
|
+
}
|
|
344
|
+
else if (c === ':' && colon < 0) {
|
|
345
|
+
colon = i;
|
|
346
|
+
}
|
|
347
|
+
i++;
|
|
348
|
+
}
|
|
349
|
+
const end = Math.min(i, text.length);
|
|
350
|
+
return at < 0 || colon < 0 || colon + 1 >= at ? [-1, -1, end] : [colon + 1, at, end];
|
|
351
|
+
}
|
|
199
352
|
/** What a hidden value becomes, in the store and on the console. */
|
|
200
353
|
export const redacted = '"[redacted]"';
|
|
201
354
|
/**
|