@tinyshed/tinystore 0.1.0-rc.4 → 0.1.0-rc.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +37 -37
- package/bin/tinystore.js +12 -1
- package/dist/backup.d.ts +33 -0
- package/dist/backup.js +49 -0
- package/dist/blobs.d.ts +160 -79
- package/dist/blobs.js +510 -217
- package/dist/channels.d.ts +90 -0
- package/dist/channels.js +291 -0
- package/dist/clock.js +3 -3
- package/dist/config.d.ts +209 -86
- package/dist/config.js +581 -538
- package/dist/connection.d.ts +64 -2
- package/dist/connection.js +201 -16
- package/dist/console.d.ts +27 -3
- package/dist/console.js +165 -12
- package/dist/dotenv.d.ts +7 -0
- package/dist/dotenv.js +79 -0
- package/dist/errors.d.ts +61 -1
- package/dist/errors.js +81 -1
- package/dist/handles.d.ts +44 -2
- package/dist/handles.js +81 -2
- package/dist/index.d.ts +17 -11
- package/dist/index.js +5 -3
- package/dist/instruments.d.ts +177 -0
- package/dist/instruments.js +819 -0
- package/dist/jobs.d.ts +335 -239
- package/dist/jobs.js +716 -536
- package/dist/json.d.ts +2 -0
- package/dist/json.js +122 -0
- package/dist/kv.d.ts +130 -243
- package/dist/kv.js +273 -640
- package/dist/limits.d.ts +80 -0
- package/dist/limits.js +139 -0
- package/dist/locks.d.ts +62 -0
- package/dist/locks.js +237 -0
- package/dist/logger.d.ts +45 -9
- package/dist/logger.js +150 -38
- package/dist/metrics.d.ts +210 -209
- package/dist/metrics.js +319 -556
- package/dist/once.d.ts +19 -18
- package/dist/once.js +74 -63
- package/dist/query.d.ts +181 -0
- package/dist/query.js +788 -0
- package/dist/records.d.ts +241 -150
- package/dist/records.js +507 -291
- package/dist/runtime/addon.d.ts +14 -0
- package/dist/runtime/addon.js +69 -0
- package/dist/runtime/bun.js +5 -7
- package/dist/runtime/node.js +5 -7
- package/dist/runtime/pipe.d.ts +17 -0
- package/dist/runtime/pipe.js +133 -0
- package/dist/runtime/shared.d.ts +6 -0
- package/dist/runtime/shared.js +19 -0
- package/dist/runtime.d.ts +5 -2
- package/dist/search.d.ts +189 -0
- package/dist/search.js +281 -0
- package/dist/session.d.ts +31 -3
- package/dist/session.js +153 -41
- package/dist/sql.d.ts +228 -78
- package/dist/sql.js +745 -240
- package/dist/store.d.ts +244 -24
- package/dist/store.js +270 -67
- package/dist/time.d.ts +5 -1
- package/dist/time.js +22 -2
- package/dist/tx.d.ts +61 -0
- package/dist/tx.js +314 -0
- package/dist/wire/codec.d.ts +23 -7
- package/dist/wire/codec.js +56 -35
- package/dist/wire/frame.d.ts +2 -0
- package/dist/wire/frame.js +31 -14
- package/dist/wire/messages.d.ts +259 -0
- package/dist/wire/messages.js +22 -0
- package/dist/wire/msgpack.d.ts +49 -1
- package/dist/wire/msgpack.js +293 -56
- package/dist/wire/protocol.d.ts +2683 -0
- package/dist/wire/protocol.js +10846 -0
- package/package.json +39 -38
- package/src/backup.ts +81 -0
- package/src/blobs.ts +678 -301
- package/src/channels.ts +377 -0
- package/src/clock.ts +4 -4
- package/src/config.ts +902 -612
- package/src/connection.ts +279 -21
- package/src/console.ts +180 -12
- package/src/dotenv.ts +82 -0
- package/src/errors.ts +105 -0
- package/src/handles.ts +157 -8
- package/src/index.ts +158 -52
- package/src/instruments.ts +1042 -0
- package/src/jobs.ts +1017 -733
- package/src/json.ts +133 -0
- package/src/kv.ts +424 -847
- package/src/limits.ts +202 -0
- package/src/locks.ts +320 -0
- package/src/logger.ts +209 -40
- package/src/metrics.ts +557 -752
- package/src/once.ts +86 -83
- package/src/query.ts +1005 -0
- package/src/records.ts +795 -418
- package/src/runtime/addon.ts +99 -0
- package/src/runtime/bun.ts +5 -7
- package/src/runtime/node.ts +5 -7
- package/src/runtime/pipe.ts +178 -0
- package/src/runtime/shared.ts +20 -0
- package/src/runtime.ts +5 -2
- package/src/search.ts +480 -0
- package/src/session.ts +187 -41
- package/src/sql.ts +993 -306
- package/src/store.ts +473 -78
- package/src/time.ts +48 -3
- package/src/tx.ts +426 -0
- package/src/wire/codec.ts +78 -37
- package/src/wire/frame.ts +44 -14
- package/src/wire/messages.ts +25 -0
- package/src/wire/msgpack.ts +327 -56
- package/src/wire/protocol.ts +11709 -0
- package/dist/limiter.d.ts +0 -45
- package/dist/limiter.js +0 -68
- package/dist/quota.d.ts +0 -44
- package/dist/quota.js +0 -98
- package/src/limiter.ts +0 -99
- package/src/quota.ts +0 -137
package/dist/logger.js
CHANGED
|
@@ -4,16 +4,24 @@
|
|
|
4
4
|
// Lines wait in a bounded buffer and go every second, or once half of it
|
|
5
5
|
// waits; what does not fit, or a write that fails, is dropped and counted,
|
|
6
6
|
// and said in TinyStore's own lines, since no call is waiting to be told.
|
|
7
|
+
import { readFileSync } from 'node:fs';
|
|
8
|
+
import { fileURLToPath } from 'node:url';
|
|
7
9
|
import { DropNotice, FailureLog } from './background.js';
|
|
8
|
-
import {
|
|
10
|
+
import { env as environmentOf, live, oneOf } from './config.js';
|
|
11
|
+
import { hideUrlPasswords, jsonLine, jsonOf, prettyLine, redacted, redactor, sourceKey, } from './console.js';
|
|
12
|
+
import { parseDotenv } from './dotenv.js';
|
|
9
13
|
import { InvalidError } from './errors.js';
|
|
10
14
|
import { currentTrace } from './trace.js';
|
|
11
|
-
|
|
15
|
+
// slog's numbers, trace below debug as Python's levels map, so that a logger
|
|
16
|
+
// at trace takes every line
|
|
17
|
+
const levels = { trace: -8, debug: -4, info: 0, warn: 4, error: 8 };
|
|
18
|
+
const formats = ['pretty', 'json', 'off'];
|
|
19
|
+
const times = ['clock', 'full', 'off'];
|
|
12
20
|
const flushEvery = 1000;
|
|
13
21
|
/**
|
|
14
22
|
* A logger of the console alone, for a program that wants the logger and not
|
|
15
23
|
* the records: its lines are written as a store's logger writes them, and
|
|
16
|
-
* `store.
|
|
24
|
+
* `store.logger(stream)` in its place keeps them too.
|
|
17
25
|
*
|
|
18
26
|
* ```ts
|
|
19
27
|
* const log = logger('app', { redact: ['password'] })
|
|
@@ -25,6 +33,43 @@ export function logger(stream, options) {
|
|
|
25
33
|
}
|
|
26
34
|
/** A logger of one stream, its lines appended through append when it is a store's. */
|
|
27
35
|
export function newLogger(append, stream, options = {}, out) {
|
|
36
|
+
return new Logger(sharedOf(append, stream, options, out).shared, []);
|
|
37
|
+
}
|
|
38
|
+
/** A store's logger of one stream, and what it declares its config `log` with. */
|
|
39
|
+
export function newStoreLogger(append, stream, options = {}) {
|
|
40
|
+
const { shared, spelled } = sharedOf(append, stream, options, undefined);
|
|
41
|
+
const level = options.level === undefined ? 'trace' : nameOf(levelOf(options.level));
|
|
42
|
+
const format = options.console ?? (prettyUntold(options, undefined) ? 'pretty' : 'json');
|
|
43
|
+
const time = options.time ?? 'clock';
|
|
44
|
+
const others = (all, one) => all.filter(name => name !== one);
|
|
45
|
+
const declaration = {
|
|
46
|
+
level: live(oneOf(level, others(Object.keys(levels), level))),
|
|
47
|
+
format: oneOf(format, others(formats, format)),
|
|
48
|
+
time: oneOf(time, others(times, time)),
|
|
49
|
+
};
|
|
50
|
+
const env = options.env ?? environmentOf('');
|
|
51
|
+
const take = (name) => {
|
|
52
|
+
if (Object.hasOwn(levels, name)) {
|
|
53
|
+
shared.least = levels[name];
|
|
54
|
+
}
|
|
55
|
+
};
|
|
56
|
+
if (env === false) {
|
|
57
|
+
return [new Logger(shared, []), { layers: [declaration], take }];
|
|
58
|
+
}
|
|
59
|
+
const prefix = env.prefix === '' ? 'LOG' : `${env.prefix}_LOG`;
|
|
60
|
+
const layers = [declaration, environmentOf(prefix, { from: spelled })];
|
|
61
|
+
return [new Logger(shared, []), { layers, take }];
|
|
62
|
+
}
|
|
63
|
+
/** The least named level at or above a level the code gave as a number. */
|
|
64
|
+
function nameOf(least) {
|
|
65
|
+
const found = Object.entries(levels).find(([, number]) => number >= least);
|
|
66
|
+
return found?.[0] ?? 'error';
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* What a logger shares with the loggers its `with` makes, and the variables
|
|
70
|
+
* it read as the config `log` spells them.
|
|
71
|
+
*/
|
|
72
|
+
function sharedOf(append, stream, options, out) {
|
|
28
73
|
if (stream === '') {
|
|
29
74
|
throw new InvalidError('a logger of no stream');
|
|
30
75
|
}
|
|
@@ -32,13 +77,15 @@ export function newLogger(append, stream, options = {}, out) {
|
|
|
32
77
|
if (!Number.isInteger(most) || most < 1) {
|
|
33
78
|
throw new InvalidError(`a logger's buffer of ${most} lines`);
|
|
34
79
|
}
|
|
35
|
-
const [read, ignored] = fromEnvironment(options);
|
|
80
|
+
const [read, ignored, spelled] = fromEnvironment(options);
|
|
36
81
|
const shared = {
|
|
37
82
|
append,
|
|
38
83
|
stream,
|
|
39
84
|
most,
|
|
40
85
|
least: read.level === undefined ? Number.NEGATIVE_INFINITY : levelOf(read.level),
|
|
41
|
-
|
|
86
|
+
redact: redactor(read.redact, read.keepUrlPasswords),
|
|
87
|
+
replace: read.replace,
|
|
88
|
+
source: read.source === true,
|
|
42
89
|
print: printer(read, out),
|
|
43
90
|
waiting: [],
|
|
44
91
|
timer: undefined,
|
|
@@ -48,47 +95,75 @@ export function newLogger(append, stream, options = {}, out) {
|
|
|
48
95
|
drops: append === undefined ? undefined : new DropNotice(stream, most, sayOwn),
|
|
49
96
|
};
|
|
50
97
|
sayIgnored(shared.print, ignored);
|
|
51
|
-
return
|
|
98
|
+
return { shared, spelled };
|
|
52
99
|
}
|
|
53
|
-
/**
|
|
100
|
+
/**
|
|
101
|
+
* LOG_LEVEL, LOG_FORMAT and LOG_TIME over the options, after the prefix env
|
|
102
|
+
* names, as Go and Python read them; a console turned off stays off. Each
|
|
103
|
+
* variable read is spelled as the config `log` reads it, by its name.
|
|
104
|
+
*/
|
|
54
105
|
function fromEnvironment(options) {
|
|
55
106
|
const read = { ...options };
|
|
56
107
|
const ignored = [];
|
|
57
|
-
const
|
|
108
|
+
const spelled = {};
|
|
109
|
+
const env = options.env ?? environmentOf('');
|
|
110
|
+
if (env === false) {
|
|
111
|
+
return [read, ignored, spelled];
|
|
112
|
+
}
|
|
113
|
+
const variables = environment(env);
|
|
114
|
+
const name = (variable) => (env.prefix === '' ? variable : `${env.prefix}_${variable}`);
|
|
115
|
+
const level = variables(name('LOG_LEVEL'));
|
|
58
116
|
if (level !== undefined) {
|
|
59
|
-
const
|
|
60
|
-
if (Object.hasOwn(levels,
|
|
61
|
-
read.level =
|
|
117
|
+
const known = level.toLowerCase() === 'warning' ? 'warn' : level.toLowerCase();
|
|
118
|
+
if (Object.hasOwn(levels, known)) {
|
|
119
|
+
read.level = levels[known];
|
|
120
|
+
spelled[name('LOG_LEVEL')] = known;
|
|
62
121
|
}
|
|
63
122
|
else {
|
|
64
|
-
ignored.push(['LOG_LEVEL', level, 'debug, info, warn or error']);
|
|
123
|
+
ignored.push([name('LOG_LEVEL'), level, 'trace, debug, info, warn or error']);
|
|
65
124
|
}
|
|
66
125
|
}
|
|
67
|
-
const format =
|
|
126
|
+
const format = variables(name('LOG_FORMAT'));
|
|
68
127
|
if (format !== undefined) {
|
|
69
|
-
const
|
|
70
|
-
if (
|
|
71
|
-
ignored.push(['LOG_FORMAT', format, 'pretty, json or off']);
|
|
128
|
+
const known = format.toLowerCase();
|
|
129
|
+
if (known !== 'pretty' && known !== 'json' && known !== 'off') {
|
|
130
|
+
ignored.push([name('LOG_FORMAT'), format, 'pretty, json or off']);
|
|
72
131
|
}
|
|
73
132
|
else if (read.console !== 'off') {
|
|
74
|
-
read.console =
|
|
133
|
+
read.console = known;
|
|
134
|
+
spelled[name('LOG_FORMAT')] = known;
|
|
75
135
|
}
|
|
76
136
|
}
|
|
77
|
-
const time =
|
|
137
|
+
const time = variables(name('LOG_TIME'));
|
|
78
138
|
if (time !== undefined) {
|
|
79
|
-
const
|
|
80
|
-
if (
|
|
81
|
-
read.time =
|
|
139
|
+
const known = time.toLowerCase();
|
|
140
|
+
if (known === 'clock' || known === 'full' || known === 'off') {
|
|
141
|
+
read.time = known;
|
|
142
|
+
spelled[name('LOG_TIME')] = known;
|
|
82
143
|
}
|
|
83
144
|
else {
|
|
84
|
-
ignored.push(['LOG_TIME', time, 'clock, full or off']);
|
|
145
|
+
ignored.push([name('LOG_TIME'), time, 'clock, full or off']);
|
|
85
146
|
}
|
|
86
147
|
}
|
|
87
|
-
return [read, ignored];
|
|
148
|
+
return [read, ignored, spelled];
|
|
88
149
|
}
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
150
|
+
/** The variables of env's files, the process's or `from`'s over them, a missing file skipped; an empty one is none. */
|
|
151
|
+
function environment(env) {
|
|
152
|
+
const fromFiles = {};
|
|
153
|
+
for (const file of env.files) {
|
|
154
|
+
let text;
|
|
155
|
+
try {
|
|
156
|
+
text = readFileSync(file, 'utf8');
|
|
157
|
+
}
|
|
158
|
+
catch {
|
|
159
|
+
continue;
|
|
160
|
+
}
|
|
161
|
+
Object.assign(fromFiles, parseDotenv(text));
|
|
162
|
+
}
|
|
163
|
+
return name => {
|
|
164
|
+
const text = ((env.from ?? process.env)[name] ?? fromFiles[name])?.trim();
|
|
165
|
+
return text === '' ? undefined : text;
|
|
166
|
+
};
|
|
92
167
|
}
|
|
93
168
|
/** The values said to be ignored, so that a program making many loggers says each once. */
|
|
94
169
|
const saidIgnored = new Set();
|
|
@@ -118,16 +193,21 @@ function forceColor() {
|
|
|
118
193
|
const force = process.env.FORCE_COLOR;
|
|
119
194
|
return force !== undefined && force !== '' && force !== '0';
|
|
120
195
|
}
|
|
196
|
+
/** Whether a console told no format prints pretty lines: on a terminal, or where FORCE_COLOR asks. */
|
|
197
|
+
function prettyUntold(options, out) {
|
|
198
|
+
const stream = out ?? options.to ?? process.stderr;
|
|
199
|
+
// FORCE_COLOR other than 0 says a person reads a pipe, as an IDE's run console is one
|
|
200
|
+
return stream.isTTY === true || (out === undefined && forceColor());
|
|
201
|
+
}
|
|
121
202
|
/** How a logger prints its lines, or undefined when it prints none. */
|
|
122
203
|
function printer(options, out) {
|
|
123
204
|
if (options.console === 'off') {
|
|
124
205
|
return undefined;
|
|
125
206
|
}
|
|
126
207
|
const stream = out ?? options.to ?? process.stderr;
|
|
127
|
-
// FORCE_COLOR other than 0 says a person reads a pipe, as an IDE's run console is one
|
|
128
208
|
const forced = out === undefined && forceColor();
|
|
129
209
|
const terminal = stream.isTTY === true;
|
|
130
|
-
const pretty = options.console === 'pretty' || (options.console === undefined && (
|
|
210
|
+
const pretty = options.console === 'pretty' || (options.console === undefined && prettyUntold(options, out));
|
|
131
211
|
if (!pretty) {
|
|
132
212
|
return line => void stream.write(jsonLine(line));
|
|
133
213
|
}
|
|
@@ -149,7 +229,7 @@ function printer(options, out) {
|
|
|
149
229
|
* its context, who is speaking.
|
|
150
230
|
*
|
|
151
231
|
* ```ts
|
|
152
|
-
* const log = store.
|
|
232
|
+
* const log = store.logger('api')
|
|
153
233
|
* log.info('server started', { port: 3000 })
|
|
154
234
|
* log.with({ requestId }).warn('slow request', { ms: 1200 })
|
|
155
235
|
* log.event('user.created', { userId: 42 }) // a record named for the event
|
|
@@ -159,7 +239,7 @@ export class Logger {
|
|
|
159
239
|
#shared;
|
|
160
240
|
/** key, JSON, key, JSON…, hidden as the logger's redact says */
|
|
161
241
|
#context;
|
|
162
|
-
/** Loggers come from `logger(stream)`, `store.
|
|
242
|
+
/** Loggers come from `logger(stream)`, `store.logger(stream)` and their `with`. */
|
|
163
243
|
constructor(shared, context) {
|
|
164
244
|
this.#shared = shared;
|
|
165
245
|
this.#context = context;
|
|
@@ -194,7 +274,7 @@ export class Logger {
|
|
|
194
274
|
with(context) {
|
|
195
275
|
return new Logger(this.#shared, [
|
|
196
276
|
...this.#context,
|
|
197
|
-
...encodeFields(context, this.#shared.
|
|
277
|
+
...encodeFields(context, this.#shared.redact, this.#shared.replace),
|
|
198
278
|
]);
|
|
199
279
|
}
|
|
200
280
|
/** Hands what the buffer holds to the server now, as the timer does every second. */
|
|
@@ -228,12 +308,14 @@ export class Logger {
|
|
|
228
308
|
}
|
|
229
309
|
#emit(what, attrs) {
|
|
230
310
|
const trace = traceOf(currentTrace());
|
|
311
|
+
const shared = this.#shared;
|
|
312
|
+
const encoded = attrs === undefined ? [] : encodeFields(attrs, shared.redact, shared.replace);
|
|
231
313
|
const line = {
|
|
232
314
|
at: new Date(),
|
|
233
|
-
stream:
|
|
315
|
+
stream: shared.stream,
|
|
234
316
|
...what,
|
|
235
317
|
context: this.#context,
|
|
236
|
-
attrs:
|
|
318
|
+
attrs: shared.source ? [...encoded, sourceKey, sourceOfCaller()] : encoded,
|
|
237
319
|
...trace,
|
|
238
320
|
};
|
|
239
321
|
this.#shared.print?.(line);
|
|
@@ -281,17 +363,23 @@ async function writeWaiting(shared, append) {
|
|
|
281
363
|
}
|
|
282
364
|
/**
|
|
283
365
|
* Fields as key, JSON pairs, as a console writes them and the store keeps
|
|
284
|
-
* them: an Error's value its stack, which JSON would write as {},
|
|
285
|
-
*
|
|
366
|
+
* them: an Error's value its stack, which JSON would write as {}, a secret's
|
|
367
|
+
* value hidden at any depth, and a URL's password unless kept.
|
|
286
368
|
*/
|
|
287
|
-
export function encodeFields(fields,
|
|
369
|
+
export function encodeFields(fields, redact, replace) {
|
|
288
370
|
const pairs = Array.isArray(fields)
|
|
289
371
|
? fields
|
|
290
372
|
: Object.entries(fields);
|
|
291
373
|
const flat = [];
|
|
292
|
-
for (const [key,
|
|
374
|
+
for (const [key, given] of pairs) {
|
|
375
|
+
const value = replace === undefined ? given : replace(key, given);
|
|
293
376
|
const shown = value instanceof Error ? (value.stack ?? String(value)) : value;
|
|
294
|
-
|
|
377
|
+
if (redact?.hides(key)) {
|
|
378
|
+
flat.push(key, redacted);
|
|
379
|
+
continue;
|
|
380
|
+
}
|
|
381
|
+
const json = jsonOf(shown, redact?.hides);
|
|
382
|
+
flat.push(key, redact?.urls === true ? hideUrlPasswords(json) : json);
|
|
295
383
|
}
|
|
296
384
|
return flat;
|
|
297
385
|
}
|
|
@@ -315,3 +403,27 @@ function traceOf(trace) {
|
|
|
315
403
|
}
|
|
316
404
|
return spanId !== undefined && /^[0-9a-f]{16}$/.test(spanId) ? { traceId, spanId } : { traceId };
|
|
317
405
|
}
|
|
406
|
+
/** This file, whose frames a line's source skips. */
|
|
407
|
+
const here = pathOf(import.meta.url);
|
|
408
|
+
/**
|
|
409
|
+
* Where a line was logged, as slog's handlers spell a source: the first frame
|
|
410
|
+
* of the stack outside this file.
|
|
411
|
+
*/
|
|
412
|
+
function sourceOfCaller() {
|
|
413
|
+
for (const frame of (new Error().stack ?? '').split('\n').slice(1)) {
|
|
414
|
+
const found = /at (?:(.+?) \()?(.+):(\d+):\d+\)?$/.exec(frame.trim());
|
|
415
|
+
if (found === null || pathOf(found[2]) === here) {
|
|
416
|
+
continue;
|
|
417
|
+
}
|
|
418
|
+
return JSON.stringify({
|
|
419
|
+
function: found[1] ?? '',
|
|
420
|
+
file: pathOf(found[2]),
|
|
421
|
+
line: Number(found[3]),
|
|
422
|
+
});
|
|
423
|
+
}
|
|
424
|
+
return '{}';
|
|
425
|
+
}
|
|
426
|
+
/** A frame's file as a path, which Node writes as a file: URL. */
|
|
427
|
+
function pathOf(file) {
|
|
428
|
+
return file.startsWith('file://') ? fileURLToPath(file) : file;
|
|
429
|
+
}
|