@tinyshed/tinystore 0.1.0-rc.5 → 0.1.0-rc.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (122) hide show
  1. package/README.md +37 -37
  2. package/bin/tinystore.js +12 -1
  3. package/dist/backup.d.ts +33 -0
  4. package/dist/backup.js +49 -0
  5. package/dist/blobs.d.ts +160 -79
  6. package/dist/blobs.js +510 -217
  7. package/dist/channels.d.ts +90 -0
  8. package/dist/channels.js +291 -0
  9. package/dist/clock.js +3 -3
  10. package/dist/config.d.ts +209 -86
  11. package/dist/config.js +581 -538
  12. package/dist/connection.d.ts +64 -2
  13. package/dist/connection.js +201 -16
  14. package/dist/console.d.ts +27 -3
  15. package/dist/console.js +165 -12
  16. package/dist/dotenv.d.ts +7 -0
  17. package/dist/dotenv.js +79 -0
  18. package/dist/errors.d.ts +61 -1
  19. package/dist/errors.js +81 -1
  20. package/dist/handles.d.ts +44 -2
  21. package/dist/handles.js +81 -2
  22. package/dist/index.d.ts +17 -11
  23. package/dist/index.js +5 -3
  24. package/dist/instruments.d.ts +177 -0
  25. package/dist/instruments.js +819 -0
  26. package/dist/jobs.d.ts +335 -239
  27. package/dist/jobs.js +716 -536
  28. package/dist/json.d.ts +2 -0
  29. package/dist/json.js +122 -0
  30. package/dist/kv.d.ts +130 -243
  31. package/dist/kv.js +273 -640
  32. package/dist/limits.d.ts +80 -0
  33. package/dist/limits.js +139 -0
  34. package/dist/locks.d.ts +62 -0
  35. package/dist/locks.js +237 -0
  36. package/dist/logger.d.ts +45 -9
  37. package/dist/logger.js +150 -38
  38. package/dist/metrics.d.ts +210 -209
  39. package/dist/metrics.js +319 -556
  40. package/dist/once.d.ts +19 -18
  41. package/dist/once.js +74 -63
  42. package/dist/query.d.ts +181 -0
  43. package/dist/query.js +788 -0
  44. package/dist/records.d.ts +241 -150
  45. package/dist/records.js +507 -291
  46. package/dist/runtime/addon.d.ts +14 -0
  47. package/dist/runtime/addon.js +69 -0
  48. package/dist/runtime/bun.js +5 -7
  49. package/dist/runtime/node.js +5 -7
  50. package/dist/runtime/pipe.d.ts +17 -0
  51. package/dist/runtime/pipe.js +133 -0
  52. package/dist/runtime/shared.d.ts +6 -0
  53. package/dist/runtime/shared.js +19 -0
  54. package/dist/runtime.d.ts +5 -2
  55. package/dist/search.d.ts +189 -0
  56. package/dist/search.js +281 -0
  57. package/dist/session.d.ts +31 -3
  58. package/dist/session.js +153 -41
  59. package/dist/sql.d.ts +228 -78
  60. package/dist/sql.js +745 -240
  61. package/dist/store.d.ts +244 -24
  62. package/dist/store.js +270 -67
  63. package/dist/time.d.ts +5 -1
  64. package/dist/time.js +22 -2
  65. package/dist/tx.d.ts +61 -0
  66. package/dist/tx.js +314 -0
  67. package/dist/wire/codec.d.ts +23 -7
  68. package/dist/wire/codec.js +56 -35
  69. package/dist/wire/frame.d.ts +2 -0
  70. package/dist/wire/frame.js +31 -14
  71. package/dist/wire/messages.d.ts +259 -0
  72. package/dist/wire/messages.js +22 -0
  73. package/dist/wire/msgpack.d.ts +49 -1
  74. package/dist/wire/msgpack.js +293 -56
  75. package/dist/wire/protocol.d.ts +2683 -0
  76. package/dist/wire/protocol.js +10846 -0
  77. package/package.json +39 -38
  78. package/src/backup.ts +81 -0
  79. package/src/blobs.ts +678 -301
  80. package/src/channels.ts +377 -0
  81. package/src/clock.ts +4 -4
  82. package/src/config.ts +902 -612
  83. package/src/connection.ts +279 -21
  84. package/src/console.ts +180 -12
  85. package/src/dotenv.ts +82 -0
  86. package/src/errors.ts +105 -0
  87. package/src/handles.ts +157 -8
  88. package/src/index.ts +158 -52
  89. package/src/instruments.ts +1042 -0
  90. package/src/jobs.ts +1017 -733
  91. package/src/json.ts +133 -0
  92. package/src/kv.ts +424 -847
  93. package/src/limits.ts +202 -0
  94. package/src/locks.ts +320 -0
  95. package/src/logger.ts +209 -40
  96. package/src/metrics.ts +557 -752
  97. package/src/once.ts +86 -83
  98. package/src/query.ts +1005 -0
  99. package/src/records.ts +795 -418
  100. package/src/runtime/addon.ts +99 -0
  101. package/src/runtime/bun.ts +5 -7
  102. package/src/runtime/node.ts +5 -7
  103. package/src/runtime/pipe.ts +178 -0
  104. package/src/runtime/shared.ts +20 -0
  105. package/src/runtime.ts +5 -2
  106. package/src/search.ts +480 -0
  107. package/src/session.ts +187 -41
  108. package/src/sql.ts +993 -306
  109. package/src/store.ts +473 -78
  110. package/src/time.ts +48 -3
  111. package/src/tx.ts +426 -0
  112. package/src/wire/codec.ts +78 -37
  113. package/src/wire/frame.ts +44 -14
  114. package/src/wire/messages.ts +25 -0
  115. package/src/wire/msgpack.ts +327 -56
  116. package/src/wire/protocol.ts +11709 -0
  117. package/dist/limiter.d.ts +0 -45
  118. package/dist/limiter.js +0 -68
  119. package/dist/quota.d.ts +0 -44
  120. package/dist/quota.js +0 -98
  121. package/src/limiter.ts +0 -99
  122. package/src/quota.ts +0 -137
package/dist/logger.js CHANGED
@@ -4,16 +4,24 @@
4
4
  // Lines wait in a bounded buffer and go every second, or once half of it
5
5
  // waits; what does not fit, or a write that fails, is dropped and counted,
6
6
  // and said in TinyStore's own lines, since no call is waiting to be told.
7
+ import { readFileSync } from 'node:fs';
8
+ import { fileURLToPath } from 'node:url';
7
9
  import { DropNotice, FailureLog } from './background.js';
8
- import { jsonLine, jsonOf, prettyLine, redacted, redactor, } from './console.js';
10
+ import { env as environmentOf, live, oneOf } from './config.js';
11
+ import { hideUrlPasswords, jsonLine, jsonOf, prettyLine, redacted, redactor, sourceKey, } from './console.js';
12
+ import { parseDotenv } from './dotenv.js';
9
13
  import { InvalidError } from './errors.js';
10
14
  import { currentTrace } from './trace.js';
11
- const levels = { debug: -4, info: 0, warn: 4, error: 8 };
15
+ // slog's numbers, trace below debug as Python's levels map, so that a logger
16
+ // at trace takes every line
17
+ const levels = { trace: -8, debug: -4, info: 0, warn: 4, error: 8 };
18
+ const formats = ['pretty', 'json', 'off'];
19
+ const times = ['clock', 'full', 'off'];
12
20
  const flushEvery = 1000;
13
21
  /**
14
22
  * A logger of the console alone, for a program that wants the logger and not
15
23
  * the records: its lines are written as a store's logger writes them, and
16
- * `store.records.logger(stream)` in its place keeps them too.
24
+ * `store.logger(stream)` in its place keeps them too.
17
25
  *
18
26
  * ```ts
19
27
  * const log = logger('app', { redact: ['password'] })
@@ -25,6 +33,43 @@ export function logger(stream, options) {
25
33
  }
26
34
  /** A logger of one stream, its lines appended through append when it is a store's. */
27
35
  export function newLogger(append, stream, options = {}, out) {
36
+ return new Logger(sharedOf(append, stream, options, out).shared, []);
37
+ }
38
+ /** A store's logger of one stream, and what it declares its config `log` with. */
39
+ export function newStoreLogger(append, stream, options = {}) {
40
+ const { shared, spelled } = sharedOf(append, stream, options, undefined);
41
+ const level = options.level === undefined ? 'trace' : nameOf(levelOf(options.level));
42
+ const format = options.console ?? (prettyUntold(options, undefined) ? 'pretty' : 'json');
43
+ const time = options.time ?? 'clock';
44
+ const others = (all, one) => all.filter(name => name !== one);
45
+ const declaration = {
46
+ level: live(oneOf(level, others(Object.keys(levels), level))),
47
+ format: oneOf(format, others(formats, format)),
48
+ time: oneOf(time, others(times, time)),
49
+ };
50
+ const env = options.env ?? environmentOf('');
51
+ const take = (name) => {
52
+ if (Object.hasOwn(levels, name)) {
53
+ shared.least = levels[name];
54
+ }
55
+ };
56
+ if (env === false) {
57
+ return [new Logger(shared, []), { layers: [declaration], take }];
58
+ }
59
+ const prefix = env.prefix === '' ? 'LOG' : `${env.prefix}_LOG`;
60
+ const layers = [declaration, environmentOf(prefix, { from: spelled })];
61
+ return [new Logger(shared, []), { layers, take }];
62
+ }
63
+ /** The least named level at or above a level the code gave as a number. */
64
+ function nameOf(least) {
65
+ const found = Object.entries(levels).find(([, number]) => number >= least);
66
+ return found?.[0] ?? 'error';
67
+ }
68
+ /**
69
+ * What a logger shares with the loggers its `with` makes, and the variables
70
+ * it read as the config `log` spells them.
71
+ */
72
+ function sharedOf(append, stream, options, out) {
28
73
  if (stream === '') {
29
74
  throw new InvalidError('a logger of no stream');
30
75
  }
@@ -32,13 +77,15 @@ export function newLogger(append, stream, options = {}, out) {
32
77
  if (!Number.isInteger(most) || most < 1) {
33
78
  throw new InvalidError(`a logger's buffer of ${most} lines`);
34
79
  }
35
- const [read, ignored] = fromEnvironment(options);
80
+ const [read, ignored, spelled] = fromEnvironment(options);
36
81
  const shared = {
37
82
  append,
38
83
  stream,
39
84
  most,
40
85
  least: read.level === undefined ? Number.NEGATIVE_INFINITY : levelOf(read.level),
41
- hides: redactor(read.redact),
86
+ redact: redactor(read.redact, read.keepUrlPasswords),
87
+ replace: read.replace,
88
+ source: read.source === true,
42
89
  print: printer(read, out),
43
90
  waiting: [],
44
91
  timer: undefined,
@@ -48,47 +95,75 @@ export function newLogger(append, stream, options = {}, out) {
48
95
  drops: append === undefined ? undefined : new DropNotice(stream, most, sayOwn),
49
96
  };
50
97
  sayIgnored(shared.print, ignored);
51
- return new Logger(shared, []);
98
+ return { shared, spelled };
52
99
  }
53
- /** LOG_LEVEL, LOG_FORMAT and LOG_TIME over the options, as Go and Python read them; a console turned off stays off. */
100
+ /**
101
+ * LOG_LEVEL, LOG_FORMAT and LOG_TIME over the options, after the prefix env
102
+ * names, as Go and Python read them; a console turned off stays off. Each
103
+ * variable read is spelled as the config `log` reads it, by its name.
104
+ */
54
105
  function fromEnvironment(options) {
55
106
  const read = { ...options };
56
107
  const ignored = [];
57
- const level = variable('LOG_LEVEL');
108
+ const spelled = {};
109
+ const env = options.env ?? environmentOf('');
110
+ if (env === false) {
111
+ return [read, ignored, spelled];
112
+ }
113
+ const variables = environment(env);
114
+ const name = (variable) => (env.prefix === '' ? variable : `${env.prefix}_${variable}`);
115
+ const level = variables(name('LOG_LEVEL'));
58
116
  if (level !== undefined) {
59
- const name = level.toLowerCase() === 'warning' ? 'warn' : level.toLowerCase();
60
- if (Object.hasOwn(levels, name)) {
61
- read.level = name;
117
+ const known = level.toLowerCase() === 'warning' ? 'warn' : level.toLowerCase();
118
+ if (Object.hasOwn(levels, known)) {
119
+ read.level = levels[known];
120
+ spelled[name('LOG_LEVEL')] = known;
62
121
  }
63
122
  else {
64
- ignored.push(['LOG_LEVEL', level, 'debug, info, warn or error']);
123
+ ignored.push([name('LOG_LEVEL'), level, 'trace, debug, info, warn or error']);
65
124
  }
66
125
  }
67
- const format = variable('LOG_FORMAT');
126
+ const format = variables(name('LOG_FORMAT'));
68
127
  if (format !== undefined) {
69
- const name = format.toLowerCase();
70
- if (name !== 'pretty' && name !== 'json' && name !== 'off') {
71
- ignored.push(['LOG_FORMAT', format, 'pretty, json or off']);
128
+ const known = format.toLowerCase();
129
+ if (known !== 'pretty' && known !== 'json' && known !== 'off') {
130
+ ignored.push([name('LOG_FORMAT'), format, 'pretty, json or off']);
72
131
  }
73
132
  else if (read.console !== 'off') {
74
- read.console = name;
133
+ read.console = known;
134
+ spelled[name('LOG_FORMAT')] = known;
75
135
  }
76
136
  }
77
- const time = variable('LOG_TIME');
137
+ const time = variables(name('LOG_TIME'));
78
138
  if (time !== undefined) {
79
- const name = time.toLowerCase();
80
- if (name === 'clock' || name === 'full' || name === 'off') {
81
- read.time = name;
139
+ const known = time.toLowerCase();
140
+ if (known === 'clock' || known === 'full' || known === 'off') {
141
+ read.time = known;
142
+ spelled[name('LOG_TIME')] = known;
82
143
  }
83
144
  else {
84
- ignored.push(['LOG_TIME', time, 'clock, full or off']);
145
+ ignored.push([name('LOG_TIME'), time, 'clock, full or off']);
85
146
  }
86
147
  }
87
- return [read, ignored];
148
+ return [read, ignored, spelled];
88
149
  }
89
- function variable(name) {
90
- const text = process.env[name]?.trim();
91
- return text === '' ? undefined : text;
150
+ /** The variables of env's files, the process's or `from`'s over them, a missing file skipped; an empty one is none. */
151
+ function environment(env) {
152
+ const fromFiles = {};
153
+ for (const file of env.files) {
154
+ let text;
155
+ try {
156
+ text = readFileSync(file, 'utf8');
157
+ }
158
+ catch {
159
+ continue;
160
+ }
161
+ Object.assign(fromFiles, parseDotenv(text));
162
+ }
163
+ return name => {
164
+ const text = ((env.from ?? process.env)[name] ?? fromFiles[name])?.trim();
165
+ return text === '' ? undefined : text;
166
+ };
92
167
  }
93
168
  /** The values said to be ignored, so that a program making many loggers says each once. */
94
169
  const saidIgnored = new Set();
@@ -118,16 +193,21 @@ function forceColor() {
118
193
  const force = process.env.FORCE_COLOR;
119
194
  return force !== undefined && force !== '' && force !== '0';
120
195
  }
196
+ /** Whether a console told no format prints pretty lines: on a terminal, or where FORCE_COLOR asks. */
197
+ function prettyUntold(options, out) {
198
+ const stream = out ?? options.to ?? process.stderr;
199
+ // FORCE_COLOR other than 0 says a person reads a pipe, as an IDE's run console is one
200
+ return stream.isTTY === true || (out === undefined && forceColor());
201
+ }
121
202
  /** How a logger prints its lines, or undefined when it prints none. */
122
203
  function printer(options, out) {
123
204
  if (options.console === 'off') {
124
205
  return undefined;
125
206
  }
126
207
  const stream = out ?? options.to ?? process.stderr;
127
- // FORCE_COLOR other than 0 says a person reads a pipe, as an IDE's run console is one
128
208
  const forced = out === undefined && forceColor();
129
209
  const terminal = stream.isTTY === true;
130
- const pretty = options.console === 'pretty' || (options.console === undefined && (terminal || forced));
210
+ const pretty = options.console === 'pretty' || (options.console === undefined && prettyUntold(options, out));
131
211
  if (!pretty) {
132
212
  return line => void stream.write(jsonLine(line));
133
213
  }
@@ -149,7 +229,7 @@ function printer(options, out) {
149
229
  * its context, who is speaking.
150
230
  *
151
231
  * ```ts
152
- * const log = store.records.logger('api')
232
+ * const log = store.logger('api')
153
233
  * log.info('server started', { port: 3000 })
154
234
  * log.with({ requestId }).warn('slow request', { ms: 1200 })
155
235
  * log.event('user.created', { userId: 42 }) // a record named for the event
@@ -159,7 +239,7 @@ export class Logger {
159
239
  #shared;
160
240
  /** key, JSON, key, JSON…, hidden as the logger's redact says */
161
241
  #context;
162
- /** Loggers come from `logger(stream)`, `store.records.logger(stream)` and their `with`. */
242
+ /** Loggers come from `logger(stream)`, `store.logger(stream)` and their `with`. */
163
243
  constructor(shared, context) {
164
244
  this.#shared = shared;
165
245
  this.#context = context;
@@ -194,7 +274,7 @@ export class Logger {
194
274
  with(context) {
195
275
  return new Logger(this.#shared, [
196
276
  ...this.#context,
197
- ...encodeFields(context, this.#shared.hides),
277
+ ...encodeFields(context, this.#shared.redact, this.#shared.replace),
198
278
  ]);
199
279
  }
200
280
  /** Hands what the buffer holds to the server now, as the timer does every second. */
@@ -228,12 +308,14 @@ export class Logger {
228
308
  }
229
309
  #emit(what, attrs) {
230
310
  const trace = traceOf(currentTrace());
311
+ const shared = this.#shared;
312
+ const encoded = attrs === undefined ? [] : encodeFields(attrs, shared.redact, shared.replace);
231
313
  const line = {
232
314
  at: new Date(),
233
- stream: this.#shared.stream,
315
+ stream: shared.stream,
234
316
  ...what,
235
317
  context: this.#context,
236
- attrs: attrs === undefined ? [] : encodeFields(attrs, this.#shared.hides),
318
+ attrs: shared.source ? [...encoded, sourceKey, sourceOfCaller()] : encoded,
237
319
  ...trace,
238
320
  };
239
321
  this.#shared.print?.(line);
@@ -281,17 +363,23 @@ async function writeWaiting(shared, append) {
281
363
  }
282
364
  /**
283
365
  * Fields as key, JSON pairs, as a console writes them and the store keeps
284
- * them: an Error's value its stack, which JSON would write as {}, and a field
285
- * hides names hidden at any depth.
366
+ * them: an Error's value its stack, which JSON would write as {}, a secret's
367
+ * value hidden at any depth, and a URL's password unless kept.
286
368
  */
287
- export function encodeFields(fields, hides) {
369
+ export function encodeFields(fields, redact, replace) {
288
370
  const pairs = Array.isArray(fields)
289
371
  ? fields
290
372
  : Object.entries(fields);
291
373
  const flat = [];
292
- for (const [key, value] of pairs) {
374
+ for (const [key, given] of pairs) {
375
+ const value = replace === undefined ? given : replace(key, given);
293
376
  const shown = value instanceof Error ? (value.stack ?? String(value)) : value;
294
- flat.push(key, hides?.(key) ? redacted : jsonOf(shown, hides));
377
+ if (redact?.hides(key)) {
378
+ flat.push(key, redacted);
379
+ continue;
380
+ }
381
+ const json = jsonOf(shown, redact?.hides);
382
+ flat.push(key, redact?.urls === true ? hideUrlPasswords(json) : json);
295
383
  }
296
384
  return flat;
297
385
  }
@@ -315,3 +403,27 @@ function traceOf(trace) {
315
403
  }
316
404
  return spanId !== undefined && /^[0-9a-f]{16}$/.test(spanId) ? { traceId, spanId } : { traceId };
317
405
  }
406
+ /** This file, whose frames a line's source skips. */
407
+ const here = pathOf(import.meta.url);
408
+ /**
409
+ * Where a line was logged, as slog's handlers spell a source: the first frame
410
+ * of the stack outside this file.
411
+ */
412
+ function sourceOfCaller() {
413
+ for (const frame of (new Error().stack ?? '').split('\n').slice(1)) {
414
+ const found = /at (?:(.+?) \()?(.+):(\d+):\d+\)?$/.exec(frame.trim());
415
+ if (found === null || pathOf(found[2]) === here) {
416
+ continue;
417
+ }
418
+ return JSON.stringify({
419
+ function: found[1] ?? '',
420
+ file: pathOf(found[2]),
421
+ line: Number(found[3]),
422
+ });
423
+ }
424
+ return '{}';
425
+ }
426
+ /** A frame's file as a path, which Node writes as a file: URL. */
427
+ function pathOf(file) {
428
+ return file.startsWith('file://') ? fileURLToPath(file) : file;
429
+ }