@tinyshed/tinystore 0.1.0-rc.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (100) hide show
  1. package/LICENSE +202 -0
  2. package/README.md +262 -0
  3. package/bin/tinystore.js +50 -0
  4. package/dist/background.d.ts +30 -0
  5. package/dist/background.js +93 -0
  6. package/dist/blobs.d.ts +98 -0
  7. package/dist/blobs.js +299 -0
  8. package/dist/cancel.d.ts +14 -0
  9. package/dist/cancel.js +22 -0
  10. package/dist/clock.d.ts +21 -0
  11. package/dist/clock.js +35 -0
  12. package/dist/config.d.ts +60 -0
  13. package/dist/config.js +417 -0
  14. package/dist/connection.d.ts +78 -0
  15. package/dist/connection.js +405 -0
  16. package/dist/console.d.ts +48 -0
  17. package/dist/console.js +195 -0
  18. package/dist/errors.d.ts +103 -0
  19. package/dist/errors.js +158 -0
  20. package/dist/handles.d.ts +28 -0
  21. package/dist/handles.js +37 -0
  22. package/dist/index.d.ts +21 -0
  23. package/dist/index.js +7 -0
  24. package/dist/jobs.d.ts +304 -0
  25. package/dist/jobs.js +684 -0
  26. package/dist/kv.d.ts +283 -0
  27. package/dist/kv.js +815 -0
  28. package/dist/limiter.d.ts +45 -0
  29. package/dist/limiter.js +68 -0
  30. package/dist/logger.d.ts +96 -0
  31. package/dist/logger.js +243 -0
  32. package/dist/metrics.d.ts +249 -0
  33. package/dist/metrics.js +610 -0
  34. package/dist/once.d.ts +31 -0
  35. package/dist/once.js +116 -0
  36. package/dist/quota.d.ts +44 -0
  37. package/dist/quota.js +98 -0
  38. package/dist/records.d.ts +182 -0
  39. package/dist/records.js +346 -0
  40. package/dist/runtime/bun.d.ts +2 -0
  41. package/dist/runtime/bun.js +164 -0
  42. package/dist/runtime/node.d.ts +2 -0
  43. package/dist/runtime/node.js +157 -0
  44. package/dist/runtime/shared.d.ts +25 -0
  45. package/dist/runtime/shared.js +45 -0
  46. package/dist/runtime.d.ts +48 -0
  47. package/dist/runtime.js +4 -0
  48. package/dist/schema.d.ts +39 -0
  49. package/dist/schema.js +21 -0
  50. package/dist/session.d.ts +120 -0
  51. package/dist/session.js +546 -0
  52. package/dist/sql.d.ts +121 -0
  53. package/dist/sql.js +342 -0
  54. package/dist/store.d.ts +103 -0
  55. package/dist/store.js +153 -0
  56. package/dist/time.d.ts +35 -0
  57. package/dist/time.js +44 -0
  58. package/dist/trace.d.ts +18 -0
  59. package/dist/trace.js +22 -0
  60. package/dist/wire/codec.d.ts +90 -0
  61. package/dist/wire/codec.js +383 -0
  62. package/dist/wire/frame.d.ts +41 -0
  63. package/dist/wire/frame.js +102 -0
  64. package/dist/wire/messages.d.ts +1403 -0
  65. package/dist/wire/messages.js +472 -0
  66. package/dist/wire/msgpack.d.ts +93 -0
  67. package/dist/wire/msgpack.js +680 -0
  68. package/package.json +40 -0
  69. package/src/background.ts +108 -0
  70. package/src/blobs.ts +437 -0
  71. package/src/cancel.ts +26 -0
  72. package/src/clock.ts +45 -0
  73. package/src/config.ts +501 -0
  74. package/src/connection.ts +512 -0
  75. package/src/console.ts +228 -0
  76. package/src/errors.ts +211 -0
  77. package/src/handles.ts +68 -0
  78. package/src/index.ts +92 -0
  79. package/src/jobs.ts +963 -0
  80. package/src/kv.ts +1093 -0
  81. package/src/limiter.ts +99 -0
  82. package/src/logger.ts +326 -0
  83. package/src/metrics.ts +896 -0
  84. package/src/once.ts +142 -0
  85. package/src/quota.ts +137 -0
  86. package/src/records.ts +537 -0
  87. package/src/runtime/bun.ts +190 -0
  88. package/src/runtime/node.ts +177 -0
  89. package/src/runtime/shared.ts +61 -0
  90. package/src/runtime.ts +57 -0
  91. package/src/schema.ts +69 -0
  92. package/src/session.ts +665 -0
  93. package/src/sql.ts +478 -0
  94. package/src/store.ts +219 -0
  95. package/src/time.ts +87 -0
  96. package/src/trace.ts +32 -0
  97. package/src/wire/codec.ts +445 -0
  98. package/src/wire/frame.ts +139 -0
  99. package/src/wire/messages.ts +562 -0
  100. package/src/wire/msgpack.ts +712 -0
package/src/limiter.ts ADDED
@@ -0,0 +1,99 @@
1
+ // A limiter of requests by key, as Go's kv.Limiter keeps one: the generic
2
+ // cell rate algorithm, one time a key, which the server holds in memory and
3
+ // writes every second, so a crash lets at most one burst more through.
4
+
5
+ import type { Link } from './connection.ts'
6
+ import { InvalidError } from './errors.ts'
7
+ import { checkName, handleOn, ownerText } from './handles.ts'
8
+ import { ms, type Unit } from './time.ts'
9
+ import type { Key } from './wire/codec.ts'
10
+ import { KvAllowance, KvBucket, KvCall, methods } from './wire/messages.ts'
11
+
12
+ /**
13
+ * How many pass a span: '100/s', '5/10s', '1000/h', '300/7d', a span of one
14
+ * unit leaving out its 1. The type checks it as it is written.
15
+ */
16
+ export type Rate = `${number}/${Unit}` | `${number}/${number}${Unit}`
17
+
18
+ export interface LimiterOptions {
19
+ /** how many requests pass a span, each key on its own */
20
+ rate: Rate
21
+ /** how many may pass at once before the rate holds the next; the rate's count when absent */
22
+ burst?: number
23
+ }
24
+
25
+ /** What a limiter answers a request. */
26
+ export interface Allowance {
27
+ ok: boolean
28
+ /** how many more requests would pass now */
29
+ left: number
30
+ /** milliseconds until the request would pass; 0 when ok */
31
+ retryAfter: number
32
+ }
33
+
34
+ /**
35
+ * A rate's count and span in milliseconds: a span of one unit may leave out
36
+ * its 1.
37
+ *
38
+ * '100/s' → 100 a 1000 ms, '5/10s' → 5 a 10000 ms
39
+ */
40
+ export function rateOf(rate: string): { count: number; per: number } {
41
+ const parts = /^(\d+)\/(\d*)([a-z]+)$/.exec(rate)
42
+ if (parts === null) {
43
+ throw new InvalidError(`the rate ${JSON.stringify(rate)}; write it as 100/s, 5/10s or 1000/h`)
44
+ }
45
+ const [, count, span, unit] = parts
46
+ return { count: Number(count), per: ms(`${span === '' ? '1' : span}${unit}`) }
47
+ }
48
+
49
+ /** A limiter: open it with `store.kv.limiter(name, { rate })`. */
50
+ export class Limiter {
51
+ readonly name: string
52
+ readonly #link: Link
53
+ readonly #open: Uint8Array
54
+ readonly #owners: (string | Uint8Array)[]
55
+
56
+ /** Limiters come from `store.kv.limiter` and their `of`. */
57
+ constructor(link: Link, name: string, open: Uint8Array, owners: (string | Uint8Array)[]) {
58
+ this.#link = link
59
+ this.name = name
60
+ this.#open = open
61
+ this.#owners = owners
62
+ }
63
+
64
+ /** The limiter of a branch, each key apart from the same key elsewhere. */
65
+ of(...owners: Key[]): Limiter {
66
+ return new Limiter(this.#link, this.name, this.#open, [
67
+ ...this.#owners,
68
+ ...owners.map(ownerText),
69
+ ])
70
+ }
71
+
72
+ /** Asks for n requests of key, one unless given: all pass or none does; past the burst is InvalidError. */
73
+ async allow(key: Key, n = 1): Promise<Allowance> {
74
+ const answer = await this.#link.run('write', async connection => {
75
+ const handle = await handleOn(connection, methods['kv.open'], this.#open)
76
+ const body = KvCall.encode({
77
+ handle,
78
+ owners: this.#owners.length > 0 ? this.#owners : undefined,
79
+ key,
80
+ n: n === 1 ? undefined : n,
81
+ })
82
+ return KvAllowance.decode(await connection.session.call(methods['kv.allow'], body))
83
+ })
84
+ return { ok: answer.ok === true, left: answer.left ?? 0, retryAfter: answer.retryAfter ?? 0 }
85
+ }
86
+ }
87
+
88
+ /** A limiter's open: its name and rate, checked before anything leaves. */
89
+ export function limiterOpen(name: string, options: LimiterOptions): Uint8Array {
90
+ checkName(name, 'limiter')
91
+ const { count, per } = rateOf(options.rate)
92
+ if (count < 1 || per < 1) {
93
+ throw new InvalidError(`limiter ${name}: a rate of ${options.rate}`)
94
+ }
95
+ if (options.burst !== undefined && (!Number.isInteger(options.burst) || options.burst < 1)) {
96
+ throw new InvalidError(`limiter ${name}: a burst of ${options.burst}`)
97
+ }
98
+ return KvBucket.encode({ name, rate: count, per, burst: options.burst })
99
+ }
package/src/logger.ts ADDED
@@ -0,0 +1,326 @@
1
+ // The application's own log lines, as Go's slog handler and Python's
2
+ // logging.Handler write them: each on the console as it is logged, and as a
3
+ // record when the logger is a store's. A call never waits for the server.
4
+ // Lines wait in a bounded buffer and go every second, or once half of it
5
+ // waits; what does not fit, or a write that fails, is dropped and counted,
6
+ // and said in TinyStore's own lines, since no call is waiting to be told.
7
+
8
+ import { DropNotice, FailureLog, type Say } from './background.ts'
9
+ import {
10
+ type ConsoleFormat,
11
+ type ConsoleLine,
12
+ jsonLine,
13
+ jsonOf,
14
+ prettyLine,
15
+ redacted,
16
+ redactor,
17
+ } from './console.ts'
18
+ import { InvalidError } from './errors.ts'
19
+ import type { Fields, Level } from './records.ts'
20
+ import { currentTrace, type Trace } from './trace.ts'
21
+
22
+ export interface LoggerOptions {
23
+ /** the lines held between writes; past it a line is dropped and counted: 1024 */
24
+ buffer?: number
25
+ /** the least level a line is kept at; every level when absent */
26
+ level?: Level
27
+ /** how a line is written as it is logged: pretty on a terminal and JSON otherwise when absent */
28
+ console?: ConsoleFormat
29
+ /** fields whose values are hidden, in the store and on the console, at any depth, the case ignored */
30
+ redact?: readonly string[]
31
+ /** the console lines go to stdout rather than stderr */
32
+ stdout?: boolean
33
+ }
34
+
35
+ /** Where a console line goes: a process's stream, or a test's. */
36
+ export interface ConsoleOut {
37
+ write(text: string): unknown
38
+ readonly isTTY?: boolean
39
+ }
40
+
41
+ const levels = { debug: -4, info: 0, warn: 4, error: 8 } as const
42
+ const flushEvery = 1000
43
+
44
+ /** What a logger and the loggers its `with` makes share: one buffer, one timer, one count, one console. */
45
+ interface Shared {
46
+ /** undefined for a logger of the console alone */
47
+ append: ((lines: ConsoleLine[]) => Promise<void>) | undefined
48
+ stream: string
49
+ most: number
50
+ least: number
51
+ hides: ((key: string) => boolean) | undefined
52
+ print: ((line: ConsoleLine) => void) | undefined
53
+ waiting: ConsoleLine[]
54
+ timer: ReturnType<typeof setInterval> | undefined
55
+ writing: Promise<void> | undefined
56
+ dropped: number
57
+ /** undefined for a logger of the console alone, which writes nothing */
58
+ failures: FailureLog | undefined
59
+ drops: DropNotice | undefined
60
+ }
61
+
62
+ /**
63
+ * A logger of the console alone, for a program that wants the logger and not
64
+ * the records: its lines are written as a store's logger writes them, and
65
+ * `store.records.logger(stream)` in its place keeps them too.
66
+ *
67
+ * ```ts
68
+ * const log = logger('app', { redact: ['password'] })
69
+ * const db = log.with({ module: 'db' }) // passed to what needs it
70
+ * ```
71
+ */
72
+ export function logger(stream: string, options?: LoggerOptions): Logger {
73
+ return newLogger(undefined, stream, options)
74
+ }
75
+
76
+ /** A logger of one stream, its lines appended through append when it is a store's. */
77
+ export function newLogger(
78
+ append: ((lines: ConsoleLine[]) => Promise<void>) | undefined,
79
+ stream: string,
80
+ options: LoggerOptions = {},
81
+ out?: ConsoleOut & { utc?: boolean },
82
+ ): Logger {
83
+ if (stream === '') {
84
+ throw new InvalidError('a logger of no stream')
85
+ }
86
+ const most = options.buffer ?? 1024
87
+ if (!Number.isInteger(most) || most < 1) {
88
+ throw new InvalidError(`a logger's buffer of ${most} lines`)
89
+ }
90
+ const shared: Shared = {
91
+ append,
92
+ stream,
93
+ most,
94
+ least: options.level === undefined ? Number.NEGATIVE_INFINITY : levelOf(options.level),
95
+ hides: redactor(options.redact),
96
+ print: printer(options, out),
97
+ waiting: [],
98
+ timer: undefined,
99
+ writing: undefined,
100
+ dropped: 0,
101
+ failures: append === undefined ? undefined : new FailureLog(`records flush ${stream}`, sayOwn),
102
+ drops: append === undefined ? undefined : new DropNotice(stream, most, sayOwn),
103
+ }
104
+ return new Logger(shared, [])
105
+ }
106
+
107
+ let own: Logger | undefined
108
+
109
+ /** TinyStore's own lines, on the console of a logger of stream tinystore made when first needed. */
110
+ export const sayOwn: Say = (level, message, fields) => {
111
+ own ??= logger('tinystore')
112
+ own[level](message, fields)
113
+ }
114
+
115
+ /** How a logger prints its lines, or undefined when it prints none. */
116
+ function printer(options: LoggerOptions, out?: ConsoleOut & { utc?: boolean }): Shared['print'] {
117
+ if (options.console === 'off') {
118
+ return undefined
119
+ }
120
+ const stream: ConsoleOut = out ?? (options.stdout === true ? process.stdout : process.stderr)
121
+ const terminal = stream.isTTY === true
122
+ const pretty = options.console === 'pretty' || (options.console === undefined && terminal)
123
+ if (!pretty) {
124
+ return line => void stream.write(jsonLine(line))
125
+ }
126
+ const color =
127
+ terminal && out === undefined && !process.env.NO_COLOR && process.env.TERM !== 'dumb'
128
+ return line => void stream.write(prettyLine(line, color, out?.utc === true))
129
+ }
130
+
131
+ /**
132
+ * A logger of one stream. `info`, `warn` and the rest return at once; a line
133
+ * goes to the console as it is logged, and becomes a record named `log`, its
134
+ * message the body, the call's fields its attributes and the fields of `with`
135
+ * its context, who is speaking.
136
+ *
137
+ * ```ts
138
+ * const log = store.records.logger('api')
139
+ * log.info('server started', { port: 3000 })
140
+ * log.with({ requestId }).warn('slow request', { ms: 1200 })
141
+ * log.event('user.created', { userId: 42 }) // a record named for the event
142
+ * ```
143
+ */
144
+ export class Logger {
145
+ readonly #shared: Shared
146
+ /** key, JSON, key, JSON…, hidden as the logger's redact says */
147
+ readonly #context: string[]
148
+
149
+ /** Loggers come from `logger(stream)`, `store.records.logger(stream)` and their `with`. */
150
+ constructor(shared: Shared, context: string[]) {
151
+ this.#shared = shared
152
+ this.#context = context
153
+ }
154
+
155
+ /**
156
+ * The lines dropped since the logger began: a full buffer, or a write that
157
+ * failed. Each is also said on stderr, at most once in ten minutes.
158
+ */
159
+ get dropped(): number {
160
+ return this.#shared.dropped
161
+ }
162
+
163
+ debug(message: string, attrs?: Fields): void {
164
+ this.#log(levels.debug, message, attrs)
165
+ }
166
+
167
+ info(message: string, attrs?: Fields): void {
168
+ this.#log(levels.info, message, attrs)
169
+ }
170
+
171
+ warn(message: string, attrs?: Fields): void {
172
+ this.#log(levels.warn, message, attrs)
173
+ }
174
+
175
+ error(message: string, attrs?: Fields): void {
176
+ this.#log(levels.error, message, attrs)
177
+ }
178
+
179
+ /** An event of the stream: a record named for it, with no level and no body. */
180
+ event(name: string, attrs?: Fields): void {
181
+ if (name === '') {
182
+ throw new InvalidError('an event of no name')
183
+ }
184
+ this.#emit({ event: name }, attrs)
185
+ }
186
+
187
+ /** The logger of the same stream, buffer and console, its context these fields besides its own. */
188
+ with(context: Fields): Logger {
189
+ return new Logger(this.#shared, [
190
+ ...this.#context,
191
+ ...encodeFields(context, this.#shared.hides),
192
+ ])
193
+ }
194
+
195
+ /** Hands what the buffer holds to the server now, as the timer does every second. */
196
+ async flush(): Promise<void> {
197
+ const shared = this.#shared
198
+ while (shared.writing !== undefined) {
199
+ await shared.writing
200
+ }
201
+ shared.drops?.sayIfDue()
202
+ if (shared.waiting.length === 0 || shared.append === undefined) {
203
+ return
204
+ }
205
+ shared.writing = writeWaiting(shared, shared.append).finally(() => {
206
+ shared.writing = undefined
207
+ })
208
+ await shared.writing
209
+ }
210
+
211
+ /** Hands over what the buffer holds and stops the timer; a later line starts it again. */
212
+ async stop(): Promise<void> {
213
+ await this.flush()
214
+ if (this.#shared.timer !== undefined) {
215
+ clearInterval(this.#shared.timer)
216
+ this.#shared.timer = undefined
217
+ }
218
+ }
219
+
220
+ #log(level: number, message: string, attrs: Fields | undefined): void {
221
+ if (level < this.#shared.least) {
222
+ return
223
+ }
224
+ this.#emit({ level, msg: message }, attrs)
225
+ }
226
+
227
+ #emit(what: Pick<ConsoleLine, 'level' | 'event' | 'msg'>, attrs: Fields | undefined): void {
228
+ const trace = traceOf(currentTrace())
229
+ const line: ConsoleLine = {
230
+ at: new Date(),
231
+ stream: this.#shared.stream,
232
+ ...what,
233
+ context: this.#context,
234
+ attrs: attrs === undefined ? [] : encodeFields(attrs, this.#shared.hides),
235
+ ...trace,
236
+ }
237
+ this.#shared.print?.(line)
238
+ this.#queue(line)
239
+ }
240
+
241
+ #queue(line: ConsoleLine): void {
242
+ const shared = this.#shared
243
+ if (shared.append === undefined) {
244
+ return
245
+ }
246
+ if (shared.waiting.length >= shared.most) {
247
+ shared.dropped++
248
+ shared.drops?.dropped()
249
+ return
250
+ }
251
+ shared.waiting.push(line)
252
+ if (shared.timer === undefined) {
253
+ shared.timer = setInterval(() => void this.flush(), flushEvery)
254
+ shared.timer.unref?.()
255
+ }
256
+ if (shared.waiting.length >= shared.most / 2 && shared.writing === undefined) {
257
+ void this.flush()
258
+ }
259
+ }
260
+ }
261
+
262
+ /**
263
+ * Writes what the buffer holds, and again while half of it filled during the
264
+ * write before. A full buffer drops a line before the line could ask for a
265
+ * write, so a burst that filled it while a write ran would otherwise wait for
266
+ * the timer's next second, every line until then dropped.
267
+ */
268
+ async function writeWaiting(
269
+ shared: Shared,
270
+ append: (lines: ConsoleLine[]) => Promise<void>,
271
+ ): Promise<void> {
272
+ let batch = shared.waiting.splice(0)
273
+ while (batch.length > 0) {
274
+ try {
275
+ await append(batch)
276
+ shared.failures?.succeeded()
277
+ } catch (err) {
278
+ shared.dropped += batch.length
279
+ shared.failures?.failed(err)
280
+ }
281
+ batch = shared.waiting.length >= shared.most / 2 ? shared.waiting.splice(0) : []
282
+ }
283
+ }
284
+
285
+ /**
286
+ * Fields as key, JSON pairs, as a console writes them and the store keeps
287
+ * them: an Error's value its stack, which JSON would write as {}, and a field
288
+ * hides names hidden at any depth.
289
+ */
290
+ export function encodeFields(
291
+ fields: Fields,
292
+ hides: ((key: string) => boolean) | undefined,
293
+ ): string[] {
294
+ const pairs = Array.isArray(fields)
295
+ ? (fields as readonly (readonly [string, unknown])[])
296
+ : Object.entries(fields as Record<string, unknown>)
297
+ const flat: string[] = []
298
+ for (const [key, value] of pairs) {
299
+ const shown = value instanceof Error ? (value.stack ?? String(value)) : value
300
+ flat.push(key, hides?.(key) ? redacted : jsonOf(shown, hides))
301
+ }
302
+ return flat
303
+ }
304
+
305
+ function levelOf(level: Level): number {
306
+ return typeof level === 'number' ? level : levels[level]
307
+ }
308
+
309
+ /** A trace's ids as hex, or none when they are not a trace's sixteen bytes and a span's eight. */
310
+ function traceOf(trace: Trace | undefined): Pick<ConsoleLine, 'traceId' | 'spanId'> {
311
+ if (trace === undefined) {
312
+ return {}
313
+ }
314
+ const hex = (id: Uint8Array | string | undefined) =>
315
+ typeof id === 'string'
316
+ ? id.toLowerCase()
317
+ : id === undefined
318
+ ? undefined
319
+ : Buffer.from(id).toString('hex')
320
+ const traceId = hex(trace.traceId)
321
+ const spanId = hex(trace.spanId)
322
+ if (traceId === undefined || !/^[0-9a-f]{32}$/.test(traceId)) {
323
+ return {}
324
+ }
325
+ return spanId !== undefined && /^[0-9a-f]{16}$/.test(spanId) ? { traceId, spanId } : { traceId }
326
+ }