@tinyshed/tinystore 0.7.1 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@tinyshed/tinystore",
3
- "version": "0.7.1",
3
+ "version": "0.9.0",
4
4
  "description": "TinyStore for Bun and Node: kv, jobs, blobs, SQL, records and metrics in one directory, served by a sidecar",
5
5
  "license": "Apache-2.0",
6
6
  "homepage": "https://tinyshed.org/tinystore",
@@ -31,12 +31,12 @@
31
31
  "README.md"
32
32
  ],
33
33
  "optionalDependencies": {
34
- "@tinyshed/tinystore-linux-x64": "0.7.1",
35
- "@tinyshed/tinystore-linux-arm64": "0.7.1",
36
- "@tinyshed/tinystore-linux-x64-musl": "0.7.1",
37
- "@tinyshed/tinystore-linux-arm64-musl": "0.7.1",
38
- "@tinyshed/tinystore-darwin-arm64": "0.7.1",
39
- "@tinyshed/tinystore-darwin-x64": "0.7.1",
40
- "@tinyshed/tinystore-win32-x64": "0.7.1"
34
+ "@tinyshed/tinystore-linux-x64": "0.9.0",
35
+ "@tinyshed/tinystore-linux-arm64": "0.9.0",
36
+ "@tinyshed/tinystore-linux-x64-musl": "0.9.0",
37
+ "@tinyshed/tinystore-linux-arm64-musl": "0.9.0",
38
+ "@tinyshed/tinystore-darwin-arm64": "0.9.0",
39
+ "@tinyshed/tinystore-darwin-x64": "0.9.0",
40
+ "@tinyshed/tinystore-win32-x64": "0.9.0"
41
41
  }
42
42
  }
package/src/backup.ts CHANGED
@@ -35,6 +35,8 @@ export interface Backup {
35
35
  * hold: they open with that key alone, kept apart
36
36
  */
37
37
  keyLeftOut: boolean
38
+ /** the databases opened with `backup: false`, by name, of which it holds nothing */
39
+ excluded: string[]
38
40
  }
39
41
 
40
42
  const nothing = new Uint8Array()
@@ -73,6 +75,7 @@ export async function backup(link: Link, path: string, options: BackupOptions):
73
75
  files: Number(backed.files ?? 0),
74
76
  bytes: Number(backed.bytes ?? 0),
75
77
  keyLeftOut: backed.keyLeftOut === true,
78
+ excluded: backed.excluded ?? [],
76
79
  }
77
80
  } catch (err) {
78
81
  await rm(partial, { force: true })
package/src/blobs.ts CHANGED
@@ -357,48 +357,8 @@ export class Files {
357
357
  offset?: number,
358
358
  length?: number,
359
359
  ): ReadableStream<Uint8Array> {
360
- let stream: Stream | undefined
361
- const open = () =>
362
- this.#link.run('read', async connection => {
363
- const read = BlobsRead.encode({
364
- handle: await this.#handle(connection),
365
- folder: this.#folder,
366
- path,
367
- ifMatch,
368
- offset,
369
- length,
370
- })
371
- const opened = await connection.session.open(methods['blobs.read'], read, true)
372
- return { opened, first: await opened.next() }
373
- })
374
- return new ReadableStream<Uint8Array>({
375
- async pull(controller) {
376
- try {
377
- if (stream === undefined) {
378
- // the RESPONSE's piece is no DATA, so it takes none of the credit
379
- const { opened, first } = await open()
380
- stream = opened
381
- enqueue(controller, first.body)
382
- if (first.end) {
383
- controller.close()
384
- }
385
- return
386
- }
387
- const event = await stream.next()
388
- if (event.end) {
389
- controller.close()
390
- return
391
- }
392
- stream.consumed(event.body.length)
393
- enqueue(controller, event.body)
394
- } catch (err) {
395
- controller.error(err)
396
- }
397
- },
398
- cancel() {
399
- stream?.cancel()
400
- },
401
- })
360
+ const at = { folder: this.#folder, path, ifMatch, offset, length }
361
+ return readStream(this.#link, connection => this.#handle(connection), at)
402
362
  }
403
363
 
404
364
  async #move(
@@ -594,9 +554,15 @@ export class FileBytes {
594
554
  // a copy, which a Buffer's slice under Node is not
595
555
  return new Uint8Array(this.#held)
596
556
  }
597
- const whole = new Uint8Array(this.size)
557
+ let whole = new Uint8Array(this.size)
598
558
  let at = 0
599
559
  for await (const piece of this.stream() as AsyncIterable<Uint8Array>) {
560
+ // a growing directory's file reads past the size its get gave
561
+ if (at + piece.length > whole.length) {
562
+ const grown = new Uint8Array(Math.max(at + piece.length, whole.length * 2))
563
+ grown.set(whole.subarray(0, at))
564
+ whole = grown
565
+ }
600
566
  whole.set(piece, at)
601
567
  at += piece.length
602
568
  }
@@ -666,6 +632,61 @@ export class StoredFile extends FileBytes implements FileInfo {
666
632
  }
667
633
  }
668
634
 
635
+ /** Where a read of a file's bytes stands, and the ETag it holds to; none holds to no version. */
636
+ export interface ReadAt {
637
+ folder: string[]
638
+ path: string
639
+ ifMatch: string | undefined
640
+ offset?: number | undefined
641
+ length?: number | undefined
642
+ }
643
+
644
+ /**
645
+ * A file's bytes as S3's GET reads them with a range and If-Match, on the
646
+ * handle `handle` opens on a connection; nothing is asked before the first read.
647
+ */
648
+ export function readStream(
649
+ link: Link,
650
+ handle: (connection: Connection) => Promise<number>,
651
+ at: ReadAt,
652
+ ): ReadableStream<Uint8Array> {
653
+ let stream: Stream | undefined
654
+ const open = () =>
655
+ link.run('read', async connection => {
656
+ const read = BlobsRead.encode({ ...at, handle: await handle(connection) })
657
+ const opened = await connection.session.open(methods['blobs.read'], read, true)
658
+ return { opened, first: await opened.next() }
659
+ })
660
+ return new ReadableStream<Uint8Array>({
661
+ async pull(controller) {
662
+ try {
663
+ if (stream === undefined) {
664
+ // the RESPONSE's piece is no DATA, so it takes none of the credit
665
+ const { opened, first } = await open()
666
+ stream = opened
667
+ enqueue(controller, first.body)
668
+ if (first.end) {
669
+ controller.close()
670
+ }
671
+ return
672
+ }
673
+ const event = await stream.next()
674
+ if (event.end) {
675
+ controller.close()
676
+ return
677
+ }
678
+ stream.consumed(event.body.length)
679
+ enqueue(controller, event.body)
680
+ } catch (err) {
681
+ controller.error(err)
682
+ }
683
+ },
684
+ cancel() {
685
+ stream?.cancel()
686
+ },
687
+ })
688
+ }
689
+
669
690
  /** A place in bytes as Blob.slice reads one: negative from the end, and within the size. */
670
691
  function placeOf(place: number | undefined, size: number, absent: number): number {
671
692
  if (place === undefined) {
@@ -859,7 +880,7 @@ function writtenOf(body: Uint8Array): FileInfo | undefined {
859
880
  return written.info === undefined ? undefined : infoOf(written.info)
860
881
  }
861
882
 
862
- function infoOf(info: ReturnType<typeof BlobsInfo.decode>): FileInfo {
883
+ export function infoOf(info: ReturnType<typeof BlobsInfo.decode>): FileInfo {
863
884
  return {
864
885
  path: info.path ?? '',
865
886
  size: info.size ?? 0,
@@ -872,7 +893,7 @@ function infoOf(info: ReturnType<typeof BlobsInfo.decode>): FileInfo {
872
893
  }
873
894
 
874
895
  /** A folder's segment as text, an integer its decimal spelling; the server refuses one that is not a segment. */
875
- function segmentOf(segment: string | number | bigint): string {
896
+ export function segmentOf(segment: string | number | bigint): string {
876
897
  if (typeof segment === 'number' && !Number.isSafeInteger(segment)) {
877
898
  throw new InvalidError(`the folder segment ${segment}: a segment is text or an integer`)
878
899
  }
package/src/console.ts CHANGED
@@ -81,9 +81,12 @@ const cyan = '\x1b[36m'
81
81
  /**
82
82
  * A line as a person reads it: the time of day, the level padded to five, the
83
83
  * stream, the message, the context's fields and the attributes', and the
84
- * trace's first eight digits. A value of several lines, a stack or a
85
- * traceback, follows the line, indented. A key or a string shows without its
86
- * quotes when it is one word with no '=', quote or backslash.
84
+ * trace's first eight digits. A message's further lines, and then a value of
85
+ * several lines, a stack or a traceback, follow the line, indented, so that
86
+ * none of them starts where a line does. A key or a string shows without its
87
+ * quotes when it is one word with no '=', quote or backslash. What a program
88
+ * logged shows no control character but a tab: the message, the event's name,
89
+ * a value below the line and a source each show it as its escape.
87
90
  */
88
91
  export function prettyLine(line: ConsoleLine, look: Look): string {
89
92
  const paint = (hue: string, text: string) => (look.color ? hue + text + reset : text)
@@ -96,7 +99,7 @@ export function prettyLine(line: ConsoleLine, look: Look): string {
96
99
  out += `${separator}${paint(cyan, line.stream)}`
97
100
  separator = ' '
98
101
  }
99
- const message = prettyMessage(line)
102
+ const [message, further] = prettyMessage(line)
100
103
  if (message !== '') {
101
104
  out += `${separator}${message}`
102
105
  separator = ' '
@@ -119,18 +122,48 @@ export function prettyLine(line: ConsoleLine, look: Look): string {
119
122
  if (line.traceId !== undefined) {
120
123
  field('trace', line.traceId.slice(0, 8))
121
124
  }
125
+ for (const part of further) {
126
+ out += `\n ${escaped(part)}`
127
+ }
122
128
  for (const [key, text] of below) {
123
- text
124
- .replace(/[\r\n]+$/, '')
125
- .split('\n')
126
- .forEach((part, i) => {
127
- const shown = part.replace(/\r$/, '')
128
- out += `\n ${paint(dim, i === 0 ? `${key}: ${shown}` : shown)}`
129
- })
129
+ linesOf(text).forEach((part, i) => {
130
+ out += `\n ${paint(dim, escaped(i === 0 ? `${key}: ${part}` : part))}`
131
+ })
130
132
  }
131
133
  return `${out}\n`
132
134
  }
133
135
 
136
+ /**
137
+ * The lines of a text of several, without the line ends after the last, and
138
+ * each without the carriage return before its newline: "line one\r\nline
139
+ * two\r\n" is line one, line two.
140
+ */
141
+ function linesOf(text: string): string[] {
142
+ const lines = text.replace(/[\r\n]+$/, '').split('\n')
143
+ return lines.map(part => part.replace(/\r$/, ''))
144
+ }
145
+
146
+ /**
147
+ * Text a program logged as a pretty line shows it: each control character but
148
+ * a tab, and DEL, as its JSON escape, so that none reaches the terminal as an
149
+ * escape sequence, a carriage return or a line of its own. The colours the
150
+ * console paints with are its own, never in this text.
151
+ *
152
+ * "\x1b[2J" → \u001b[2J "a\rb" → a\rb "\x7f" → \u007f
153
+ */
154
+ function escaped(text: string): string {
155
+ let out = ''
156
+ for (const c of text) {
157
+ const code = c.codePointAt(0) as number
158
+ if ((code < 0x20 && c !== '\t') || code === 0x7f) {
159
+ out += code === 0x7f ? '\\u007f' : JSON.stringify(c).slice(1, -1)
160
+ } else {
161
+ out += c
162
+ }
163
+ }
164
+ return out
165
+ }
166
+
134
167
  /** A level as slog spells it: INFO, WARN+2, DEBUG-4. */
135
168
  export function levelLabel(level: number): string {
136
169
  const spell = (base: string, from: number) => {
@@ -157,12 +190,19 @@ function prettyLevel(level: number | undefined): [string, string] {
157
190
  return [levelLabel(level), hue]
158
191
  }
159
192
 
160
- function prettyMessage(line: ConsoleLine): string {
161
- const body = line.msg ?? ''
193
+ /**
194
+ * A log line's message, or an event's name and the message it may carry, as
195
+ * the line shows them; and the further lines of a message of several, which
196
+ * go below the line as a value's do.
197
+ */
198
+ function prettyMessage(line: ConsoleLine): [string, string[]] {
199
+ const [first, ...further] = linesOf(line.msg ?? '')
200
+ const body = escaped(first ?? '')
162
201
  if (line.event === undefined) {
163
- return body
202
+ return [body, further]
164
203
  }
165
- return body === '' ? line.event : `${line.event} ${body}`
204
+ const event = escaped(line.event)
205
+ return [body === '' ? event : `${event} ${body}`, further]
166
206
  }
167
207
 
168
208
  /** How a field's JSON shows, and whether it is text of several lines, which goes under the line. */
@@ -247,7 +287,7 @@ function shortSource(json: string): string | undefined {
247
287
  }
248
288
  const parts = source.file.split(/[\\/]+/).filter(part => part !== '')
249
289
  const line = typeof source.line === 'number' ? source.line : 0
250
- return `${parts.slice(-2).join('/')}:${line}`
290
+ return escaped(`${parts.slice(-2).join('/')}:${line}`)
251
291
  }
252
292
 
253
293
  /** The names `redact` takes to hide the usual secrets, as Go's console.Secrets and Python's SECRETS. */
@@ -0,0 +1,235 @@
1
+ // A directory, as docs/guides/blobs/directories.md has it: a folder of the
2
+ // store's machine under a name, which the store lists and reads and never
3
+ // writes. A scan writes down what appeared, changed or went; changes keep a
4
+ // copy in step from a cursor; a read comes from the disk, held to the version
5
+ // its get opened, or in a growing directory to none.
6
+
7
+ import {
8
+ type FileInfo,
9
+ type FileListOptions,
10
+ type FilePage,
11
+ infoOf,
12
+ readStream,
13
+ StoredFile,
14
+ segmentOf,
15
+ } from './blobs.ts'
16
+ import type { Connection, Link } from './connection.ts'
17
+ import { checkName, handleOn, type Via, viaLink } from './handles.ts'
18
+ import { type Duration, ms } from './time.ts'
19
+ import {
20
+ BlobsAt,
21
+ BlobsChanges,
22
+ BlobsDirectory,
23
+ BlobsFolder,
24
+ BlobsGot,
25
+ BlobsHead,
26
+ BlobsList,
27
+ BlobsPage,
28
+ BlobsScanned,
29
+ BlobsSince,
30
+ BlobsUsage,
31
+ methods,
32
+ } from './wire/protocol.ts'
33
+
34
+ export interface DirectoryOptions {
35
+ /** The folder on the store's machine; a server's own, by the name, when left out. */
36
+ path?: string | undefined
37
+ /** A scan on the store's background work, a period apart: minutes, not seconds. */
38
+ every?: Duration | undefined
39
+ /** Names a scan passes over, a file's or a folder's, `*` for any run of a name's characters. */
40
+ ignore?: string[] | undefined
41
+ /** For files written while they are read: a read holds to no version and goes up to the size now. */
42
+ growing?: boolean | undefined
43
+ }
44
+
45
+ /** What a scan found, and what the list holds once it ended. */
46
+ export interface Scan {
47
+ added: number
48
+ changed: number
49
+ removed: number
50
+ /** what it met and could not list: a link, a folder it cannot read, another disk */
51
+ skipped: number
52
+ count: number
53
+ size: number
54
+ }
55
+
56
+ export type Change = 'added' | 'changed' | 'removed'
57
+
58
+ /** A file's change since a cursor: its path in the directory, and what it carries now, none once removed. */
59
+ export interface FileChange {
60
+ path: string
61
+ change: Change
62
+ info: FileInfo | undefined
63
+ }
64
+
65
+ /** A page of changes in the order the scans wrote them; `next` is always there. */
66
+ export interface DirectoryChanges {
67
+ items: FileChange[]
68
+ next: string
69
+ }
70
+
71
+ export function openDirectory(link: Link, name: string, options?: DirectoryOptions): Directory {
72
+ checkName(name, 'directory')
73
+ const open = BlobsDirectory.encode({
74
+ name,
75
+ path: options?.path,
76
+ every: options?.every === undefined ? undefined : ms(options.every),
77
+ ignore: options?.ignore ?? [],
78
+ growing: options?.growing === true,
79
+ })
80
+ return new Directory(link, name, open, [], options?.growing === true)
81
+ }
82
+
83
+ /**
84
+ * A directory, or a folder of it: `get`, `head`, `list` and `usage` are a set
85
+ * of files' calls with a set of files' answers, answered from the list the
86
+ * last scan wrote but a get's bytes, which come from the disk.
87
+ */
88
+ export class Directory {
89
+ readonly name: string
90
+ readonly #link: Link
91
+ readonly #via: Via
92
+ readonly #open: Uint8Array
93
+ readonly #folder: string[]
94
+ readonly #growing: boolean
95
+
96
+ /** Directories come from `store.directory`. */
97
+ constructor(link: Link, name: string, open: Uint8Array, folder: string[], growing: boolean) {
98
+ this.name = name
99
+ this.#link = link
100
+ this.#via = viaLink(link)
101
+ this.#open = open
102
+ this.#folder = folder
103
+ this.#growing = growing
104
+ }
105
+
106
+ /** The folder of these segments, one each, whose paths are relative to it. */
107
+ folder(...segments: (string | number | bigint)[]): Directory {
108
+ const folder = [...this.#folder, ...segments.map(segmentOf)]
109
+ return new Directory(this.#link, this.name, this.#open, folder, this.#growing)
110
+ }
111
+
112
+ /** Walks the folder and writes down what appeared, changed or went since the last scan. */
113
+ async scan(): Promise<Scan> {
114
+ const answer = await this.#call(
115
+ 'blobs.scan',
116
+ handle => BlobsFolder.encode({ handle, folder: this.#folder }),
117
+ 'write',
118
+ )
119
+ const scan = BlobsScanned.decode(answer)
120
+ return {
121
+ added: scan.added ?? 0,
122
+ changed: scan.changed ?? 0,
123
+ removed: scan.removed ?? 0,
124
+ skipped: scan.skipped ?? 0,
125
+ count: scan.count ?? 0,
126
+ size: scan.size ?? 0,
127
+ }
128
+ }
129
+
130
+ /** The changes past a cursor, a page's `next`, or from the first file the directory listed. */
131
+ async changes(options?: { after?: string | undefined }): Promise<DirectoryChanges> {
132
+ const answer = await this.#call(
133
+ 'blobs.changes',
134
+ handle => BlobsSince.encode({ handle, after: options?.after }),
135
+ 'read',
136
+ )
137
+ const page = BlobsChanges.decode(answer)
138
+ const items = (page.items ?? []).map(item => ({
139
+ path: item.path ?? '',
140
+ change: (item.change ?? 'added') as Change,
141
+ info: item.info === undefined ? undefined : infoOf(item.info),
142
+ }))
143
+ return { items, next: page.next ?? '0' }
144
+ }
145
+
146
+ /**
147
+ * The file as a scan listed it and as the disk holds it now; undefined when
148
+ * no scan listed it, or it is gone since. A read of a file that moved since
149
+ * fails ConflictError, but in a growing directory, which reads up to the size now.
150
+ */
151
+ async get(path: string): Promise<StoredFile | undefined> {
152
+ const answer = await this.#call(
153
+ 'blobs.get',
154
+ handle => BlobsAt.encode({ handle, folder: this.#folder, path }),
155
+ 'read',
156
+ )
157
+ const got = BlobsGot.view(answer)
158
+ if (got.info === undefined) {
159
+ return undefined
160
+ }
161
+ const info = infoOf(got.info)
162
+ const bytes = got.bytes ?? new Uint8Array()
163
+ // a growing file's bytes come from the disk as it is when they are read
164
+ const whole = !this.#growing && bytes.length === info.size ? bytes : undefined
165
+ const ifMatch = this.#growing ? undefined : info.etag
166
+ const handle = (connection: Connection) =>
167
+ handleOn(connection, methods['blobs.directory'], this.#open)
168
+ return new StoredFile(info, whole, (offset, length) =>
169
+ readStream(this.#link, handle, { folder: this.#folder, path, ifMatch, offset, length }),
170
+ )
171
+ }
172
+
173
+ /** What the last scan wrote down of the file; it reads no disk. */
174
+ async head(path: string): Promise<FileInfo | undefined> {
175
+ const answer = await this.#call(
176
+ 'blobs.head',
177
+ handle => BlobsAt.encode({ handle, folder: this.#folder, path }),
178
+ 'read',
179
+ )
180
+ const head = BlobsHead.decode(answer)
181
+ return head.info === undefined ? undefined : infoOf(head.info)
182
+ }
183
+
184
+ /** A page of the folder's files, in the byte order of their paths, as the last scan wrote them. */
185
+ async list(options?: FileListOptions): Promise<FilePage> {
186
+ const answer = await this.#call(
187
+ 'blobs.list',
188
+ handle =>
189
+ BlobsList.encode({
190
+ handle,
191
+ folder: this.#folder,
192
+ prefix: options?.prefix ?? '',
193
+ after: options?.after,
194
+ limit: options?.limit,
195
+ }),
196
+ 'read',
197
+ )
198
+ const page = BlobsPage.decode(answer)
199
+ return { files: (page.files ?? []).map(infoOf), next: page.next }
200
+ }
201
+
202
+ /** The files the last scan listed in the folder, and their bytes. */
203
+ async usage(): Promise<{ count: number; size: number }> {
204
+ const answer = await this.#call(
205
+ 'blobs.usage',
206
+ handle => BlobsFolder.encode({ handle, folder: this.#folder }),
207
+ 'read',
208
+ )
209
+ const usage = BlobsUsage.decode(answer)
210
+ return { count: usage.count ?? 0, size: usage.size ?? 0 }
211
+ }
212
+
213
+ /** Removes every file of the list, each told removed, and nothing on the disk. */
214
+ async forget(): Promise<void> {
215
+ await this.#call(
216
+ 'blobs.forget',
217
+ handle => BlobsFolder.encode({ handle, folder: this.#folder }),
218
+ 'write',
219
+ )
220
+ }
221
+
222
+ #call(
223
+ method: keyof typeof methods,
224
+ body: (handle: number) => Uint8Array,
225
+ idempotence: 'read' | 'write',
226
+ ): Promise<Uint8Array> {
227
+ return this.#via.call(
228
+ methods['blobs.directory'],
229
+ this.#open,
230
+ methods[method],
231
+ body,
232
+ idempotence,
233
+ )
234
+ }
235
+ }
package/src/index.ts CHANGED
@@ -61,6 +61,14 @@ export {
61
61
  } from './config.ts'
62
62
  export type { Durability, SelfMetrics } from './connection.ts'
63
63
  export { type ConsoleFormat, type ConsoleTime, secrets } from './console.ts'
64
+ export type {
65
+ Change,
66
+ Directory,
67
+ DirectoryChanges,
68
+ DirectoryOptions,
69
+ FileChange,
70
+ Scan,
71
+ } from './directory.ts'
64
72
  export * from './errors.ts'
65
73
  export type { Page } from './handles.ts'
66
74
  export type {
@@ -71,6 +79,7 @@ export type {
71
79
  Concurrency,
72
80
  Handler,
73
81
  Job,
82
+ JobCounts,
74
83
  JobOptions,
75
84
  JobPage,
76
85
  JobState,