@tinyshed/tinystore 0.8.0 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@tinyshed/tinystore",
3
- "version": "0.8.0",
3
+ "version": "0.9.0",
4
4
  "description": "TinyStore for Bun and Node: kv, jobs, blobs, SQL, records and metrics in one directory, served by a sidecar",
5
5
  "license": "Apache-2.0",
6
6
  "homepage": "https://tinyshed.org/tinystore",
@@ -31,12 +31,12 @@
31
31
  "README.md"
32
32
  ],
33
33
  "optionalDependencies": {
34
- "@tinyshed/tinystore-linux-x64": "0.8.0",
35
- "@tinyshed/tinystore-linux-arm64": "0.8.0",
36
- "@tinyshed/tinystore-linux-x64-musl": "0.8.0",
37
- "@tinyshed/tinystore-linux-arm64-musl": "0.8.0",
38
- "@tinyshed/tinystore-darwin-arm64": "0.8.0",
39
- "@tinyshed/tinystore-darwin-x64": "0.8.0",
40
- "@tinyshed/tinystore-win32-x64": "0.8.0"
34
+ "@tinyshed/tinystore-linux-x64": "0.9.0",
35
+ "@tinyshed/tinystore-linux-arm64": "0.9.0",
36
+ "@tinyshed/tinystore-linux-x64-musl": "0.9.0",
37
+ "@tinyshed/tinystore-linux-arm64-musl": "0.9.0",
38
+ "@tinyshed/tinystore-darwin-arm64": "0.9.0",
39
+ "@tinyshed/tinystore-darwin-x64": "0.9.0",
40
+ "@tinyshed/tinystore-win32-x64": "0.9.0"
41
41
  }
42
42
  }
package/src/backup.ts CHANGED
@@ -35,6 +35,8 @@ export interface Backup {
35
35
  * hold: they open with that key alone, kept apart
36
36
  */
37
37
  keyLeftOut: boolean
38
+ /** the databases opened with `backup: false`, by name, of which it holds nothing */
39
+ excluded: string[]
38
40
  }
39
41
 
40
42
  const nothing = new Uint8Array()
@@ -73,6 +75,7 @@ export async function backup(link: Link, path: string, options: BackupOptions):
73
75
  files: Number(backed.files ?? 0),
74
76
  bytes: Number(backed.bytes ?? 0),
75
77
  keyLeftOut: backed.keyLeftOut === true,
78
+ excluded: backed.excluded ?? [],
76
79
  }
77
80
  } catch (err) {
78
81
  await rm(partial, { force: true })
package/src/blobs.ts CHANGED
@@ -357,48 +357,8 @@ export class Files {
357
357
  offset?: number,
358
358
  length?: number,
359
359
  ): ReadableStream<Uint8Array> {
360
- let stream: Stream | undefined
361
- const open = () =>
362
- this.#link.run('read', async connection => {
363
- const read = BlobsRead.encode({
364
- handle: await this.#handle(connection),
365
- folder: this.#folder,
366
- path,
367
- ifMatch,
368
- offset,
369
- length,
370
- })
371
- const opened = await connection.session.open(methods['blobs.read'], read, true)
372
- return { opened, first: await opened.next() }
373
- })
374
- return new ReadableStream<Uint8Array>({
375
- async pull(controller) {
376
- try {
377
- if (stream === undefined) {
378
- // the RESPONSE's piece is no DATA, so it takes none of the credit
379
- const { opened, first } = await open()
380
- stream = opened
381
- enqueue(controller, first.body)
382
- if (first.end) {
383
- controller.close()
384
- }
385
- return
386
- }
387
- const event = await stream.next()
388
- if (event.end) {
389
- controller.close()
390
- return
391
- }
392
- stream.consumed(event.body.length)
393
- enqueue(controller, event.body)
394
- } catch (err) {
395
- controller.error(err)
396
- }
397
- },
398
- cancel() {
399
- stream?.cancel()
400
- },
401
- })
360
+ const at = { folder: this.#folder, path, ifMatch, offset, length }
361
+ return readStream(this.#link, connection => this.#handle(connection), at)
402
362
  }
403
363
 
404
364
  async #move(
@@ -594,9 +554,15 @@ export class FileBytes {
594
554
  // a copy, which a Buffer's slice under Node is not
595
555
  return new Uint8Array(this.#held)
596
556
  }
597
- const whole = new Uint8Array(this.size)
557
+ let whole = new Uint8Array(this.size)
598
558
  let at = 0
599
559
  for await (const piece of this.stream() as AsyncIterable<Uint8Array>) {
560
+ // a growing directory's file reads past the size its get gave
561
+ if (at + piece.length > whole.length) {
562
+ const grown = new Uint8Array(Math.max(at + piece.length, whole.length * 2))
563
+ grown.set(whole.subarray(0, at))
564
+ whole = grown
565
+ }
600
566
  whole.set(piece, at)
601
567
  at += piece.length
602
568
  }
@@ -666,6 +632,61 @@ export class StoredFile extends FileBytes implements FileInfo {
666
632
  }
667
633
  }
668
634
 
635
+ /** Where a read of a file's bytes stands, and the ETag it holds to; none holds to no version. */
636
+ export interface ReadAt {
637
+ folder: string[]
638
+ path: string
639
+ ifMatch: string | undefined
640
+ offset?: number | undefined
641
+ length?: number | undefined
642
+ }
643
+
644
+ /**
645
+ * A file's bytes as S3's GET reads them with a range and If-Match, on the
646
+ * handle `handle` opens on a connection; nothing is asked before the first read.
647
+ */
648
+ export function readStream(
649
+ link: Link,
650
+ handle: (connection: Connection) => Promise<number>,
651
+ at: ReadAt,
652
+ ): ReadableStream<Uint8Array> {
653
+ let stream: Stream | undefined
654
+ const open = () =>
655
+ link.run('read', async connection => {
656
+ const read = BlobsRead.encode({ ...at, handle: await handle(connection) })
657
+ const opened = await connection.session.open(methods['blobs.read'], read, true)
658
+ return { opened, first: await opened.next() }
659
+ })
660
+ return new ReadableStream<Uint8Array>({
661
+ async pull(controller) {
662
+ try {
663
+ if (stream === undefined) {
664
+ // the RESPONSE's piece is no DATA, so it takes none of the credit
665
+ const { opened, first } = await open()
666
+ stream = opened
667
+ enqueue(controller, first.body)
668
+ if (first.end) {
669
+ controller.close()
670
+ }
671
+ return
672
+ }
673
+ const event = await stream.next()
674
+ if (event.end) {
675
+ controller.close()
676
+ return
677
+ }
678
+ stream.consumed(event.body.length)
679
+ enqueue(controller, event.body)
680
+ } catch (err) {
681
+ controller.error(err)
682
+ }
683
+ },
684
+ cancel() {
685
+ stream?.cancel()
686
+ },
687
+ })
688
+ }
689
+
669
690
  /** A place in bytes as Blob.slice reads one: negative from the end, and within the size. */
670
691
  function placeOf(place: number | undefined, size: number, absent: number): number {
671
692
  if (place === undefined) {
@@ -859,7 +880,7 @@ function writtenOf(body: Uint8Array): FileInfo | undefined {
859
880
  return written.info === undefined ? undefined : infoOf(written.info)
860
881
  }
861
882
 
862
- function infoOf(info: ReturnType<typeof BlobsInfo.decode>): FileInfo {
883
+ export function infoOf(info: ReturnType<typeof BlobsInfo.decode>): FileInfo {
863
884
  return {
864
885
  path: info.path ?? '',
865
886
  size: info.size ?? 0,
@@ -872,7 +893,7 @@ function infoOf(info: ReturnType<typeof BlobsInfo.decode>): FileInfo {
872
893
  }
873
894
 
874
895
  /** A folder's segment as text, an integer its decimal spelling; the server refuses one that is not a segment. */
875
- function segmentOf(segment: string | number | bigint): string {
896
+ export function segmentOf(segment: string | number | bigint): string {
876
897
  if (typeof segment === 'number' && !Number.isSafeInteger(segment)) {
877
898
  throw new InvalidError(`the folder segment ${segment}: a segment is text or an integer`)
878
899
  }
@@ -0,0 +1,235 @@
1
+ // A directory, as docs/guides/blobs/directories.md has it: a folder of the
2
+ // store's machine under a name, which the store lists and reads and never
3
+ // writes. A scan writes down what appeared, changed or went; changes keep a
4
+ // copy in step from a cursor; a read comes from the disk, held to the version
5
+ // its get opened, or in a growing directory to none.
6
+
7
+ import {
8
+ type FileInfo,
9
+ type FileListOptions,
10
+ type FilePage,
11
+ infoOf,
12
+ readStream,
13
+ StoredFile,
14
+ segmentOf,
15
+ } from './blobs.ts'
16
+ import type { Connection, Link } from './connection.ts'
17
+ import { checkName, handleOn, type Via, viaLink } from './handles.ts'
18
+ import { type Duration, ms } from './time.ts'
19
+ import {
20
+ BlobsAt,
21
+ BlobsChanges,
22
+ BlobsDirectory,
23
+ BlobsFolder,
24
+ BlobsGot,
25
+ BlobsHead,
26
+ BlobsList,
27
+ BlobsPage,
28
+ BlobsScanned,
29
+ BlobsSince,
30
+ BlobsUsage,
31
+ methods,
32
+ } from './wire/protocol.ts'
33
+
34
+ export interface DirectoryOptions {
35
+ /** The folder on the store's machine; a server's own, by the name, when left out. */
36
+ path?: string | undefined
37
+ /** A scan on the store's background work, a period apart: minutes, not seconds. */
38
+ every?: Duration | undefined
39
+ /** Names a scan passes over, a file's or a folder's, `*` for any run of a name's characters. */
40
+ ignore?: string[] | undefined
41
+ /** For files written while they are read: a read holds to no version and goes up to the size now. */
42
+ growing?: boolean | undefined
43
+ }
44
+
45
+ /** What a scan found, and what the list holds once it ended. */
46
+ export interface Scan {
47
+ added: number
48
+ changed: number
49
+ removed: number
50
+ /** what it met and could not list: a link, a folder it cannot read, another disk */
51
+ skipped: number
52
+ count: number
53
+ size: number
54
+ }
55
+
56
+ export type Change = 'added' | 'changed' | 'removed'
57
+
58
+ /** A file's change since a cursor: its path in the directory, and what it carries now, none once removed. */
59
+ export interface FileChange {
60
+ path: string
61
+ change: Change
62
+ info: FileInfo | undefined
63
+ }
64
+
65
+ /** A page of changes in the order the scans wrote them; `next` is always there. */
66
+ export interface DirectoryChanges {
67
+ items: FileChange[]
68
+ next: string
69
+ }
70
+
71
+ export function openDirectory(link: Link, name: string, options?: DirectoryOptions): Directory {
72
+ checkName(name, 'directory')
73
+ const open = BlobsDirectory.encode({
74
+ name,
75
+ path: options?.path,
76
+ every: options?.every === undefined ? undefined : ms(options.every),
77
+ ignore: options?.ignore ?? [],
78
+ growing: options?.growing === true,
79
+ })
80
+ return new Directory(link, name, open, [], options?.growing === true)
81
+ }
82
+
83
+ /**
84
+ * A directory, or a folder of it: `get`, `head`, `list` and `usage` are a set
85
+ * of files' calls with a set of files' answers, answered from the list the
86
+ * last scan wrote but a get's bytes, which come from the disk.
87
+ */
88
+ export class Directory {
89
+ readonly name: string
90
+ readonly #link: Link
91
+ readonly #via: Via
92
+ readonly #open: Uint8Array
93
+ readonly #folder: string[]
94
+ readonly #growing: boolean
95
+
96
+ /** Directories come from `store.directory`. */
97
+ constructor(link: Link, name: string, open: Uint8Array, folder: string[], growing: boolean) {
98
+ this.name = name
99
+ this.#link = link
100
+ this.#via = viaLink(link)
101
+ this.#open = open
102
+ this.#folder = folder
103
+ this.#growing = growing
104
+ }
105
+
106
+ /** The folder of these segments, one each, whose paths are relative to it. */
107
+ folder(...segments: (string | number | bigint)[]): Directory {
108
+ const folder = [...this.#folder, ...segments.map(segmentOf)]
109
+ return new Directory(this.#link, this.name, this.#open, folder, this.#growing)
110
+ }
111
+
112
+ /** Walks the folder and writes down what appeared, changed or went since the last scan. */
113
+ async scan(): Promise<Scan> {
114
+ const answer = await this.#call(
115
+ 'blobs.scan',
116
+ handle => BlobsFolder.encode({ handle, folder: this.#folder }),
117
+ 'write',
118
+ )
119
+ const scan = BlobsScanned.decode(answer)
120
+ return {
121
+ added: scan.added ?? 0,
122
+ changed: scan.changed ?? 0,
123
+ removed: scan.removed ?? 0,
124
+ skipped: scan.skipped ?? 0,
125
+ count: scan.count ?? 0,
126
+ size: scan.size ?? 0,
127
+ }
128
+ }
129
+
130
+ /** The changes past a cursor, a page's `next`, or from the first file the directory listed. */
131
+ async changes(options?: { after?: string | undefined }): Promise<DirectoryChanges> {
132
+ const answer = await this.#call(
133
+ 'blobs.changes',
134
+ handle => BlobsSince.encode({ handle, after: options?.after }),
135
+ 'read',
136
+ )
137
+ const page = BlobsChanges.decode(answer)
138
+ const items = (page.items ?? []).map(item => ({
139
+ path: item.path ?? '',
140
+ change: (item.change ?? 'added') as Change,
141
+ info: item.info === undefined ? undefined : infoOf(item.info),
142
+ }))
143
+ return { items, next: page.next ?? '0' }
144
+ }
145
+
146
+ /**
147
+ * The file as a scan listed it and as the disk holds it now; undefined when
148
+ * no scan listed it, or it is gone since. A read of a file that moved since
149
+ * fails ConflictError, but in a growing directory, which reads up to the size now.
150
+ */
151
+ async get(path: string): Promise<StoredFile | undefined> {
152
+ const answer = await this.#call(
153
+ 'blobs.get',
154
+ handle => BlobsAt.encode({ handle, folder: this.#folder, path }),
155
+ 'read',
156
+ )
157
+ const got = BlobsGot.view(answer)
158
+ if (got.info === undefined) {
159
+ return undefined
160
+ }
161
+ const info = infoOf(got.info)
162
+ const bytes = got.bytes ?? new Uint8Array()
163
+ // a growing file's bytes come from the disk as it is when they are read
164
+ const whole = !this.#growing && bytes.length === info.size ? bytes : undefined
165
+ const ifMatch = this.#growing ? undefined : info.etag
166
+ const handle = (connection: Connection) =>
167
+ handleOn(connection, methods['blobs.directory'], this.#open)
168
+ return new StoredFile(info, whole, (offset, length) =>
169
+ readStream(this.#link, handle, { folder: this.#folder, path, ifMatch, offset, length }),
170
+ )
171
+ }
172
+
173
+ /** What the last scan wrote down of the file; it reads no disk. */
174
+ async head(path: string): Promise<FileInfo | undefined> {
175
+ const answer = await this.#call(
176
+ 'blobs.head',
177
+ handle => BlobsAt.encode({ handle, folder: this.#folder, path }),
178
+ 'read',
179
+ )
180
+ const head = BlobsHead.decode(answer)
181
+ return head.info === undefined ? undefined : infoOf(head.info)
182
+ }
183
+
184
+ /** A page of the folder's files, in the byte order of their paths, as the last scan wrote them. */
185
+ async list(options?: FileListOptions): Promise<FilePage> {
186
+ const answer = await this.#call(
187
+ 'blobs.list',
188
+ handle =>
189
+ BlobsList.encode({
190
+ handle,
191
+ folder: this.#folder,
192
+ prefix: options?.prefix ?? '',
193
+ after: options?.after,
194
+ limit: options?.limit,
195
+ }),
196
+ 'read',
197
+ )
198
+ const page = BlobsPage.decode(answer)
199
+ return { files: (page.files ?? []).map(infoOf), next: page.next }
200
+ }
201
+
202
+ /** The files the last scan listed in the folder, and their bytes. */
203
+ async usage(): Promise<{ count: number; size: number }> {
204
+ const answer = await this.#call(
205
+ 'blobs.usage',
206
+ handle => BlobsFolder.encode({ handle, folder: this.#folder }),
207
+ 'read',
208
+ )
209
+ const usage = BlobsUsage.decode(answer)
210
+ return { count: usage.count ?? 0, size: usage.size ?? 0 }
211
+ }
212
+
213
+ /** Removes every file of the list, each told removed, and nothing on the disk. */
214
+ async forget(): Promise<void> {
215
+ await this.#call(
216
+ 'blobs.forget',
217
+ handle => BlobsFolder.encode({ handle, folder: this.#folder }),
218
+ 'write',
219
+ )
220
+ }
221
+
222
+ #call(
223
+ method: keyof typeof methods,
224
+ body: (handle: number) => Uint8Array,
225
+ idempotence: 'read' | 'write',
226
+ ): Promise<Uint8Array> {
227
+ return this.#via.call(
228
+ methods['blobs.directory'],
229
+ this.#open,
230
+ methods[method],
231
+ body,
232
+ idempotence,
233
+ )
234
+ }
235
+ }
package/src/index.ts CHANGED
@@ -61,6 +61,14 @@ export {
61
61
  } from './config.ts'
62
62
  export type { Durability, SelfMetrics } from './connection.ts'
63
63
  export { type ConsoleFormat, type ConsoleTime, secrets } from './console.ts'
64
+ export type {
65
+ Change,
66
+ Directory,
67
+ DirectoryChanges,
68
+ DirectoryOptions,
69
+ FileChange,
70
+ Scan,
71
+ } from './directory.ts'
64
72
  export * from './errors.ts'
65
73
  export type { Page } from './handles.ts'
66
74
  export type {
@@ -71,6 +79,7 @@ export type {
71
79
  Concurrency,
72
80
  Handler,
73
81
  Job,
82
+ JobCounts,
74
83
  JobOptions,
75
84
  JobPage,
76
85
  JobState,