@tinyshed/tinystore 0.2.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/connection.ts CHANGED
@@ -293,6 +293,8 @@ export interface Started {
293
293
  durability?: Durability | undefined
294
294
  /** the file of the store's encryption key, when it is not the store's own */
295
295
  encryptionKeyFile?: string | undefined
296
+ /** the files of the store's older keys, which open what they sealed */
297
+ olderEncryptionKeys?: string[] | undefined
296
298
  /** the node the store is, when it is one */
297
299
  node?: NodeOptions | undefined
298
300
  /** measures the store into its own metrics, the machine too unless it says system: false */
@@ -319,8 +321,13 @@ function selfMetricsOf(given: SelfMetrics | undefined): { system: boolean } | un
319
321
  function storeFlags(started: Started, dir: string): string[] {
320
322
  const committing = started.durability === undefined ? [] : ['--durability', started.durability]
321
323
  const key = started.encryptionKeyFile
322
- const keyed =
324
+ const sealing =
323
325
  key === undefined ? committing : [...committing, '--encryption-key-file', resolve(key)]
326
+ const older = (started.olderEncryptionKeys ?? []).flatMap(file => [
327
+ '--older-encryption-key',
328
+ resolve(file),
329
+ ])
330
+ const keyed = [...sealing, ...older]
324
331
  const asked = selfMetricsOf(started.selfMetrics)
325
332
  const measured =
326
333
  asked === undefined
@@ -743,6 +750,7 @@ export function embedded(
743
750
  const options = StoreOptions.encode({
744
751
  durability: started.durability,
745
752
  encryptionKeyFile,
753
+ olderEncryptionKeys: (started.olderEncryptionKeys ?? []).map(file => resolve(file)),
746
754
  node: node !== undefined,
747
755
  from: node?.from ?? [],
748
756
  serve: node?.serve,
package/src/index.ts CHANGED
@@ -104,6 +104,7 @@ export type {
104
104
  export { type ConsoleOut, type Logger, type LoggerOptions, logger } from './logger.ts'
105
105
  export {
106
106
  type Aggregate,
107
+ type AggregatePage,
107
108
  type AggregateQuery,
108
109
  type Bucket as MetricsBucket,
109
110
  type Buckets,
@@ -135,6 +136,7 @@ export {
135
136
  type Series,
136
137
  type SeriesInput,
137
138
  type SeriesKind,
139
+ type SeriesPage,
138
140
  type Timer,
139
141
  type TimerOptions,
140
142
  type Timing,
package/src/kv.ts CHANGED
@@ -27,6 +27,7 @@ import {
27
27
  KvFound,
28
28
  KvList,
29
29
  KvPage,
30
+ KvResealed,
30
31
  KvWritten,
31
32
  type Method,
32
33
  methods,
@@ -239,6 +240,26 @@ export class Bucket<T> {
239
240
  )
240
241
  }
241
242
 
243
+ /**
244
+ * Seals again with the store's sealing key every value of this branch, and
245
+ * of the branches under it, that an older key sealed, and says how many.
246
+ * Once every value of the bucket is, the older key can leave the store's
247
+ * options. A value sealed with a key the store does not hold stops it with
248
+ * an `InvalidError` that names the key, and a reseal after the key is given
249
+ * goes on.
250
+ */
251
+ async reseal(): Promise<number> {
252
+ const under = underField(this.#under)
253
+ const answer = await this.#via.call(
254
+ methods['kv.bucket.open'],
255
+ this.#open,
256
+ methods['kv.reseal'],
257
+ handle => KvBranch.encode({ handle, under }),
258
+ 'write',
259
+ )
260
+ return KvResealed.decode(answer).count ?? 0
261
+ }
262
+
242
263
  /** One page of this branch's own keys, in the byte order of their text: `"10"` before `"9"`. */
243
264
  async list(options?: { limit?: number | undefined; after?: Key | undefined }): Promise<Page<T>> {
244
265
  const under = underField(this.#under)
package/src/metrics.ts CHANGED
@@ -92,6 +92,10 @@ export interface Query {
92
92
  from?: Time | undefined
93
93
  /** the range's end, excluded; now, a sample taken now in the range, unless given */
94
94
  to?: Time | undefined
95
+ /** a page of read or aggregate: this many samples or buckets at most, 100,000 at most; without it, the whole answer or a limit error */
96
+ limit?: number | undefined
97
+ /** the page after the one whose next this is, read with the same query and limit */
98
+ after?: string | undefined
95
99
  }
96
100
 
97
101
  /** A series to write: its samples as [time, value] pairs, or as times and values columns. */
@@ -125,6 +129,19 @@ export interface Series {
125
129
  times: Float64Array
126
130
  /** the values, whose bytes keep each one's bits: -0 and a NaN's payload survive */
127
131
  values: Float64Array
132
+ /** a page cut the series here: the page its next reads begins with the rest of it */
133
+ more: boolean
134
+ }
135
+
136
+ /**
137
+ * One page of a read: its series, and next while there is more. Walking it
138
+ * with for await gives its series, then those of each page after it, one
139
+ * page held at a time. Without limit it is the whole answer.
140
+ */
141
+ export interface SeriesPage extends AsyncIterable<Series> {
142
+ series: Series[]
143
+ /** where the page after it begins; absent on the last */
144
+ next?: string
128
145
  }
129
146
 
130
147
  /** p and a rank: 'p95', 'p99.9'. The ranks offered are those asked most; any other is written the same way. */
@@ -209,6 +226,18 @@ export interface Aggregate<O extends Op = Op> {
209
226
  labels: Labels
210
227
  /** a bucket without samples is left out */
211
228
  buckets: Bucket<O>[]
229
+ /** a page cut the group here: the page its next reads begins with the rest of its buckets */
230
+ more: boolean
231
+ }
232
+
233
+ /**
234
+ * One page of an aggregate: its groups, and next while there is more,
235
+ * walked as a SeriesPage is. A page ends between two whole buckets.
236
+ */
237
+ export interface AggregatePage<O extends Op = Op> extends AsyncIterable<Aggregate<O>> {
238
+ groups: Aggregate<O>[]
239
+ /** where the page after it begins; absent on the last */
240
+ next?: string
212
241
  }
213
242
 
214
243
  /** A series' newest sample in the range. */
@@ -348,25 +377,47 @@ export class Metrics {
348
377
  })
349
378
  }
350
379
 
351
- /** The samples of each series the query finds, in its range: whole, or a limit error naming its bound. */
352
- async read(query: Query): Promise<Series[]> {
353
- const parts = await this.#download('metrics.read', MetricsQuery.encode(queryOf(query)))
380
+ /**
381
+ * The samples of each series the query finds, in its range: without limit
382
+ * the whole answer, or a limit error naming its bound; with limit a page,
383
+ * which may hold fewer samples than limit and still have next.
384
+ */
385
+ async read(query: Query): Promise<SeriesPage> {
386
+ const parts = (await this.#download('metrics.read', MetricsQuery.encode(queryOf(query)))).map(
387
+ part => MetricsRead.decode(part),
388
+ )
354
389
  const series: Series[] = []
355
390
  let more = false
356
- for (const piece of parts.flatMap(part => MetricsRead.decode(part).series ?? [])) {
391
+ for (const piece of parts.flatMap(part => part.series ?? [])) {
357
392
  const times = timesOf(piece.times ?? new Uint8Array(0))
358
393
  const values = valuesOf(piece.values ?? new Uint8Array(0))
359
394
  const last = series.at(-1)
360
395
  if (more && last !== undefined) {
361
396
  last.times = joined(last.times, times)
362
397
  last.values = joined(last.values, values)
398
+ last.more = piece.more === true
363
399
  } else {
364
400
  const kind = (piece.kind ?? 'gauge') as SeriesKind
365
- series.push({ name: piece.name ?? '', kind, labels: piece.labels ?? {}, times, values })
401
+ const labels = piece.labels ?? {}
402
+ series.push({
403
+ name: piece.name ?? '',
404
+ kind,
405
+ labels,
406
+ times,
407
+ values,
408
+ more: piece.more === true,
409
+ })
366
410
  }
367
411
  more = piece.more === true
368
412
  }
369
- return series
413
+ const next = parts.at(-1)?.next
414
+ const page: { series: Series[]; next?: string } =
415
+ next === undefined ? { series } : { series, next }
416
+ return walked(
417
+ page,
418
+ at => at.series,
419
+ after => this.read({ ...query, after }),
420
+ )
370
421
  }
371
422
 
372
423
  /** The newest sample of each series the query finds, when it has one in the range. */
@@ -386,24 +437,37 @@ export class Metrics {
386
437
  * The buckets of each group of the series the query finds, each one exact
387
438
  * value of op, rounded once; a bucket of count … last has its value.
388
439
  */
389
- async aggregate<O extends Op>(query: AggregateQuery<O>): Promise<Aggregate<O>[]> {
390
- const parts = await this.#download(
391
- 'metrics.aggregate',
392
- MetricsAggregateAsked.encode(aggregateOf(query)),
440
+ async aggregate<O extends Op>(query: AggregateQuery<O>): Promise<AggregatePage<O>> {
441
+ const asked = MetricsAggregateAsked.encode(aggregateOf(query))
442
+ const parts = (await this.#download('metrics.aggregate', asked)).map(part =>
443
+ MetricsAggregates.decode(part),
393
444
  )
394
445
  const groups: Aggregate<O>[] = []
395
446
  let more = false
396
- for (const piece of parts.flatMap(part => MetricsAggregates.decode(part).groups ?? [])) {
447
+ for (const piece of parts.flatMap(part => part.groups ?? [])) {
397
448
  const buckets = (piece.buckets ?? []).map(bucket => bucketOf(bucket) as Bucket<O>)
398
449
  const last = groups.at(-1)
399
450
  if (more && last !== undefined) {
400
451
  last.buckets.push(...buckets)
452
+ last.more = piece.more === true
401
453
  } else {
402
- groups.push({ name: piece.name ?? '', labels: piece.labels ?? {}, buckets })
454
+ groups.push({
455
+ name: piece.name ?? '',
456
+ labels: piece.labels ?? {},
457
+ buckets,
458
+ more: piece.more === true,
459
+ })
403
460
  }
404
461
  more = piece.more === true
405
462
  }
406
- return groups
463
+ const next = parts.at(-1)?.next
464
+ const page: { groups: Aggregate<O>[]; next?: string } =
465
+ next === undefined ? { groups } : { groups, next }
466
+ return walked(
467
+ page,
468
+ at => at.groups,
469
+ after => this.aggregate({ ...query, after }),
470
+ )
407
471
  }
408
472
 
409
473
  /** What a read, or an aggregate when given its op, would spend of each bound; it decodes no sample. */
@@ -591,6 +655,28 @@ export class Metrics {
591
655
  }
592
656
  }
593
657
 
658
+ /**
659
+ * A page that for await walks: its items, then those of each page after it,
660
+ * the next page read only once the walk reaches the end of the one it holds.
661
+ */
662
+ function walked<P extends { next?: string }, T>(
663
+ page: P,
664
+ items: (page: P) => readonly T[],
665
+ readOn: (after: string) => Promise<P>,
666
+ ): P & AsyncIterable<T> {
667
+ return Object.assign(page, {
668
+ async *[Symbol.asyncIterator](): AsyncGenerator<T> {
669
+ for (let at = page; ; ) {
670
+ yield* items(at)
671
+ if (at.next === undefined) {
672
+ return
673
+ }
674
+ at = await readOn(at.next)
675
+ }
676
+ },
677
+ })
678
+ }
679
+
594
680
  /** A query as the wire carries it; a condition given as undefined is refused, never left out. */
595
681
  function queryOf(query: Query): Parameters<typeof MetricsQuery.encode>[0] {
596
682
  const where: Record<string, { values?: string[]; not?: boolean; prefix?: string }> = {}
@@ -606,6 +692,8 @@ function queryOf(query: Query): Parameters<typeof MetricsQuery.encode>[0] {
606
692
  since: query.since === undefined ? undefined : ms(query.since),
607
693
  from: query.from === undefined ? undefined : unixMs(query.from),
608
694
  to: query.to === undefined ? undefined : unixMs(query.to),
695
+ limit: query.limit,
696
+ after: query.after,
609
697
  }
610
698
  }
611
699
 
package/src/sql.ts CHANGED
@@ -8,6 +8,7 @@ import { AsyncLocalStorage } from 'node:async_hooks'
8
8
  import { readdir, readFile } from 'node:fs/promises'
9
9
  import { join } from 'node:path'
10
10
 
11
+ import { Files, type FilesOptions, filesOpen, filesParts } from './blobs.ts'
11
12
  import { type Channel, channelParts, type TxChannel, txChannel } from './channels.ts'
12
13
  import type { Connection, Durability, Idempotence, Link } from './connection.ts'
13
14
  import { checkDurability, download } from './connection.ts'
@@ -774,6 +775,22 @@ export class Database implements Runner, Home {
774
775
  return openQueue(this.#link, this.#workers, name, options, this)
775
776
  }
776
777
 
778
+ /**
779
+ * A set of files kept in this database's file, as `store.files` keeps one
780
+ * in blobs.db: the same calls, and its writes commit with the rows in
781
+ * `db.tx`, through `tx.with(files)`. A file's bytes past 16 KiB lie in the
782
+ * folder beside the database's file.
783
+ *
784
+ * const packages = db.files('packages', { maxFileSize: '10MiB' })
785
+ */
786
+ files(name: string, options?: FilesOptions): Files {
787
+ const bytes = filesOpen(name, options)
788
+ const open = this.keep(methods['blobs.open'], bytes, async connection =>
789
+ filesOpen(name, options, await this.handle(connection)),
790
+ )
791
+ return new Files(this.#link, name, open, [], this)
792
+ }
793
+
777
794
  /**
778
795
  * An index of a table of this database, kept in its file: its rows found
779
796
  * again by their words, vectors or both, each hit's value its row as
@@ -1059,9 +1076,10 @@ export class SqlTx implements Runner {
1059
1076
  with<T>(bucket: Bucket<T>): Bucket<T>
1060
1077
  with<T>(queue: Queue<T>): TxQueue<T>
1061
1078
  with<T>(channel: Channel<T>): TxChannel<T>
1079
+ with(files: Files): Files
1062
1080
  with(
1063
- handle: Bucket<unknown> | Queue<unknown> | Channel<unknown>,
1064
- ): Bucket<unknown> | TxQueue<unknown> | TxChannel<unknown> {
1081
+ handle: Bucket<unknown> | Queue<unknown> | Channel<unknown> | Files,
1082
+ ): Bucket<unknown> | TxQueue<unknown> | TxChannel<unknown> | Files {
1065
1083
  const channel = channelParts(handle)
1066
1084
  if (channel !== undefined) {
1067
1085
  // a message published here is handed over once the transaction commits, at the writer
@@ -1080,8 +1098,13 @@ export class SqlTx implements Runner {
1080
1098
  this.#takes(queue.source.home, queue.source.open, `jobs queue ${queue.name}`)
1081
1099
  return queueVia(queue, this.#via)
1082
1100
  }
1101
+ const files = filesParts(handle)
1102
+ if (files !== undefined) {
1103
+ this.#takes(files.home, files.open, `files ${files.name}`)
1104
+ return files.inside(this.#via)
1105
+ }
1083
1106
  throw new InvalidError(
1084
- `${this.describe}: a transaction takes in a bucket or a queue opened from its database`,
1107
+ `${this.describe}: a transaction takes in a bucket, a queue or files opened from its database`,
1085
1108
  )
1086
1109
  }
1087
1110
 
package/src/store.ts CHANGED
@@ -142,6 +142,14 @@ export interface OpenOptions {
142
142
  * running keeps the key it was started with.
143
143
  */
144
144
  encryptionKeyFile?: string
145
+ /**
146
+ * The files of the store's older keys, which open the values they sealed
147
+ * and seal nothing new: a key being rotated out, until `reseal` has sealed
148
+ * every value again, or one found after a restore that left it out. Each
149
+ * file must be there. A sidecar found running keeps the keys it was
150
+ * started with.
151
+ */
152
+ olderEncryptionKeys?: string[]
145
153
  /**
146
154
  * Makes the store a node: it keeps a log of each file it writes, serves
147
155
  * its files to other nodes, and copies the files of the node it takes
@@ -573,6 +581,7 @@ export async function openWith(
573
581
  const started = {
574
582
  durability,
575
583
  encryptionKeyFile: options.encryptionKeyFile,
584
+ olderEncryptionKeys: options.olderEncryptionKeys,
576
585
  node: options.node,
577
586
  selfMetrics: options.selfMetrics,
578
587
  maxSeries: options.maxSeries,