@tinyshed/tinystore 0.2.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/blobs.d.ts +19 -2
- package/dist/blobs.js +58 -9
- package/dist/connection.d.ts +2 -0
- package/dist/connection.js +7 -1
- package/dist/index.d.ts +1 -1
- package/dist/kv.d.ts +9 -0
- package/dist/kv.js +14 -1
- package/dist/metrics.d.ts +34 -3
- package/dist/metrics.js +52 -9
- package/dist/sql.d.ts +11 -0
- package/dist/sql.js +20 -1
- package/dist/store.d.ts +8 -0
- package/dist/store.js +1 -0
- package/dist/wire/protocol.d.ts +31 -3
- package/dist/wire/protocol.js +140 -6
- package/package.json +8 -8
- package/src/blobs.ts +88 -11
- package/src/connection.ts +9 -1
- package/src/index.ts +2 -0
- package/src/kv.ts +21 -0
- package/src/metrics.ts +101 -13
- package/src/sql.ts +26 -3
- package/src/store.ts +9 -0
- package/src/wire/protocol.ts +145 -6
package/src/connection.ts
CHANGED
|
@@ -293,6 +293,8 @@ export interface Started {
|
|
|
293
293
|
durability?: Durability | undefined
|
|
294
294
|
/** the file of the store's encryption key, when it is not the store's own */
|
|
295
295
|
encryptionKeyFile?: string | undefined
|
|
296
|
+
/** the files of the store's older keys, which open what they sealed */
|
|
297
|
+
olderEncryptionKeys?: string[] | undefined
|
|
296
298
|
/** the node the store is, when it is one */
|
|
297
299
|
node?: NodeOptions | undefined
|
|
298
300
|
/** measures the store into its own metrics, the machine too unless it says system: false */
|
|
@@ -319,8 +321,13 @@ function selfMetricsOf(given: SelfMetrics | undefined): { system: boolean } | un
|
|
|
319
321
|
function storeFlags(started: Started, dir: string): string[] {
|
|
320
322
|
const committing = started.durability === undefined ? [] : ['--durability', started.durability]
|
|
321
323
|
const key = started.encryptionKeyFile
|
|
322
|
-
const
|
|
324
|
+
const sealing =
|
|
323
325
|
key === undefined ? committing : [...committing, '--encryption-key-file', resolve(key)]
|
|
326
|
+
const older = (started.olderEncryptionKeys ?? []).flatMap(file => [
|
|
327
|
+
'--older-encryption-key',
|
|
328
|
+
resolve(file),
|
|
329
|
+
])
|
|
330
|
+
const keyed = [...sealing, ...older]
|
|
324
331
|
const asked = selfMetricsOf(started.selfMetrics)
|
|
325
332
|
const measured =
|
|
326
333
|
asked === undefined
|
|
@@ -743,6 +750,7 @@ export function embedded(
|
|
|
743
750
|
const options = StoreOptions.encode({
|
|
744
751
|
durability: started.durability,
|
|
745
752
|
encryptionKeyFile,
|
|
753
|
+
olderEncryptionKeys: (started.olderEncryptionKeys ?? []).map(file => resolve(file)),
|
|
746
754
|
node: node !== undefined,
|
|
747
755
|
from: node?.from ?? [],
|
|
748
756
|
serve: node?.serve,
|
package/src/index.ts
CHANGED
|
@@ -104,6 +104,7 @@ export type {
|
|
|
104
104
|
export { type ConsoleOut, type Logger, type LoggerOptions, logger } from './logger.ts'
|
|
105
105
|
export {
|
|
106
106
|
type Aggregate,
|
|
107
|
+
type AggregatePage,
|
|
107
108
|
type AggregateQuery,
|
|
108
109
|
type Bucket as MetricsBucket,
|
|
109
110
|
type Buckets,
|
|
@@ -135,6 +136,7 @@ export {
|
|
|
135
136
|
type Series,
|
|
136
137
|
type SeriesInput,
|
|
137
138
|
type SeriesKind,
|
|
139
|
+
type SeriesPage,
|
|
138
140
|
type Timer,
|
|
139
141
|
type TimerOptions,
|
|
140
142
|
type Timing,
|
package/src/kv.ts
CHANGED
|
@@ -27,6 +27,7 @@ import {
|
|
|
27
27
|
KvFound,
|
|
28
28
|
KvList,
|
|
29
29
|
KvPage,
|
|
30
|
+
KvResealed,
|
|
30
31
|
KvWritten,
|
|
31
32
|
type Method,
|
|
32
33
|
methods,
|
|
@@ -239,6 +240,26 @@ export class Bucket<T> {
|
|
|
239
240
|
)
|
|
240
241
|
}
|
|
241
242
|
|
|
243
|
+
/**
|
|
244
|
+
* Seals again with the store's sealing key every value of this branch, and
|
|
245
|
+
* of the branches under it, that an older key sealed, and says how many.
|
|
246
|
+
* Once every value of the bucket is, the older key can leave the store's
|
|
247
|
+
* options. A value sealed with a key the store does not hold stops it with
|
|
248
|
+
* an `InvalidError` that names the key, and a reseal after the key is given
|
|
249
|
+
* goes on.
|
|
250
|
+
*/
|
|
251
|
+
async reseal(): Promise<number> {
|
|
252
|
+
const under = underField(this.#under)
|
|
253
|
+
const answer = await this.#via.call(
|
|
254
|
+
methods['kv.bucket.open'],
|
|
255
|
+
this.#open,
|
|
256
|
+
methods['kv.reseal'],
|
|
257
|
+
handle => KvBranch.encode({ handle, under }),
|
|
258
|
+
'write',
|
|
259
|
+
)
|
|
260
|
+
return KvResealed.decode(answer).count ?? 0
|
|
261
|
+
}
|
|
262
|
+
|
|
242
263
|
/** One page of this branch's own keys, in the byte order of their text: `"10"` before `"9"`. */
|
|
243
264
|
async list(options?: { limit?: number | undefined; after?: Key | undefined }): Promise<Page<T>> {
|
|
244
265
|
const under = underField(this.#under)
|
package/src/metrics.ts
CHANGED
|
@@ -92,6 +92,10 @@ export interface Query {
|
|
|
92
92
|
from?: Time | undefined
|
|
93
93
|
/** the range's end, excluded; now, a sample taken now in the range, unless given */
|
|
94
94
|
to?: Time | undefined
|
|
95
|
+
/** a page of read or aggregate: this many samples or buckets at most, 100,000 at most; without it, the whole answer or a limit error */
|
|
96
|
+
limit?: number | undefined
|
|
97
|
+
/** the page after the one whose next this is, read with the same query and limit */
|
|
98
|
+
after?: string | undefined
|
|
95
99
|
}
|
|
96
100
|
|
|
97
101
|
/** A series to write: its samples as [time, value] pairs, or as times and values columns. */
|
|
@@ -125,6 +129,19 @@ export interface Series {
|
|
|
125
129
|
times: Float64Array
|
|
126
130
|
/** the values, whose bytes keep each one's bits: -0 and a NaN's payload survive */
|
|
127
131
|
values: Float64Array
|
|
132
|
+
/** a page cut the series here: the page its next reads begins with the rest of it */
|
|
133
|
+
more: boolean
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/**
|
|
137
|
+
* One page of a read: its series, and next while there is more. Walking it
|
|
138
|
+
* with for await gives its series, then those of each page after it, one
|
|
139
|
+
* page held at a time. Without limit it is the whole answer.
|
|
140
|
+
*/
|
|
141
|
+
export interface SeriesPage extends AsyncIterable<Series> {
|
|
142
|
+
series: Series[]
|
|
143
|
+
/** where the page after it begins; absent on the last */
|
|
144
|
+
next?: string
|
|
128
145
|
}
|
|
129
146
|
|
|
130
147
|
/** p and a rank: 'p95', 'p99.9'. The ranks offered are those asked most; any other is written the same way. */
|
|
@@ -209,6 +226,18 @@ export interface Aggregate<O extends Op = Op> {
|
|
|
209
226
|
labels: Labels
|
|
210
227
|
/** a bucket without samples is left out */
|
|
211
228
|
buckets: Bucket<O>[]
|
|
229
|
+
/** a page cut the group here: the page its next reads begins with the rest of its buckets */
|
|
230
|
+
more: boolean
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
/**
|
|
234
|
+
* One page of an aggregate: its groups, and next while there is more,
|
|
235
|
+
* walked as a SeriesPage is. A page ends between two whole buckets.
|
|
236
|
+
*/
|
|
237
|
+
export interface AggregatePage<O extends Op = Op> extends AsyncIterable<Aggregate<O>> {
|
|
238
|
+
groups: Aggregate<O>[]
|
|
239
|
+
/** where the page after it begins; absent on the last */
|
|
240
|
+
next?: string
|
|
212
241
|
}
|
|
213
242
|
|
|
214
243
|
/** A series' newest sample in the range. */
|
|
@@ -348,25 +377,47 @@ export class Metrics {
|
|
|
348
377
|
})
|
|
349
378
|
}
|
|
350
379
|
|
|
351
|
-
/**
|
|
352
|
-
|
|
353
|
-
|
|
380
|
+
/**
|
|
381
|
+
* The samples of each series the query finds, in its range: without limit
|
|
382
|
+
* the whole answer, or a limit error naming its bound; with limit a page,
|
|
383
|
+
* which may hold fewer samples than limit and still have next.
|
|
384
|
+
*/
|
|
385
|
+
async read(query: Query): Promise<SeriesPage> {
|
|
386
|
+
const parts = (await this.#download('metrics.read', MetricsQuery.encode(queryOf(query)))).map(
|
|
387
|
+
part => MetricsRead.decode(part),
|
|
388
|
+
)
|
|
354
389
|
const series: Series[] = []
|
|
355
390
|
let more = false
|
|
356
|
-
for (const piece of parts.flatMap(part =>
|
|
391
|
+
for (const piece of parts.flatMap(part => part.series ?? [])) {
|
|
357
392
|
const times = timesOf(piece.times ?? new Uint8Array(0))
|
|
358
393
|
const values = valuesOf(piece.values ?? new Uint8Array(0))
|
|
359
394
|
const last = series.at(-1)
|
|
360
395
|
if (more && last !== undefined) {
|
|
361
396
|
last.times = joined(last.times, times)
|
|
362
397
|
last.values = joined(last.values, values)
|
|
398
|
+
last.more = piece.more === true
|
|
363
399
|
} else {
|
|
364
400
|
const kind = (piece.kind ?? 'gauge') as SeriesKind
|
|
365
|
-
|
|
401
|
+
const labels = piece.labels ?? {}
|
|
402
|
+
series.push({
|
|
403
|
+
name: piece.name ?? '',
|
|
404
|
+
kind,
|
|
405
|
+
labels,
|
|
406
|
+
times,
|
|
407
|
+
values,
|
|
408
|
+
more: piece.more === true,
|
|
409
|
+
})
|
|
366
410
|
}
|
|
367
411
|
more = piece.more === true
|
|
368
412
|
}
|
|
369
|
-
|
|
413
|
+
const next = parts.at(-1)?.next
|
|
414
|
+
const page: { series: Series[]; next?: string } =
|
|
415
|
+
next === undefined ? { series } : { series, next }
|
|
416
|
+
return walked(
|
|
417
|
+
page,
|
|
418
|
+
at => at.series,
|
|
419
|
+
after => this.read({ ...query, after }),
|
|
420
|
+
)
|
|
370
421
|
}
|
|
371
422
|
|
|
372
423
|
/** The newest sample of each series the query finds, when it has one in the range. */
|
|
@@ -386,24 +437,37 @@ export class Metrics {
|
|
|
386
437
|
* The buckets of each group of the series the query finds, each one exact
|
|
387
438
|
* value of op, rounded once; a bucket of count … last has its value.
|
|
388
439
|
*/
|
|
389
|
-
async aggregate<O extends Op>(query: AggregateQuery<O>): Promise<
|
|
390
|
-
const
|
|
391
|
-
|
|
392
|
-
|
|
440
|
+
async aggregate<O extends Op>(query: AggregateQuery<O>): Promise<AggregatePage<O>> {
|
|
441
|
+
const asked = MetricsAggregateAsked.encode(aggregateOf(query))
|
|
442
|
+
const parts = (await this.#download('metrics.aggregate', asked)).map(part =>
|
|
443
|
+
MetricsAggregates.decode(part),
|
|
393
444
|
)
|
|
394
445
|
const groups: Aggregate<O>[] = []
|
|
395
446
|
let more = false
|
|
396
|
-
for (const piece of parts.flatMap(part =>
|
|
447
|
+
for (const piece of parts.flatMap(part => part.groups ?? [])) {
|
|
397
448
|
const buckets = (piece.buckets ?? []).map(bucket => bucketOf(bucket) as Bucket<O>)
|
|
398
449
|
const last = groups.at(-1)
|
|
399
450
|
if (more && last !== undefined) {
|
|
400
451
|
last.buckets.push(...buckets)
|
|
452
|
+
last.more = piece.more === true
|
|
401
453
|
} else {
|
|
402
|
-
groups.push({
|
|
454
|
+
groups.push({
|
|
455
|
+
name: piece.name ?? '',
|
|
456
|
+
labels: piece.labels ?? {},
|
|
457
|
+
buckets,
|
|
458
|
+
more: piece.more === true,
|
|
459
|
+
})
|
|
403
460
|
}
|
|
404
461
|
more = piece.more === true
|
|
405
462
|
}
|
|
406
|
-
|
|
463
|
+
const next = parts.at(-1)?.next
|
|
464
|
+
const page: { groups: Aggregate<O>[]; next?: string } =
|
|
465
|
+
next === undefined ? { groups } : { groups, next }
|
|
466
|
+
return walked(
|
|
467
|
+
page,
|
|
468
|
+
at => at.groups,
|
|
469
|
+
after => this.aggregate({ ...query, after }),
|
|
470
|
+
)
|
|
407
471
|
}
|
|
408
472
|
|
|
409
473
|
/** What a read, or an aggregate when given its op, would spend of each bound; it decodes no sample. */
|
|
@@ -591,6 +655,28 @@ export class Metrics {
|
|
|
591
655
|
}
|
|
592
656
|
}
|
|
593
657
|
|
|
658
|
+
/**
|
|
659
|
+
* A page that for await walks: its items, then those of each page after it,
|
|
660
|
+
* the next page read only once the walk reaches the end of the one it holds.
|
|
661
|
+
*/
|
|
662
|
+
function walked<P extends { next?: string }, T>(
|
|
663
|
+
page: P,
|
|
664
|
+
items: (page: P) => readonly T[],
|
|
665
|
+
readOn: (after: string) => Promise<P>,
|
|
666
|
+
): P & AsyncIterable<T> {
|
|
667
|
+
return Object.assign(page, {
|
|
668
|
+
async *[Symbol.asyncIterator](): AsyncGenerator<T> {
|
|
669
|
+
for (let at = page; ; ) {
|
|
670
|
+
yield* items(at)
|
|
671
|
+
if (at.next === undefined) {
|
|
672
|
+
return
|
|
673
|
+
}
|
|
674
|
+
at = await readOn(at.next)
|
|
675
|
+
}
|
|
676
|
+
},
|
|
677
|
+
})
|
|
678
|
+
}
|
|
679
|
+
|
|
594
680
|
/** A query as the wire carries it; a condition given as undefined is refused, never left out. */
|
|
595
681
|
function queryOf(query: Query): Parameters<typeof MetricsQuery.encode>[0] {
|
|
596
682
|
const where: Record<string, { values?: string[]; not?: boolean; prefix?: string }> = {}
|
|
@@ -606,6 +692,8 @@ function queryOf(query: Query): Parameters<typeof MetricsQuery.encode>[0] {
|
|
|
606
692
|
since: query.since === undefined ? undefined : ms(query.since),
|
|
607
693
|
from: query.from === undefined ? undefined : unixMs(query.from),
|
|
608
694
|
to: query.to === undefined ? undefined : unixMs(query.to),
|
|
695
|
+
limit: query.limit,
|
|
696
|
+
after: query.after,
|
|
609
697
|
}
|
|
610
698
|
}
|
|
611
699
|
|
package/src/sql.ts
CHANGED
|
@@ -8,6 +8,7 @@ import { AsyncLocalStorage } from 'node:async_hooks'
|
|
|
8
8
|
import { readdir, readFile } from 'node:fs/promises'
|
|
9
9
|
import { join } from 'node:path'
|
|
10
10
|
|
|
11
|
+
import { Files, type FilesOptions, filesOpen, filesParts } from './blobs.ts'
|
|
11
12
|
import { type Channel, channelParts, type TxChannel, txChannel } from './channels.ts'
|
|
12
13
|
import type { Connection, Durability, Idempotence, Link } from './connection.ts'
|
|
13
14
|
import { checkDurability, download } from './connection.ts'
|
|
@@ -774,6 +775,22 @@ export class Database implements Runner, Home {
|
|
|
774
775
|
return openQueue(this.#link, this.#workers, name, options, this)
|
|
775
776
|
}
|
|
776
777
|
|
|
778
|
+
/**
|
|
779
|
+
* A set of files kept in this database's file, as `store.files` keeps one
|
|
780
|
+
* in blobs.db: the same calls, and its writes commit with the rows in
|
|
781
|
+
* `db.tx`, through `tx.with(files)`. A file's bytes past 16 KiB lie in the
|
|
782
|
+
* folder beside the database's file.
|
|
783
|
+
*
|
|
784
|
+
* const packages = db.files('packages', { maxFileSize: '10MiB' })
|
|
785
|
+
*/
|
|
786
|
+
files(name: string, options?: FilesOptions): Files {
|
|
787
|
+
const bytes = filesOpen(name, options)
|
|
788
|
+
const open = this.keep(methods['blobs.open'], bytes, async connection =>
|
|
789
|
+
filesOpen(name, options, await this.handle(connection)),
|
|
790
|
+
)
|
|
791
|
+
return new Files(this.#link, name, open, [], this)
|
|
792
|
+
}
|
|
793
|
+
|
|
777
794
|
/**
|
|
778
795
|
* An index of a table of this database, kept in its file: its rows found
|
|
779
796
|
* again by their words, vectors or both, each hit's value its row as
|
|
@@ -1059,9 +1076,10 @@ export class SqlTx implements Runner {
|
|
|
1059
1076
|
with<T>(bucket: Bucket<T>): Bucket<T>
|
|
1060
1077
|
with<T>(queue: Queue<T>): TxQueue<T>
|
|
1061
1078
|
with<T>(channel: Channel<T>): TxChannel<T>
|
|
1079
|
+
with(files: Files): Files
|
|
1062
1080
|
with(
|
|
1063
|
-
handle: Bucket<unknown> | Queue<unknown> | Channel<unknown
|
|
1064
|
-
): Bucket<unknown> | TxQueue<unknown> | TxChannel<unknown> {
|
|
1081
|
+
handle: Bucket<unknown> | Queue<unknown> | Channel<unknown> | Files,
|
|
1082
|
+
): Bucket<unknown> | TxQueue<unknown> | TxChannel<unknown> | Files {
|
|
1065
1083
|
const channel = channelParts(handle)
|
|
1066
1084
|
if (channel !== undefined) {
|
|
1067
1085
|
// a message published here is handed over once the transaction commits, at the writer
|
|
@@ -1080,8 +1098,13 @@ export class SqlTx implements Runner {
|
|
|
1080
1098
|
this.#takes(queue.source.home, queue.source.open, `jobs queue ${queue.name}`)
|
|
1081
1099
|
return queueVia(queue, this.#via)
|
|
1082
1100
|
}
|
|
1101
|
+
const files = filesParts(handle)
|
|
1102
|
+
if (files !== undefined) {
|
|
1103
|
+
this.#takes(files.home, files.open, `files ${files.name}`)
|
|
1104
|
+
return files.inside(this.#via)
|
|
1105
|
+
}
|
|
1083
1106
|
throw new InvalidError(
|
|
1084
|
-
`${this.describe}: a transaction takes in a bucket
|
|
1107
|
+
`${this.describe}: a transaction takes in a bucket, a queue or files opened from its database`,
|
|
1085
1108
|
)
|
|
1086
1109
|
}
|
|
1087
1110
|
|
package/src/store.ts
CHANGED
|
@@ -142,6 +142,14 @@ export interface OpenOptions {
|
|
|
142
142
|
* running keeps the key it was started with.
|
|
143
143
|
*/
|
|
144
144
|
encryptionKeyFile?: string
|
|
145
|
+
/**
|
|
146
|
+
* The files of the store's older keys, which open the values they sealed
|
|
147
|
+
* and seal nothing new: a key being rotated out, until `reseal` has sealed
|
|
148
|
+
* every value again, or one found after a restore that left it out. Each
|
|
149
|
+
* file must be there. A sidecar found running keeps the keys it was
|
|
150
|
+
* started with.
|
|
151
|
+
*/
|
|
152
|
+
olderEncryptionKeys?: string[]
|
|
145
153
|
/**
|
|
146
154
|
* Makes the store a node: it keeps a log of each file it writes, serves
|
|
147
155
|
* its files to other nodes, and copies the files of the node it takes
|
|
@@ -573,6 +581,7 @@ export async function openWith(
|
|
|
573
581
|
const started = {
|
|
574
582
|
durability,
|
|
575
583
|
encryptionKeyFile: options.encryptionKeyFile,
|
|
584
|
+
olderEncryptionKeys: options.olderEncryptionKeys,
|
|
576
585
|
node: options.node,
|
|
577
586
|
selfMetrics: options.selfMetrics,
|
|
578
587
|
maxSeries: options.maxSeries,
|