@tinyshed/tinystore 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/blobs.d.ts +19 -2
- package/dist/blobs.js +58 -9
- package/dist/connection.d.ts +2 -0
- package/dist/connection.js +7 -1
- package/dist/kv.d.ts +9 -0
- package/dist/kv.js +14 -1
- package/dist/sql.d.ts +11 -0
- package/dist/sql.js +20 -1
- package/dist/store.d.ts +8 -0
- package/dist/store.js +1 -0
- package/dist/wire/protocol.d.ts +11 -0
- package/dist/wire/protocol.js +65 -1
- package/package.json +8 -8
- package/src/blobs.ts +88 -11
- package/src/connection.ts +9 -1
- package/src/kv.ts +21 -0
- package/src/sql.ts +26 -3
- package/src/store.ts +9 -0
- package/src/wire/protocol.ts +70 -1
package/dist/blobs.d.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { type Link } from './connection.ts';
|
|
2
|
+
import { type Home, type Open, type Via } from './handles.ts';
|
|
2
3
|
import { type Stream } from './session.ts';
|
|
3
4
|
import { type Duration } from './time.ts';
|
|
4
5
|
/** A size in bytes: a number, or text in binary units, 64KiB or 2GiB; '2GB' does not compile. */
|
|
@@ -51,6 +52,18 @@ export interface FilePage {
|
|
|
51
52
|
}
|
|
52
53
|
/** Files as `store.files` opens them. */
|
|
53
54
|
export declare function openFiles(link: Link, name: string, options?: FilesOptions): Files;
|
|
55
|
+
/** The open of a set of files, kept in the database of the handle `database` when it is given. */
|
|
56
|
+
export declare function filesOpen(name: string, options?: FilesOptions, database?: number): Uint8Array;
|
|
57
|
+
/** What a transaction needs of a set of files it takes in. */
|
|
58
|
+
export interface FilesParts {
|
|
59
|
+
readonly name: string;
|
|
60
|
+
readonly open: Open;
|
|
61
|
+
/** the database whose file keeps the files; the store's own when undefined */
|
|
62
|
+
readonly home: Home | undefined;
|
|
63
|
+
/** the same files, their calls through the transaction's `via` */
|
|
64
|
+
inside(via: Via): Files;
|
|
65
|
+
}
|
|
66
|
+
export declare function filesParts(handle: object): FilesParts | undefined;
|
|
54
67
|
/**
|
|
55
68
|
* A named set of files by path, or a folder of it, whose calls are the
|
|
56
69
|
* folder's: a path in it is the folder's segments and then its own.
|
|
@@ -58,8 +71,12 @@ export declare function openFiles(link: Link, name: string, options?: FilesOptio
|
|
|
58
71
|
export declare class Files {
|
|
59
72
|
#private;
|
|
60
73
|
readonly name: string;
|
|
61
|
-
/**
|
|
62
|
-
|
|
74
|
+
/**
|
|
75
|
+
* Files come from `store.files` and `db.files`; inside a database's
|
|
76
|
+
* transaction, from `tx.with(files)`, whose writes go through the
|
|
77
|
+
* transaction.
|
|
78
|
+
*/
|
|
79
|
+
constructor(link: Link, name: string, open: Open, folder: string[], home?: Home, via?: Via);
|
|
63
80
|
/** The folder of these segments, one each: `folder('users', 42)` is users/42, never users/420. */
|
|
64
81
|
folder(...segments: (string | number | bigint)[]): Files;
|
|
65
82
|
/**
|
package/dist/blobs.js
CHANGED
|
@@ -13,13 +13,22 @@ const decoder = new TextDecoder();
|
|
|
13
13
|
const nothing = new Uint8Array(0);
|
|
14
14
|
/** Files as `store.files` opens them. */
|
|
15
15
|
export function openFiles(link, name, options) {
|
|
16
|
+
return new Files(link, name, filesOpen(name, options), []);
|
|
17
|
+
}
|
|
18
|
+
/** The open of a set of files, kept in the database of the handle `database` when it is given. */
|
|
19
|
+
export function filesOpen(name, options, database) {
|
|
16
20
|
checkName(name, 'files');
|
|
17
|
-
|
|
21
|
+
return BlobsOpen.encode({
|
|
18
22
|
name,
|
|
19
23
|
ttl: options?.ttl === undefined ? undefined : ms(options.ttl),
|
|
20
24
|
maxFileSize: options?.maxFileSize === undefined ? undefined : bytesOf(options.maxFileSize),
|
|
25
|
+
database,
|
|
21
26
|
});
|
|
22
|
-
|
|
27
|
+
}
|
|
28
|
+
// a set of files' parts, kept out of its public shape, for db.tx
|
|
29
|
+
const parts = new WeakMap();
|
|
30
|
+
export function filesParts(handle) {
|
|
31
|
+
return parts.get(handle);
|
|
23
32
|
}
|
|
24
33
|
/**
|
|
25
34
|
* A named set of files by path, or a folder of it, whose calls are the
|
|
@@ -31,20 +40,35 @@ export class Files {
|
|
|
31
40
|
#via;
|
|
32
41
|
#open;
|
|
33
42
|
#folder;
|
|
34
|
-
|
|
35
|
-
|
|
43
|
+
#home;
|
|
44
|
+
/** whether its calls go through a database's transaction */
|
|
45
|
+
#inside;
|
|
46
|
+
/**
|
|
47
|
+
* Files come from `store.files` and `db.files`; inside a database's
|
|
48
|
+
* transaction, from `tx.with(files)`, whose writes go through the
|
|
49
|
+
* transaction.
|
|
50
|
+
*/
|
|
51
|
+
constructor(link, name, open, folder, home, via) {
|
|
36
52
|
this.name = name;
|
|
37
53
|
this.#link = link;
|
|
38
|
-
this.#via = viaLink(link);
|
|
54
|
+
this.#via = via ?? home?.via(`files ${name}`) ?? viaLink(link);
|
|
39
55
|
this.#open = open;
|
|
40
56
|
this.#folder = folder;
|
|
57
|
+
this.#home = home;
|
|
58
|
+
this.#inside = via !== undefined;
|
|
59
|
+
const inside = (through) => new Files(link, name, open, folder, home, through);
|
|
60
|
+
parts.set(this, { name, open, home, inside });
|
|
41
61
|
}
|
|
42
62
|
/** The folder of these segments, one each: `folder('users', 42)` is users/42, never users/420. */
|
|
43
63
|
folder(...segments) {
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
64
|
+
const folder = [...this.#folder, ...segments.map(segmentOf)];
|
|
65
|
+
return new Files(this.#link, this.name, this.#open, folder, this.#home, this.#inside ? this.#via : undefined);
|
|
66
|
+
}
|
|
67
|
+
/** Refuses what streams a file's bytes inside a transaction, which takes a put's bytes whole. */
|
|
68
|
+
#outside(what) {
|
|
69
|
+
if (this.#inside) {
|
|
70
|
+
throw new InvalidError(`files ${this.name}: ${what} inside a transaction: put a file's bytes whole, and read them after its commit`);
|
|
71
|
+
}
|
|
48
72
|
}
|
|
49
73
|
/**
|
|
50
74
|
* Writes the file whole and replaces whatever the path held, returning
|
|
@@ -146,6 +170,7 @@ export class Files {
|
|
|
146
170
|
* leaving the block without commit aborts it.
|
|
147
171
|
*/
|
|
148
172
|
async upload(path, options) {
|
|
173
|
+
this.#outside('an upload');
|
|
149
174
|
const connection = await this.#link.connection();
|
|
150
175
|
const write = BlobsWrite.encode({
|
|
151
176
|
...this.#fields(path, options),
|
|
@@ -236,6 +261,11 @@ export class Files {
|
|
|
236
261
|
size: options?.size ?? source.size,
|
|
237
262
|
create,
|
|
238
263
|
};
|
|
264
|
+
if (this.#inside) {
|
|
265
|
+
const bytes = await allOf(source);
|
|
266
|
+
const answer = await this.#call('blobs.put', handle => BlobsWrite.encode({ ...fields, handle, bytes }), 'write');
|
|
267
|
+
return writtenOf(answer);
|
|
268
|
+
}
|
|
239
269
|
return this.#link.run('write', async (connection) => {
|
|
240
270
|
const handle = await this.#handle(connection);
|
|
241
271
|
const piece = pieceOf(connection);
|
|
@@ -485,6 +515,25 @@ function sourceOf(body) {
|
|
|
485
515
|
whole: async () => undefined,
|
|
486
516
|
};
|
|
487
517
|
}
|
|
518
|
+
/** Every byte of a body at once, as a put inside a transaction sends it. */
|
|
519
|
+
async function allOf(source) {
|
|
520
|
+
const chunks = [];
|
|
521
|
+
let size = 0;
|
|
522
|
+
for await (const chunk of source.chunks()) {
|
|
523
|
+
chunks.push(chunk);
|
|
524
|
+
size += chunk.length;
|
|
525
|
+
}
|
|
526
|
+
if (chunks.length === 1) {
|
|
527
|
+
return chunks[0];
|
|
528
|
+
}
|
|
529
|
+
const all = new Uint8Array(size);
|
|
530
|
+
let at = 0;
|
|
531
|
+
for (const chunk of chunks) {
|
|
532
|
+
all.set(chunk, at);
|
|
533
|
+
at += chunk.length;
|
|
534
|
+
}
|
|
535
|
+
return all;
|
|
536
|
+
}
|
|
488
537
|
/**
|
|
489
538
|
* The bytes of chunks again in pieces of `size` bytes, the last shorter. A
|
|
490
539
|
* chunk that holds a whole piece goes as it is, without a copy.
|
package/dist/connection.d.ts
CHANGED
|
@@ -76,6 +76,8 @@ export interface Started {
|
|
|
76
76
|
durability?: Durability | undefined;
|
|
77
77
|
/** the file of the store's encryption key, when it is not the store's own */
|
|
78
78
|
encryptionKeyFile?: string | undefined;
|
|
79
|
+
/** the files of the store's older keys, which open what they sealed */
|
|
80
|
+
olderEncryptionKeys?: string[] | undefined;
|
|
79
81
|
/** the node the store is, when it is one */
|
|
80
82
|
node?: NodeOptions | undefined;
|
|
81
83
|
/** measures the store into its own metrics, the machine too unless it says system: false */
|
package/dist/connection.js
CHANGED
|
@@ -192,7 +192,12 @@ function selfMetricsOf(given) {
|
|
|
192
192
|
function storeFlags(started, dir) {
|
|
193
193
|
const committing = started.durability === undefined ? [] : ['--durability', started.durability];
|
|
194
194
|
const key = started.encryptionKeyFile;
|
|
195
|
-
const
|
|
195
|
+
const sealing = key === undefined ? committing : [...committing, '--encryption-key-file', resolve(key)];
|
|
196
|
+
const older = (started.olderEncryptionKeys ?? []).flatMap(file => [
|
|
197
|
+
'--older-encryption-key',
|
|
198
|
+
resolve(file),
|
|
199
|
+
]);
|
|
200
|
+
const keyed = [...sealing, ...older];
|
|
196
201
|
const asked = selfMetricsOf(started.selfMetrics);
|
|
197
202
|
const measured = asked === undefined
|
|
198
203
|
? keyed
|
|
@@ -552,6 +557,7 @@ export function embedded(runtime, dir, library, started = {}) {
|
|
|
552
557
|
const options = StoreOptions.encode({
|
|
553
558
|
durability: started.durability,
|
|
554
559
|
encryptionKeyFile,
|
|
560
|
+
olderEncryptionKeys: (started.olderEncryptionKeys ?? []).map(file => resolve(file)),
|
|
555
561
|
node: node !== undefined,
|
|
556
562
|
from: node?.from ?? [],
|
|
557
563
|
serve: node?.serve,
|
package/dist/kv.d.ts
CHANGED
|
@@ -118,6 +118,15 @@ export declare class Bucket<T> {
|
|
|
118
118
|
expire(key: Key, when: Duration | Date, options?: Pick<WriteOptions, 'ifVersion'>): Promise<boolean>;
|
|
119
119
|
/** Removes this branch's keys and every branch under it, at once however many. */
|
|
120
120
|
clear(): Promise<void>;
|
|
121
|
+
/**
|
|
122
|
+
* Seals again with the store's sealing key every value of this branch, and
|
|
123
|
+
* of the branches under it, that an older key sealed, and says how many.
|
|
124
|
+
* Once every value of the bucket is, the older key can leave the store's
|
|
125
|
+
* options. A value sealed with a key the store does not hold stops it with
|
|
126
|
+
* an `InvalidError` that names the key, and a reseal after the key is given
|
|
127
|
+
* goes on.
|
|
128
|
+
*/
|
|
129
|
+
reseal(): Promise<number>;
|
|
121
130
|
/** One page of this branch's own keys, in the byte order of their text: `"10"` before `"9"`. */
|
|
122
131
|
list(options?: {
|
|
123
132
|
limit?: number | undefined;
|
package/dist/kv.js
CHANGED
|
@@ -6,7 +6,7 @@ import { CorruptError, InvalidError } from './errors.js';
|
|
|
6
6
|
import { checkName, handleOn, ownerText, viaLink, } from './handles.js';
|
|
7
7
|
import { dateOf, ms, unixMs } from './time.js';
|
|
8
8
|
import { textOf } from './wire/codec.js';
|
|
9
|
-
import { KvBranch, KvBucketOpen, KvCall, KvCount, KvCountersOpen, KvEntry, KvFound, KvList, KvPage, KvWritten, methods, } from './wire/protocol.js';
|
|
9
|
+
import { KvBranch, KvBucketOpen, KvCall, KvCount, KvCountersOpen, KvEntry, KvFound, KvList, KvPage, KvResealed, KvWritten, methods, } from './wire/protocol.js';
|
|
10
10
|
// a handle's parts, kept out of its public shape, for store.tx
|
|
11
11
|
const parts = new WeakMap();
|
|
12
12
|
export function partsOf(handle) {
|
|
@@ -98,6 +98,19 @@ export class Bucket {
|
|
|
98
98
|
const under = underField(this.#under);
|
|
99
99
|
await this.#via.call(methods['kv.bucket.open'], this.#open, methods['kv.clear'], handle => KvBranch.encode({ handle, under }), 'write');
|
|
100
100
|
}
|
|
101
|
+
/**
|
|
102
|
+
* Seals again with the store's sealing key every value of this branch, and
|
|
103
|
+
* of the branches under it, that an older key sealed, and says how many.
|
|
104
|
+
* Once every value of the bucket is, the older key can leave the store's
|
|
105
|
+
* options. A value sealed with a key the store does not hold stops it with
|
|
106
|
+
* an `InvalidError` that names the key, and a reseal after the key is given
|
|
107
|
+
* goes on.
|
|
108
|
+
*/
|
|
109
|
+
async reseal() {
|
|
110
|
+
const under = underField(this.#under);
|
|
111
|
+
const answer = await this.#via.call(methods['kv.bucket.open'], this.#open, methods['kv.reseal'], handle => KvBranch.encode({ handle, under }), 'write');
|
|
112
|
+
return KvResealed.decode(answer).count ?? 0;
|
|
113
|
+
}
|
|
101
114
|
/** One page of this branch's own keys, in the byte order of their text: `"10"` before `"9"`. */
|
|
102
115
|
async list(options) {
|
|
103
116
|
const under = underField(this.#under);
|
package/dist/sql.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { Files, type FilesOptions } from './blobs.ts';
|
|
1
2
|
import { type Channel, type TxChannel } from './channels.ts';
|
|
2
3
|
import type { Connection, Durability, Link } from './connection.ts';
|
|
3
4
|
import { type Home, type Open, type Via } from './handles.ts';
|
|
@@ -171,6 +172,15 @@ export declare class Database implements Runner, Home {
|
|
|
171
172
|
schema: S;
|
|
172
173
|
}): Queue<StandardSchemaV1.InferOutput<S>>;
|
|
173
174
|
queue<T = unknown>(name: string, options?: QueueOptions): Queue<T>;
|
|
175
|
+
/**
|
|
176
|
+
* A set of files kept in this database's file, as `store.files` keeps one
|
|
177
|
+
* in blobs.db: the same calls, and its writes commit with the rows in
|
|
178
|
+
* `db.tx`, through `tx.with(files)`. A file's bytes past 16 KiB lie in the
|
|
179
|
+
* folder beside the database's file.
|
|
180
|
+
*
|
|
181
|
+
* const packages = db.files('packages', { maxFileSize: '10MiB' })
|
|
182
|
+
*/
|
|
183
|
+
files(name: string, options?: FilesOptions): Files;
|
|
174
184
|
/**
|
|
175
185
|
* An index of a table of this database, kept in its file: its rows found
|
|
176
186
|
* again by their words, vectors or both, each hit's value its row as
|
|
@@ -266,6 +276,7 @@ export declare class SqlTx implements Runner {
|
|
|
266
276
|
with<T>(bucket: Bucket<T>): Bucket<T>;
|
|
267
277
|
with<T>(queue: Queue<T>): TxQueue<T>;
|
|
268
278
|
with<T>(channel: Channel<T>): TxChannel<T>;
|
|
279
|
+
with(files: Files): Files;
|
|
269
280
|
all<T = Row>(...statement: Statement): Promise<T[]>;
|
|
270
281
|
one<T = Row>(...statement: Statement): Promise<T | undefined>;
|
|
271
282
|
scalar<T = SqlValue>(...statement: Statement): Promise<T>;
|
package/dist/sql.js
CHANGED
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
import { AsyncLocalStorage } from 'node:async_hooks';
|
|
7
7
|
import { readdir, readFile } from 'node:fs/promises';
|
|
8
8
|
import { join } from 'node:path';
|
|
9
|
+
import { Files, filesOpen, filesParts } from './blobs.js';
|
|
9
10
|
import { channelParts, txChannel } from './channels.js';
|
|
10
11
|
import { checkDurability, download } from './connection.js';
|
|
11
12
|
import { CancelledError, errorOf, InvalidError, OutcomeUnknownError } from './errors.js';
|
|
@@ -546,6 +547,19 @@ export class Database {
|
|
|
546
547
|
queue(name, options = {}) {
|
|
547
548
|
return openQueue(this.#link, this.#workers, name, options, this);
|
|
548
549
|
}
|
|
550
|
+
/**
|
|
551
|
+
* A set of files kept in this database's file, as `store.files` keeps one
|
|
552
|
+
* in blobs.db: the same calls, and its writes commit with the rows in
|
|
553
|
+
* `db.tx`, through `tx.with(files)`. A file's bytes past 16 KiB lie in the
|
|
554
|
+
* folder beside the database's file.
|
|
555
|
+
*
|
|
556
|
+
* const packages = db.files('packages', { maxFileSize: '10MiB' })
|
|
557
|
+
*/
|
|
558
|
+
files(name, options) {
|
|
559
|
+
const bytes = filesOpen(name, options);
|
|
560
|
+
const open = this.keep(methods['blobs.open'], bytes, async (connection) => filesOpen(name, options, await this.handle(connection)));
|
|
561
|
+
return new Files(this.#link, name, open, [], this);
|
|
562
|
+
}
|
|
549
563
|
/**
|
|
550
564
|
* An index of a table of this database, kept in its file: its rows found
|
|
551
565
|
* again by their words, vectors or both, each hit's value its row as
|
|
@@ -787,7 +801,12 @@ export class SqlTx {
|
|
|
787
801
|
this.#takes(queue.source.home, queue.source.open, `jobs queue ${queue.name}`);
|
|
788
802
|
return queueVia(queue, this.#via);
|
|
789
803
|
}
|
|
790
|
-
|
|
804
|
+
const files = filesParts(handle);
|
|
805
|
+
if (files !== undefined) {
|
|
806
|
+
this.#takes(files.home, files.open, `files ${files.name}`);
|
|
807
|
+
return files.inside(this.#via);
|
|
808
|
+
}
|
|
809
|
+
throw new InvalidError(`${this.describe}: a transaction takes in a bucket, a queue or files opened from its database`);
|
|
791
810
|
}
|
|
792
811
|
async all(...statement) {
|
|
793
812
|
return all(this, statementOf(statement));
|
package/dist/store.d.ts
CHANGED
|
@@ -81,6 +81,14 @@ export interface OpenOptions {
|
|
|
81
81
|
* running keeps the key it was started with.
|
|
82
82
|
*/
|
|
83
83
|
encryptionKeyFile?: string;
|
|
84
|
+
/**
|
|
85
|
+
* The files of the store's older keys, which open the values they sealed
|
|
86
|
+
* and seal nothing new: a key being rotated out, until `reseal` has sealed
|
|
87
|
+
* every value again, or one found after a restore that left it out. Each
|
|
88
|
+
* file must be there. A sidecar found running keeps the keys it was
|
|
89
|
+
* started with.
|
|
90
|
+
*/
|
|
91
|
+
olderEncryptionKeys?: string[];
|
|
84
92
|
/**
|
|
85
93
|
* Makes the store a node: it keeps a log of each file it writes, serves
|
|
86
94
|
* its files to other nodes, and copies the files of the node it takes
|
package/dist/store.js
CHANGED
|
@@ -317,6 +317,7 @@ export async function openWith(runtime, dir, options = {}) {
|
|
|
317
317
|
const started = {
|
|
318
318
|
durability,
|
|
319
319
|
encryptionKeyFile: options.encryptionKeyFile,
|
|
320
|
+
olderEncryptionKeys: options.olderEncryptionKeys,
|
|
320
321
|
node: options.node,
|
|
321
322
|
selfMetrics: options.selfMetrics,
|
|
322
323
|
maxSeries: options.maxSeries,
|
package/dist/wire/protocol.d.ts
CHANGED
|
@@ -56,6 +56,7 @@ export declare const methods: {
|
|
|
56
56
|
readonly 'kv.expire': 264;
|
|
57
57
|
readonly 'kv.clear': 265;
|
|
58
58
|
readonly 'kv.list': 266;
|
|
59
|
+
readonly 'kv.reseal': 267;
|
|
59
60
|
readonly 'kv.counters.open': 272;
|
|
60
61
|
readonly 'kv.counters.add': 273;
|
|
61
62
|
readonly 'kv.counters.get': 274;
|
|
@@ -180,6 +181,8 @@ export declare const BlobsOpen: import("./codec.ts").Message<{
|
|
|
180
181
|
readonly ttl: readonly [2, import("./codec.ts").Codec<number, number>];
|
|
181
182
|
/** the largest file a write may leave */
|
|
182
183
|
readonly maxFileSize: readonly [3, import("./codec.ts").Codec<number, number>];
|
|
184
|
+
/** a database's handle: the files live in its file, and put, head, delete, copy and rename join its transactions; blobs.db when absent */
|
|
185
|
+
readonly database: readonly [4, import("./codec.ts").Codec<number, number>];
|
|
183
186
|
}>;
|
|
184
187
|
/**
|
|
185
188
|
* Where a call stands: the files, a folder's segments, a path within it.
|
|
@@ -529,6 +532,8 @@ export declare const StoreOptions: import("./codec.ts").Message<{
|
|
|
529
532
|
readonly withoutSystem: readonly [14, import("./codec.ts").Codec<boolean, boolean>];
|
|
530
533
|
/** the metrics series the store holds at most; 100,000 when absent */
|
|
531
534
|
readonly maxSeries: readonly [15, import("./codec.ts").Codec<number, number>];
|
|
535
|
+
/** the files of the store's older keys, which open the values they sealed and seal none */
|
|
536
|
+
readonly olderEncryptionKeys: readonly [16, import("./codec.ts").Codec<string[], readonly string[]>];
|
|
532
537
|
}>;
|
|
533
538
|
/**
|
|
534
539
|
* The last frame of a connection.
|
|
@@ -953,6 +958,12 @@ export declare const KvWritten: import("./codec.ts").Message<{
|
|
|
953
958
|
export declare const KvFound: import("./codec.ts").Message<{
|
|
954
959
|
readonly found: readonly [1, import("./codec.ts").Codec<boolean, boolean>];
|
|
955
960
|
}>;
|
|
961
|
+
/**
|
|
962
|
+
* How many values a reseal sealed again with the sealing key.
|
|
963
|
+
*/
|
|
964
|
+
export declare const KvResealed: import("./codec.ts").Message<{
|
|
965
|
+
readonly count: readonly [1, import("./codec.ts").Codec<number, number>];
|
|
966
|
+
}>;
|
|
956
967
|
/**
|
|
957
968
|
* A branch of a handle, for a clear.
|
|
958
969
|
*/
|
package/dist/wire/protocol.js
CHANGED
|
@@ -57,6 +57,7 @@ export const methods = {
|
|
|
57
57
|
'kv.expire': 0x0108,
|
|
58
58
|
'kv.clear': 0x0109,
|
|
59
59
|
'kv.list': 0x010a,
|
|
60
|
+
'kv.reseal': 0x010b,
|
|
60
61
|
'kv.counters.open': 0x0110,
|
|
61
62
|
'kv.counters.add': 0x0111,
|
|
62
63
|
'kv.counters.get': 0x0112,
|
|
@@ -291,6 +292,8 @@ export const BlobsOpen = message('blobs.Open', {
|
|
|
291
292
|
ttl: [2, uint],
|
|
292
293
|
/** the largest file a write may leave */
|
|
293
294
|
maxFileSize: [3, uint],
|
|
295
|
+
/** a database's handle: the files live in its file, and put, head, delete, copy and rename join its transactions; blobs.db when absent */
|
|
296
|
+
database: [4, uint],
|
|
294
297
|
}, {
|
|
295
298
|
write(w, v) {
|
|
296
299
|
const head = w.openMap();
|
|
@@ -310,12 +313,18 @@ export const BlobsOpen = message('blobs.Open', {
|
|
|
310
313
|
w.uint(v.maxFileSize);
|
|
311
314
|
n++;
|
|
312
315
|
}
|
|
316
|
+
if (v.database !== undefined) {
|
|
317
|
+
w.field(4);
|
|
318
|
+
w.uint(v.database);
|
|
319
|
+
n++;
|
|
320
|
+
}
|
|
313
321
|
w.closeMap(head, n);
|
|
314
322
|
},
|
|
315
323
|
read(r) {
|
|
316
324
|
let $name;
|
|
317
325
|
let $ttl;
|
|
318
326
|
let $maxFileSize;
|
|
327
|
+
let $database;
|
|
319
328
|
for (let n = r.message(); n > 0; n--) {
|
|
320
329
|
switch (r.field()) {
|
|
321
330
|
case 1:
|
|
@@ -327,12 +336,15 @@ export const BlobsOpen = message('blobs.Open', {
|
|
|
327
336
|
case 3:
|
|
328
337
|
$maxFileSize = r.uint();
|
|
329
338
|
break;
|
|
339
|
+
case 4:
|
|
340
|
+
$database = r.uint();
|
|
341
|
+
break;
|
|
330
342
|
default:
|
|
331
343
|
r.skip();
|
|
332
344
|
}
|
|
333
345
|
}
|
|
334
346
|
r.leave();
|
|
335
|
-
return { name: $name, ttl: $ttl, maxFileSize: $maxFileSize };
|
|
347
|
+
return { name: $name, ttl: $ttl, maxFileSize: $maxFileSize, database: $database };
|
|
336
348
|
},
|
|
337
349
|
});
|
|
338
350
|
/**
|
|
@@ -2111,6 +2123,8 @@ export const StoreOptions = message('store.Options', {
|
|
|
2111
2123
|
withoutSystem: [14, bool],
|
|
2112
2124
|
/** the metrics series the store holds at most; 100,000 when absent */
|
|
2113
2125
|
maxSeries: [15, uint],
|
|
2126
|
+
/** the files of the store's older keys, which open the values they sealed and seal none */
|
|
2127
|
+
olderEncryptionKeys: [16, list(str)],
|
|
2114
2128
|
}, {
|
|
2115
2129
|
write(w, v) {
|
|
2116
2130
|
const head = w.openMap();
|
|
@@ -2199,6 +2213,14 @@ export const StoreOptions = message('store.Options', {
|
|
|
2199
2213
|
w.uint(v.maxSeries);
|
|
2200
2214
|
n++;
|
|
2201
2215
|
}
|
|
2216
|
+
if (v.olderEncryptionKeys !== undefined) {
|
|
2217
|
+
w.field(16);
|
|
2218
|
+
w.array(v.olderEncryptionKeys.length);
|
|
2219
|
+
for (const item0 of v.olderEncryptionKeys) {
|
|
2220
|
+
w.str(item0);
|
|
2221
|
+
}
|
|
2222
|
+
n++;
|
|
2223
|
+
}
|
|
2202
2224
|
w.closeMap(head, n);
|
|
2203
2225
|
},
|
|
2204
2226
|
read(r) {
|
|
@@ -2217,6 +2239,7 @@ export const StoreOptions = message('store.Options', {
|
|
|
2217
2239
|
let $selfMetrics;
|
|
2218
2240
|
let $withoutSystem;
|
|
2219
2241
|
let $maxSeries;
|
|
2242
|
+
let $olderEncryptionKeys;
|
|
2220
2243
|
for (let n = r.message(); n > 0; n--) {
|
|
2221
2244
|
switch (r.field()) {
|
|
2222
2245
|
case 1:
|
|
@@ -2282,6 +2305,15 @@ export const StoreOptions = message('store.Options', {
|
|
|
2282
2305
|
case 15:
|
|
2283
2306
|
$maxSeries = r.uint();
|
|
2284
2307
|
break;
|
|
2308
|
+
case 16: {
|
|
2309
|
+
const items0 = [];
|
|
2310
|
+
for (let i0 = r.array(); i0 > 0; i0--) {
|
|
2311
|
+
items0.push(r.str());
|
|
2312
|
+
}
|
|
2313
|
+
r.leave();
|
|
2314
|
+
$olderEncryptionKeys = items0;
|
|
2315
|
+
break;
|
|
2316
|
+
}
|
|
2285
2317
|
default:
|
|
2286
2318
|
r.skip();
|
|
2287
2319
|
}
|
|
@@ -2303,6 +2335,7 @@ export const StoreOptions = message('store.Options', {
|
|
|
2303
2335
|
selfMetrics: $selfMetrics,
|
|
2304
2336
|
withoutSystem: $withoutSystem,
|
|
2305
2337
|
maxSeries: $maxSeries,
|
|
2338
|
+
olderEncryptionKeys: $olderEncryptionKeys,
|
|
2306
2339
|
};
|
|
2307
2340
|
},
|
|
2308
2341
|
});
|
|
@@ -4145,6 +4178,37 @@ export const KvFound = message('kv.Found', {
|
|
|
4145
4178
|
return { found: $found };
|
|
4146
4179
|
},
|
|
4147
4180
|
});
|
|
4181
|
+
/**
|
|
4182
|
+
* How many values a reseal sealed again with the sealing key.
|
|
4183
|
+
*/
|
|
4184
|
+
export const KvResealed = message('kv.Resealed', {
|
|
4185
|
+
count: [1, uint],
|
|
4186
|
+
}, {
|
|
4187
|
+
write(w, v) {
|
|
4188
|
+
const head = w.openMap();
|
|
4189
|
+
let n = 0;
|
|
4190
|
+
if (v.count !== undefined) {
|
|
4191
|
+
w.field(1);
|
|
4192
|
+
w.uint(v.count);
|
|
4193
|
+
n++;
|
|
4194
|
+
}
|
|
4195
|
+
w.closeMap(head, n);
|
|
4196
|
+
},
|
|
4197
|
+
read(r) {
|
|
4198
|
+
let $count;
|
|
4199
|
+
for (let n = r.message(); n > 0; n--) {
|
|
4200
|
+
switch (r.field()) {
|
|
4201
|
+
case 1:
|
|
4202
|
+
$count = r.uint();
|
|
4203
|
+
break;
|
|
4204
|
+
default:
|
|
4205
|
+
r.skip();
|
|
4206
|
+
}
|
|
4207
|
+
}
|
|
4208
|
+
r.leave();
|
|
4209
|
+
return { count: $count };
|
|
4210
|
+
},
|
|
4211
|
+
});
|
|
4148
4212
|
/**
|
|
4149
4213
|
* A branch of a handle, for a clear.
|
|
4150
4214
|
*/
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@tinyshed/tinystore",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.4.0",
|
|
4
4
|
"description": "TinyStore for Bun and Node: kv, jobs, blobs, SQL, records and metrics in one directory, served by a sidecar",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"homepage": "https://tinyshed.org/tinystore",
|
|
@@ -31,12 +31,12 @@
|
|
|
31
31
|
"README.md"
|
|
32
32
|
],
|
|
33
33
|
"optionalDependencies": {
|
|
34
|
-
"@tinyshed/tinystore-linux-x64": "0.
|
|
35
|
-
"@tinyshed/tinystore-linux-arm64": "0.
|
|
36
|
-
"@tinyshed/tinystore-linux-x64-musl": "0.
|
|
37
|
-
"@tinyshed/tinystore-linux-arm64-musl": "0.
|
|
38
|
-
"@tinyshed/tinystore-darwin-arm64": "0.
|
|
39
|
-
"@tinyshed/tinystore-darwin-x64": "0.
|
|
40
|
-
"@tinyshed/tinystore-win32-x64": "0.
|
|
34
|
+
"@tinyshed/tinystore-linux-x64": "0.4.0",
|
|
35
|
+
"@tinyshed/tinystore-linux-arm64": "0.4.0",
|
|
36
|
+
"@tinyshed/tinystore-linux-x64-musl": "0.4.0",
|
|
37
|
+
"@tinyshed/tinystore-linux-arm64-musl": "0.4.0",
|
|
38
|
+
"@tinyshed/tinystore-darwin-arm64": "0.4.0",
|
|
39
|
+
"@tinyshed/tinystore-darwin-x64": "0.4.0",
|
|
40
|
+
"@tinyshed/tinystore-win32-x64": "0.4.0"
|
|
41
41
|
}
|
|
42
42
|
}
|
package/src/blobs.ts
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
|
|
6
6
|
import { type Connection, type Idempotence, type Link, pieceOf } from './connection.ts'
|
|
7
7
|
import { ClosedError, InvalidError, OutcomeUnknownError } from './errors.ts'
|
|
8
|
-
import { checkName, handleOn, type Via, viaLink } from './handles.ts'
|
|
8
|
+
import { checkName, type Home, handleOn, type Open, type Via, viaLink } from './handles.ts'
|
|
9
9
|
import { ConnectionLostError, type Stream, watch } from './session.ts'
|
|
10
10
|
import { type Duration, dateOf, ms, unixMs } from './time.ts'
|
|
11
11
|
import {
|
|
@@ -96,13 +96,35 @@ const nothing = new Uint8Array(0)
|
|
|
96
96
|
|
|
97
97
|
/** Files as `store.files` opens them. */
|
|
98
98
|
export function openFiles(link: Link, name: string, options?: FilesOptions): Files {
|
|
99
|
+
return new Files(link, name, filesOpen(name, options), [])
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/** The open of a set of files, kept in the database of the handle `database` when it is given. */
|
|
103
|
+
export function filesOpen(name: string, options?: FilesOptions, database?: number): Uint8Array {
|
|
99
104
|
checkName(name, 'files')
|
|
100
|
-
|
|
105
|
+
return BlobsOpen.encode({
|
|
101
106
|
name,
|
|
102
107
|
ttl: options?.ttl === undefined ? undefined : ms(options.ttl),
|
|
103
108
|
maxFileSize: options?.maxFileSize === undefined ? undefined : bytesOf(options.maxFileSize),
|
|
109
|
+
database,
|
|
104
110
|
})
|
|
105
|
-
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/** What a transaction needs of a set of files it takes in. */
|
|
114
|
+
export interface FilesParts {
|
|
115
|
+
readonly name: string
|
|
116
|
+
readonly open: Open
|
|
117
|
+
/** the database whose file keeps the files; the store's own when undefined */
|
|
118
|
+
readonly home: Home | undefined
|
|
119
|
+
/** the same files, their calls through the transaction's `via` */
|
|
120
|
+
inside(via: Via): Files
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
// a set of files' parts, kept out of its public shape, for db.tx
|
|
124
|
+
const parts = new WeakMap<object, FilesParts>()
|
|
125
|
+
|
|
126
|
+
export function filesParts(handle: object): FilesParts | undefined {
|
|
127
|
+
return parts.get(handle)
|
|
106
128
|
}
|
|
107
129
|
|
|
108
130
|
/**
|
|
@@ -113,24 +135,49 @@ export class Files {
|
|
|
113
135
|
readonly name: string
|
|
114
136
|
readonly #link: Link
|
|
115
137
|
readonly #via: Via
|
|
116
|
-
readonly #open:
|
|
138
|
+
readonly #open: Open
|
|
117
139
|
readonly #folder: string[]
|
|
140
|
+
readonly #home: Home | undefined
|
|
141
|
+
/** whether its calls go through a database's transaction */
|
|
142
|
+
readonly #inside: boolean
|
|
118
143
|
|
|
119
|
-
/**
|
|
120
|
-
|
|
144
|
+
/**
|
|
145
|
+
* Files come from `store.files` and `db.files`; inside a database's
|
|
146
|
+
* transaction, from `tx.with(files)`, whose writes go through the
|
|
147
|
+
* transaction.
|
|
148
|
+
*/
|
|
149
|
+
constructor(link: Link, name: string, open: Open, folder: string[], home?: Home, via?: Via) {
|
|
121
150
|
this.name = name
|
|
122
151
|
this.#link = link
|
|
123
|
-
this.#via = viaLink(link)
|
|
152
|
+
this.#via = via ?? home?.via(`files ${name}`) ?? viaLink(link)
|
|
124
153
|
this.#open = open
|
|
125
154
|
this.#folder = folder
|
|
155
|
+
this.#home = home
|
|
156
|
+
this.#inside = via !== undefined
|
|
157
|
+
const inside = (through: Via) => new Files(link, name, open, folder, home, through)
|
|
158
|
+
parts.set(this, { name, open, home, inside })
|
|
126
159
|
}
|
|
127
160
|
|
|
128
161
|
/** The folder of these segments, one each: `folder('users', 42)` is users/42, never users/420. */
|
|
129
162
|
folder(...segments: (string | number | bigint)[]): Files {
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
163
|
+
const folder = [...this.#folder, ...segments.map(segmentOf)]
|
|
164
|
+
return new Files(
|
|
165
|
+
this.#link,
|
|
166
|
+
this.name,
|
|
167
|
+
this.#open,
|
|
168
|
+
folder,
|
|
169
|
+
this.#home,
|
|
170
|
+
this.#inside ? this.#via : undefined,
|
|
171
|
+
)
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
/** Refuses what streams a file's bytes inside a transaction, which takes a put's bytes whole. */
|
|
175
|
+
#outside(what: string): void {
|
|
176
|
+
if (this.#inside) {
|
|
177
|
+
throw new InvalidError(
|
|
178
|
+
`files ${this.name}: ${what} inside a transaction: put a file's bytes whole, and read them after its commit`,
|
|
179
|
+
)
|
|
180
|
+
}
|
|
134
181
|
}
|
|
135
182
|
|
|
136
183
|
/**
|
|
@@ -277,6 +324,7 @@ export class Files {
|
|
|
277
324
|
* leaving the block without commit aborts it.
|
|
278
325
|
*/
|
|
279
326
|
async upload(path: string, options?: FileOptions): Promise<Upload> {
|
|
327
|
+
this.#outside('an upload')
|
|
280
328
|
const connection = await this.#link.connection()
|
|
281
329
|
const write = BlobsWrite.encode({
|
|
282
330
|
...this.#fields(path, options),
|
|
@@ -396,6 +444,15 @@ export class Files {
|
|
|
396
444
|
size: options?.size ?? source.size,
|
|
397
445
|
create,
|
|
398
446
|
}
|
|
447
|
+
if (this.#inside) {
|
|
448
|
+
const bytes = await allOf(source)
|
|
449
|
+
const answer = await this.#call(
|
|
450
|
+
'blobs.put',
|
|
451
|
+
handle => BlobsWrite.encode({ ...fields, handle, bytes }),
|
|
452
|
+
'write',
|
|
453
|
+
)
|
|
454
|
+
return writtenOf(answer)
|
|
455
|
+
}
|
|
399
456
|
return this.#link.run(
|
|
400
457
|
'write',
|
|
401
458
|
async connection => {
|
|
@@ -688,6 +745,26 @@ function sourceOf(body: Body): Source {
|
|
|
688
745
|
}
|
|
689
746
|
}
|
|
690
747
|
|
|
748
|
+
/** Every byte of a body at once, as a put inside a transaction sends it. */
|
|
749
|
+
async function allOf(source: Source): Promise<Uint8Array> {
|
|
750
|
+
const chunks: Uint8Array[] = []
|
|
751
|
+
let size = 0
|
|
752
|
+
for await (const chunk of source.chunks()) {
|
|
753
|
+
chunks.push(chunk)
|
|
754
|
+
size += chunk.length
|
|
755
|
+
}
|
|
756
|
+
if (chunks.length === 1) {
|
|
757
|
+
return chunks[0] as Uint8Array
|
|
758
|
+
}
|
|
759
|
+
const all = new Uint8Array(size)
|
|
760
|
+
let at = 0
|
|
761
|
+
for (const chunk of chunks) {
|
|
762
|
+
all.set(chunk, at)
|
|
763
|
+
at += chunk.length
|
|
764
|
+
}
|
|
765
|
+
return all
|
|
766
|
+
}
|
|
767
|
+
|
|
691
768
|
/**
|
|
692
769
|
* The bytes of chunks again in pieces of `size` bytes, the last shorter. A
|
|
693
770
|
* chunk that holds a whole piece goes as it is, without a copy.
|
package/src/connection.ts
CHANGED
|
@@ -293,6 +293,8 @@ export interface Started {
|
|
|
293
293
|
durability?: Durability | undefined
|
|
294
294
|
/** the file of the store's encryption key, when it is not the store's own */
|
|
295
295
|
encryptionKeyFile?: string | undefined
|
|
296
|
+
/** the files of the store's older keys, which open what they sealed */
|
|
297
|
+
olderEncryptionKeys?: string[] | undefined
|
|
296
298
|
/** the node the store is, when it is one */
|
|
297
299
|
node?: NodeOptions | undefined
|
|
298
300
|
/** measures the store into its own metrics, the machine too unless it says system: false */
|
|
@@ -319,8 +321,13 @@ function selfMetricsOf(given: SelfMetrics | undefined): { system: boolean } | un
|
|
|
319
321
|
function storeFlags(started: Started, dir: string): string[] {
|
|
320
322
|
const committing = started.durability === undefined ? [] : ['--durability', started.durability]
|
|
321
323
|
const key = started.encryptionKeyFile
|
|
322
|
-
const
|
|
324
|
+
const sealing =
|
|
323
325
|
key === undefined ? committing : [...committing, '--encryption-key-file', resolve(key)]
|
|
326
|
+
const older = (started.olderEncryptionKeys ?? []).flatMap(file => [
|
|
327
|
+
'--older-encryption-key',
|
|
328
|
+
resolve(file),
|
|
329
|
+
])
|
|
330
|
+
const keyed = [...sealing, ...older]
|
|
324
331
|
const asked = selfMetricsOf(started.selfMetrics)
|
|
325
332
|
const measured =
|
|
326
333
|
asked === undefined
|
|
@@ -743,6 +750,7 @@ export function embedded(
|
|
|
743
750
|
const options = StoreOptions.encode({
|
|
744
751
|
durability: started.durability,
|
|
745
752
|
encryptionKeyFile,
|
|
753
|
+
olderEncryptionKeys: (started.olderEncryptionKeys ?? []).map(file => resolve(file)),
|
|
746
754
|
node: node !== undefined,
|
|
747
755
|
from: node?.from ?? [],
|
|
748
756
|
serve: node?.serve,
|
package/src/kv.ts
CHANGED
|
@@ -27,6 +27,7 @@ import {
|
|
|
27
27
|
KvFound,
|
|
28
28
|
KvList,
|
|
29
29
|
KvPage,
|
|
30
|
+
KvResealed,
|
|
30
31
|
KvWritten,
|
|
31
32
|
type Method,
|
|
32
33
|
methods,
|
|
@@ -239,6 +240,26 @@ export class Bucket<T> {
|
|
|
239
240
|
)
|
|
240
241
|
}
|
|
241
242
|
|
|
243
|
+
/**
|
|
244
|
+
* Seals again with the store's sealing key every value of this branch, and
|
|
245
|
+
* of the branches under it, that an older key sealed, and says how many.
|
|
246
|
+
* Once every value of the bucket is, the older key can leave the store's
|
|
247
|
+
* options. A value sealed with a key the store does not hold stops it with
|
|
248
|
+
* an `InvalidError` that names the key, and a reseal after the key is given
|
|
249
|
+
* goes on.
|
|
250
|
+
*/
|
|
251
|
+
async reseal(): Promise<number> {
|
|
252
|
+
const under = underField(this.#under)
|
|
253
|
+
const answer = await this.#via.call(
|
|
254
|
+
methods['kv.bucket.open'],
|
|
255
|
+
this.#open,
|
|
256
|
+
methods['kv.reseal'],
|
|
257
|
+
handle => KvBranch.encode({ handle, under }),
|
|
258
|
+
'write',
|
|
259
|
+
)
|
|
260
|
+
return KvResealed.decode(answer).count ?? 0
|
|
261
|
+
}
|
|
262
|
+
|
|
242
263
|
/** One page of this branch's own keys, in the byte order of their text: `"10"` before `"9"`. */
|
|
243
264
|
async list(options?: { limit?: number | undefined; after?: Key | undefined }): Promise<Page<T>> {
|
|
244
265
|
const under = underField(this.#under)
|
package/src/sql.ts
CHANGED
|
@@ -8,6 +8,7 @@ import { AsyncLocalStorage } from 'node:async_hooks'
|
|
|
8
8
|
import { readdir, readFile } from 'node:fs/promises'
|
|
9
9
|
import { join } from 'node:path'
|
|
10
10
|
|
|
11
|
+
import { Files, type FilesOptions, filesOpen, filesParts } from './blobs.ts'
|
|
11
12
|
import { type Channel, channelParts, type TxChannel, txChannel } from './channels.ts'
|
|
12
13
|
import type { Connection, Durability, Idempotence, Link } from './connection.ts'
|
|
13
14
|
import { checkDurability, download } from './connection.ts'
|
|
@@ -774,6 +775,22 @@ export class Database implements Runner, Home {
|
|
|
774
775
|
return openQueue(this.#link, this.#workers, name, options, this)
|
|
775
776
|
}
|
|
776
777
|
|
|
778
|
+
/**
|
|
779
|
+
* A set of files kept in this database's file, as `store.files` keeps one
|
|
780
|
+
* in blobs.db: the same calls, and its writes commit with the rows in
|
|
781
|
+
* `db.tx`, through `tx.with(files)`. A file's bytes past 16 KiB lie in the
|
|
782
|
+
* folder beside the database's file.
|
|
783
|
+
*
|
|
784
|
+
* const packages = db.files('packages', { maxFileSize: '10MiB' })
|
|
785
|
+
*/
|
|
786
|
+
files(name: string, options?: FilesOptions): Files {
|
|
787
|
+
const bytes = filesOpen(name, options)
|
|
788
|
+
const open = this.keep(methods['blobs.open'], bytes, async connection =>
|
|
789
|
+
filesOpen(name, options, await this.handle(connection)),
|
|
790
|
+
)
|
|
791
|
+
return new Files(this.#link, name, open, [], this)
|
|
792
|
+
}
|
|
793
|
+
|
|
777
794
|
/**
|
|
778
795
|
* An index of a table of this database, kept in its file: its rows found
|
|
779
796
|
* again by their words, vectors or both, each hit's value its row as
|
|
@@ -1059,9 +1076,10 @@ export class SqlTx implements Runner {
|
|
|
1059
1076
|
with<T>(bucket: Bucket<T>): Bucket<T>
|
|
1060
1077
|
with<T>(queue: Queue<T>): TxQueue<T>
|
|
1061
1078
|
with<T>(channel: Channel<T>): TxChannel<T>
|
|
1079
|
+
with(files: Files): Files
|
|
1062
1080
|
with(
|
|
1063
|
-
handle: Bucket<unknown> | Queue<unknown> | Channel<unknown
|
|
1064
|
-
): Bucket<unknown> | TxQueue<unknown> | TxChannel<unknown> {
|
|
1081
|
+
handle: Bucket<unknown> | Queue<unknown> | Channel<unknown> | Files,
|
|
1082
|
+
): Bucket<unknown> | TxQueue<unknown> | TxChannel<unknown> | Files {
|
|
1065
1083
|
const channel = channelParts(handle)
|
|
1066
1084
|
if (channel !== undefined) {
|
|
1067
1085
|
// a message published here is handed over once the transaction commits, at the writer
|
|
@@ -1080,8 +1098,13 @@ export class SqlTx implements Runner {
|
|
|
1080
1098
|
this.#takes(queue.source.home, queue.source.open, `jobs queue ${queue.name}`)
|
|
1081
1099
|
return queueVia(queue, this.#via)
|
|
1082
1100
|
}
|
|
1101
|
+
const files = filesParts(handle)
|
|
1102
|
+
if (files !== undefined) {
|
|
1103
|
+
this.#takes(files.home, files.open, `files ${files.name}`)
|
|
1104
|
+
return files.inside(this.#via)
|
|
1105
|
+
}
|
|
1083
1106
|
throw new InvalidError(
|
|
1084
|
-
`${this.describe}: a transaction takes in a bucket
|
|
1107
|
+
`${this.describe}: a transaction takes in a bucket, a queue or files opened from its database`,
|
|
1085
1108
|
)
|
|
1086
1109
|
}
|
|
1087
1110
|
|
package/src/store.ts
CHANGED
|
@@ -142,6 +142,14 @@ export interface OpenOptions {
|
|
|
142
142
|
* running keeps the key it was started with.
|
|
143
143
|
*/
|
|
144
144
|
encryptionKeyFile?: string
|
|
145
|
+
/**
|
|
146
|
+
* The files of the store's older keys, which open the values they sealed
|
|
147
|
+
* and seal nothing new: a key being rotated out, until `reseal` has sealed
|
|
148
|
+
* every value again, or one found after a restore that left it out. Each
|
|
149
|
+
* file must be there. A sidecar found running keeps the keys it was
|
|
150
|
+
* started with.
|
|
151
|
+
*/
|
|
152
|
+
olderEncryptionKeys?: string[]
|
|
145
153
|
/**
|
|
146
154
|
* Makes the store a node: it keeps a log of each file it writes, serves
|
|
147
155
|
* its files to other nodes, and copies the files of the node it takes
|
|
@@ -573,6 +581,7 @@ export async function openWith(
|
|
|
573
581
|
const started = {
|
|
574
582
|
durability,
|
|
575
583
|
encryptionKeyFile: options.encryptionKeyFile,
|
|
584
|
+
olderEncryptionKeys: options.olderEncryptionKeys,
|
|
576
585
|
node: options.node,
|
|
577
586
|
selfMetrics: options.selfMetrics,
|
|
578
587
|
maxSeries: options.maxSeries,
|
package/src/wire/protocol.ts
CHANGED
|
@@ -78,6 +78,7 @@ export const methods = {
|
|
|
78
78
|
'kv.expire': 0x0108,
|
|
79
79
|
'kv.clear': 0x0109,
|
|
80
80
|
'kv.list': 0x010a,
|
|
81
|
+
'kv.reseal': 0x010b,
|
|
81
82
|
'kv.counters.open': 0x0110,
|
|
82
83
|
'kv.counters.add': 0x0111,
|
|
83
84
|
'kv.counters.get': 0x0112,
|
|
@@ -332,6 +333,8 @@ export const BlobsOpen = message(
|
|
|
332
333
|
ttl: [2, uint],
|
|
333
334
|
/** the largest file a write may leave */
|
|
334
335
|
maxFileSize: [3, uint],
|
|
336
|
+
/** a database's handle: the files live in its file, and put, head, delete, copy and rename join its transactions; blobs.db when absent */
|
|
337
|
+
database: [4, uint],
|
|
335
338
|
},
|
|
336
339
|
{
|
|
337
340
|
write(w, v) {
|
|
@@ -352,12 +355,18 @@ export const BlobsOpen = message(
|
|
|
352
355
|
w.uint(v.maxFileSize)
|
|
353
356
|
n++
|
|
354
357
|
}
|
|
358
|
+
if (v.database !== undefined) {
|
|
359
|
+
w.field(4)
|
|
360
|
+
w.uint(v.database)
|
|
361
|
+
n++
|
|
362
|
+
}
|
|
355
363
|
w.closeMap(head, n)
|
|
356
364
|
},
|
|
357
365
|
read(r) {
|
|
358
366
|
let $name: string | undefined
|
|
359
367
|
let $ttl: number | undefined
|
|
360
368
|
let $maxFileSize: number | undefined
|
|
369
|
+
let $database: number | undefined
|
|
361
370
|
for (let n = r.message(); n > 0; n--) {
|
|
362
371
|
switch (r.field()) {
|
|
363
372
|
case 1:
|
|
@@ -369,12 +378,15 @@ export const BlobsOpen = message(
|
|
|
369
378
|
case 3:
|
|
370
379
|
$maxFileSize = r.uint()
|
|
371
380
|
break
|
|
381
|
+
case 4:
|
|
382
|
+
$database = r.uint()
|
|
383
|
+
break
|
|
372
384
|
default:
|
|
373
385
|
r.skip()
|
|
374
386
|
}
|
|
375
387
|
}
|
|
376
388
|
r.leave()
|
|
377
|
-
return { name: $name, ttl: $ttl, maxFileSize: $maxFileSize }
|
|
389
|
+
return { name: $name, ttl: $ttl, maxFileSize: $maxFileSize, database: $database }
|
|
378
390
|
},
|
|
379
391
|
},
|
|
380
392
|
)
|
|
@@ -2282,6 +2294,8 @@ export const StoreOptions = message(
|
|
|
2282
2294
|
withoutSystem: [14, bool],
|
|
2283
2295
|
/** the metrics series the store holds at most; 100,000 when absent */
|
|
2284
2296
|
maxSeries: [15, uint],
|
|
2297
|
+
/** the files of the store's older keys, which open the values they sealed and seal none */
|
|
2298
|
+
olderEncryptionKeys: [16, list(str)],
|
|
2285
2299
|
},
|
|
2286
2300
|
{
|
|
2287
2301
|
write(w, v) {
|
|
@@ -2371,6 +2385,14 @@ export const StoreOptions = message(
|
|
|
2371
2385
|
w.uint(v.maxSeries)
|
|
2372
2386
|
n++
|
|
2373
2387
|
}
|
|
2388
|
+
if (v.olderEncryptionKeys !== undefined) {
|
|
2389
|
+
w.field(16)
|
|
2390
|
+
w.array(v.olderEncryptionKeys.length)
|
|
2391
|
+
for (const item0 of v.olderEncryptionKeys) {
|
|
2392
|
+
w.str(item0)
|
|
2393
|
+
}
|
|
2394
|
+
n++
|
|
2395
|
+
}
|
|
2374
2396
|
w.closeMap(head, n)
|
|
2375
2397
|
},
|
|
2376
2398
|
read(r) {
|
|
@@ -2389,6 +2411,7 @@ export const StoreOptions = message(
|
|
|
2389
2411
|
let $selfMetrics: boolean | undefined
|
|
2390
2412
|
let $withoutSystem: boolean | undefined
|
|
2391
2413
|
let $maxSeries: number | undefined
|
|
2414
|
+
let $olderEncryptionKeys: string[] | undefined
|
|
2392
2415
|
for (let n = r.message(); n > 0; n--) {
|
|
2393
2416
|
switch (r.field()) {
|
|
2394
2417
|
case 1:
|
|
@@ -2454,6 +2477,15 @@ export const StoreOptions = message(
|
|
|
2454
2477
|
case 15:
|
|
2455
2478
|
$maxSeries = r.uint()
|
|
2456
2479
|
break
|
|
2480
|
+
case 16: {
|
|
2481
|
+
const items0: string[] = []
|
|
2482
|
+
for (let i0 = r.array(); i0 > 0; i0--) {
|
|
2483
|
+
items0.push(r.str())
|
|
2484
|
+
}
|
|
2485
|
+
r.leave()
|
|
2486
|
+
$olderEncryptionKeys = items0
|
|
2487
|
+
break
|
|
2488
|
+
}
|
|
2457
2489
|
default:
|
|
2458
2490
|
r.skip()
|
|
2459
2491
|
}
|
|
@@ -2475,6 +2507,7 @@ export const StoreOptions = message(
|
|
|
2475
2507
|
selfMetrics: $selfMetrics,
|
|
2476
2508
|
withoutSystem: $withoutSystem,
|
|
2477
2509
|
maxSeries: $maxSeries,
|
|
2510
|
+
olderEncryptionKeys: $olderEncryptionKeys,
|
|
2478
2511
|
}
|
|
2479
2512
|
},
|
|
2480
2513
|
},
|
|
@@ -4459,6 +4492,42 @@ export const KvFound = message(
|
|
|
4459
4492
|
},
|
|
4460
4493
|
)
|
|
4461
4494
|
|
|
4495
|
+
/**
|
|
4496
|
+
* How many values a reseal sealed again with the sealing key.
|
|
4497
|
+
*/
|
|
4498
|
+
export const KvResealed = message(
|
|
4499
|
+
'kv.Resealed',
|
|
4500
|
+
{
|
|
4501
|
+
count: [1, uint],
|
|
4502
|
+
},
|
|
4503
|
+
{
|
|
4504
|
+
write(w, v) {
|
|
4505
|
+
const head = w.openMap()
|
|
4506
|
+
let n = 0
|
|
4507
|
+
if (v.count !== undefined) {
|
|
4508
|
+
w.field(1)
|
|
4509
|
+
w.uint(v.count)
|
|
4510
|
+
n++
|
|
4511
|
+
}
|
|
4512
|
+
w.closeMap(head, n)
|
|
4513
|
+
},
|
|
4514
|
+
read(r) {
|
|
4515
|
+
let $count: number | undefined
|
|
4516
|
+
for (let n = r.message(); n > 0; n--) {
|
|
4517
|
+
switch (r.field()) {
|
|
4518
|
+
case 1:
|
|
4519
|
+
$count = r.uint()
|
|
4520
|
+
break
|
|
4521
|
+
default:
|
|
4522
|
+
r.skip()
|
|
4523
|
+
}
|
|
4524
|
+
}
|
|
4525
|
+
r.leave()
|
|
4526
|
+
return { count: $count }
|
|
4527
|
+
},
|
|
4528
|
+
},
|
|
4529
|
+
)
|
|
4530
|
+
|
|
4462
4531
|
/**
|
|
4463
4532
|
* A branch of a handle, for a clear.
|
|
4464
4533
|
*/
|