@tinyshed/tinystore 0.3.0 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/blobs.d.ts +19 -2
- package/dist/blobs.js +58 -9
- package/dist/connection.d.ts +2 -0
- package/dist/connection.js +7 -1
- package/dist/kv.d.ts +9 -0
- package/dist/kv.js +14 -1
- package/dist/logger.d.ts +6 -1
- package/dist/logger.js +29 -5
- package/dist/search.d.ts +6 -0
- package/dist/search.js +1 -0
- package/dist/sql.d.ts +11 -0
- package/dist/sql.js +20 -1
- package/dist/store.d.ts +8 -0
- package/dist/store.js +1 -0
- package/dist/wire/protocol.d.ts +13 -0
- package/dist/wire/protocol.js +77 -1
- package/package.json +8 -8
- package/src/blobs.ts +88 -11
- package/src/connection.ts +9 -1
- package/src/kv.ts +21 -0
- package/src/logger.ts +29 -5
- package/src/search.ts +7 -0
- package/src/sql.ts +26 -3
- package/src/store.ts +9 -0
- package/src/wire/protocol.ts +82 -1
package/dist/blobs.d.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { type Link } from './connection.ts';
|
|
2
|
+
import { type Home, type Open, type Via } from './handles.ts';
|
|
2
3
|
import { type Stream } from './session.ts';
|
|
3
4
|
import { type Duration } from './time.ts';
|
|
4
5
|
/** A size in bytes: a number, or text in binary units, 64KiB or 2GiB; '2GB' does not compile. */
|
|
@@ -51,6 +52,18 @@ export interface FilePage {
|
|
|
51
52
|
}
|
|
52
53
|
/** Files as `store.files` opens them. */
|
|
53
54
|
export declare function openFiles(link: Link, name: string, options?: FilesOptions): Files;
|
|
55
|
+
/** The open of a set of files, kept in the database of the handle `database` when it is given. */
|
|
56
|
+
export declare function filesOpen(name: string, options?: FilesOptions, database?: number): Uint8Array;
|
|
57
|
+
/** What a transaction needs of a set of files it takes in. */
|
|
58
|
+
export interface FilesParts {
|
|
59
|
+
readonly name: string;
|
|
60
|
+
readonly open: Open;
|
|
61
|
+
/** the database whose file keeps the files; the store's own when undefined */
|
|
62
|
+
readonly home: Home | undefined;
|
|
63
|
+
/** the same files, their calls through the transaction's `via` */
|
|
64
|
+
inside(via: Via): Files;
|
|
65
|
+
}
|
|
66
|
+
export declare function filesParts(handle: object): FilesParts | undefined;
|
|
54
67
|
/**
|
|
55
68
|
* A named set of files by path, or a folder of it, whose calls are the
|
|
56
69
|
* folder's: a path in it is the folder's segments and then its own.
|
|
@@ -58,8 +71,12 @@ export declare function openFiles(link: Link, name: string, options?: FilesOptio
|
|
|
58
71
|
export declare class Files {
|
|
59
72
|
#private;
|
|
60
73
|
readonly name: string;
|
|
61
|
-
/**
|
|
62
|
-
|
|
74
|
+
/**
|
|
75
|
+
* Files come from `store.files` and `db.files`; inside a database's
|
|
76
|
+
* transaction, from `tx.with(files)`, whose writes go through the
|
|
77
|
+
* transaction.
|
|
78
|
+
*/
|
|
79
|
+
constructor(link: Link, name: string, open: Open, folder: string[], home?: Home, via?: Via);
|
|
63
80
|
/** The folder of these segments, one each: `folder('users', 42)` is users/42, never users/420. */
|
|
64
81
|
folder(...segments: (string | number | bigint)[]): Files;
|
|
65
82
|
/**
|
package/dist/blobs.js
CHANGED
|
@@ -13,13 +13,22 @@ const decoder = new TextDecoder();
|
|
|
13
13
|
const nothing = new Uint8Array(0);
|
|
14
14
|
/** Files as `store.files` opens them. */
|
|
15
15
|
export function openFiles(link, name, options) {
|
|
16
|
+
return new Files(link, name, filesOpen(name, options), []);
|
|
17
|
+
}
|
|
18
|
+
/** The open of a set of files, kept in the database of the handle `database` when it is given. */
|
|
19
|
+
export function filesOpen(name, options, database) {
|
|
16
20
|
checkName(name, 'files');
|
|
17
|
-
|
|
21
|
+
return BlobsOpen.encode({
|
|
18
22
|
name,
|
|
19
23
|
ttl: options?.ttl === undefined ? undefined : ms(options.ttl),
|
|
20
24
|
maxFileSize: options?.maxFileSize === undefined ? undefined : bytesOf(options.maxFileSize),
|
|
25
|
+
database,
|
|
21
26
|
});
|
|
22
|
-
|
|
27
|
+
}
|
|
28
|
+
// a set of files' parts, kept out of its public shape, for db.tx
|
|
29
|
+
const parts = new WeakMap();
|
|
30
|
+
export function filesParts(handle) {
|
|
31
|
+
return parts.get(handle);
|
|
23
32
|
}
|
|
24
33
|
/**
|
|
25
34
|
* A named set of files by path, or a folder of it, whose calls are the
|
|
@@ -31,20 +40,35 @@ export class Files {
|
|
|
31
40
|
#via;
|
|
32
41
|
#open;
|
|
33
42
|
#folder;
|
|
34
|
-
|
|
35
|
-
|
|
43
|
+
#home;
|
|
44
|
+
/** whether its calls go through a database's transaction */
|
|
45
|
+
#inside;
|
|
46
|
+
/**
|
|
47
|
+
* Files come from `store.files` and `db.files`; inside a database's
|
|
48
|
+
* transaction, from `tx.with(files)`, whose writes go through the
|
|
49
|
+
* transaction.
|
|
50
|
+
*/
|
|
51
|
+
constructor(link, name, open, folder, home, via) {
|
|
36
52
|
this.name = name;
|
|
37
53
|
this.#link = link;
|
|
38
|
-
this.#via = viaLink(link);
|
|
54
|
+
this.#via = via ?? home?.via(`files ${name}`) ?? viaLink(link);
|
|
39
55
|
this.#open = open;
|
|
40
56
|
this.#folder = folder;
|
|
57
|
+
this.#home = home;
|
|
58
|
+
this.#inside = via !== undefined;
|
|
59
|
+
const inside = (through) => new Files(link, name, open, folder, home, through);
|
|
60
|
+
parts.set(this, { name, open, home, inside });
|
|
41
61
|
}
|
|
42
62
|
/** The folder of these segments, one each: `folder('users', 42)` is users/42, never users/420. */
|
|
43
63
|
folder(...segments) {
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
64
|
+
const folder = [...this.#folder, ...segments.map(segmentOf)];
|
|
65
|
+
return new Files(this.#link, this.name, this.#open, folder, this.#home, this.#inside ? this.#via : undefined);
|
|
66
|
+
}
|
|
67
|
+
/** Refuses what streams a file's bytes inside a transaction, which takes a put's bytes whole. */
|
|
68
|
+
#outside(what) {
|
|
69
|
+
if (this.#inside) {
|
|
70
|
+
throw new InvalidError(`files ${this.name}: ${what} inside a transaction: put a file's bytes whole, and read them after its commit`);
|
|
71
|
+
}
|
|
48
72
|
}
|
|
49
73
|
/**
|
|
50
74
|
* Writes the file whole and replaces whatever the path held, returning
|
|
@@ -146,6 +170,7 @@ export class Files {
|
|
|
146
170
|
* leaving the block without commit aborts it.
|
|
147
171
|
*/
|
|
148
172
|
async upload(path, options) {
|
|
173
|
+
this.#outside('an upload');
|
|
149
174
|
const connection = await this.#link.connection();
|
|
150
175
|
const write = BlobsWrite.encode({
|
|
151
176
|
...this.#fields(path, options),
|
|
@@ -236,6 +261,11 @@ export class Files {
|
|
|
236
261
|
size: options?.size ?? source.size,
|
|
237
262
|
create,
|
|
238
263
|
};
|
|
264
|
+
if (this.#inside) {
|
|
265
|
+
const bytes = await allOf(source);
|
|
266
|
+
const answer = await this.#call('blobs.put', handle => BlobsWrite.encode({ ...fields, handle, bytes }), 'write');
|
|
267
|
+
return writtenOf(answer);
|
|
268
|
+
}
|
|
239
269
|
return this.#link.run('write', async (connection) => {
|
|
240
270
|
const handle = await this.#handle(connection);
|
|
241
271
|
const piece = pieceOf(connection);
|
|
@@ -485,6 +515,25 @@ function sourceOf(body) {
|
|
|
485
515
|
whole: async () => undefined,
|
|
486
516
|
};
|
|
487
517
|
}
|
|
518
|
+
/** Every byte of a body at once, as a put inside a transaction sends it. */
|
|
519
|
+
async function allOf(source) {
|
|
520
|
+
const chunks = [];
|
|
521
|
+
let size = 0;
|
|
522
|
+
for await (const chunk of source.chunks()) {
|
|
523
|
+
chunks.push(chunk);
|
|
524
|
+
size += chunk.length;
|
|
525
|
+
}
|
|
526
|
+
if (chunks.length === 1) {
|
|
527
|
+
return chunks[0];
|
|
528
|
+
}
|
|
529
|
+
const all = new Uint8Array(size);
|
|
530
|
+
let at = 0;
|
|
531
|
+
for (const chunk of chunks) {
|
|
532
|
+
all.set(chunk, at);
|
|
533
|
+
at += chunk.length;
|
|
534
|
+
}
|
|
535
|
+
return all;
|
|
536
|
+
}
|
|
488
537
|
/**
|
|
489
538
|
* The bytes of chunks again in pieces of `size` bytes, the last shorter. A
|
|
490
539
|
* chunk that holds a whole piece goes as it is, without a copy.
|
package/dist/connection.d.ts
CHANGED
|
@@ -76,6 +76,8 @@ export interface Started {
|
|
|
76
76
|
durability?: Durability | undefined;
|
|
77
77
|
/** the file of the store's encryption key, when it is not the store's own */
|
|
78
78
|
encryptionKeyFile?: string | undefined;
|
|
79
|
+
/** the files of the store's older keys, which open what they sealed */
|
|
80
|
+
olderEncryptionKeys?: string[] | undefined;
|
|
79
81
|
/** the node the store is, when it is one */
|
|
80
82
|
node?: NodeOptions | undefined;
|
|
81
83
|
/** measures the store into its own metrics, the machine too unless it says system: false */
|
package/dist/connection.js
CHANGED
|
@@ -192,7 +192,12 @@ function selfMetricsOf(given) {
|
|
|
192
192
|
function storeFlags(started, dir) {
|
|
193
193
|
const committing = started.durability === undefined ? [] : ['--durability', started.durability];
|
|
194
194
|
const key = started.encryptionKeyFile;
|
|
195
|
-
const
|
|
195
|
+
const sealing = key === undefined ? committing : [...committing, '--encryption-key-file', resolve(key)];
|
|
196
|
+
const older = (started.olderEncryptionKeys ?? []).flatMap(file => [
|
|
197
|
+
'--older-encryption-key',
|
|
198
|
+
resolve(file),
|
|
199
|
+
]);
|
|
200
|
+
const keyed = [...sealing, ...older];
|
|
196
201
|
const asked = selfMetricsOf(started.selfMetrics);
|
|
197
202
|
const measured = asked === undefined
|
|
198
203
|
? keyed
|
|
@@ -552,6 +557,7 @@ export function embedded(runtime, dir, library, started = {}) {
|
|
|
552
557
|
const options = StoreOptions.encode({
|
|
553
558
|
durability: started.durability,
|
|
554
559
|
encryptionKeyFile,
|
|
560
|
+
olderEncryptionKeys: (started.olderEncryptionKeys ?? []).map(file => resolve(file)),
|
|
555
561
|
node: node !== undefined,
|
|
556
562
|
from: node?.from ?? [],
|
|
557
563
|
serve: node?.serve,
|
package/dist/kv.d.ts
CHANGED
|
@@ -118,6 +118,15 @@ export declare class Bucket<T> {
|
|
|
118
118
|
expire(key: Key, when: Duration | Date, options?: Pick<WriteOptions, 'ifVersion'>): Promise<boolean>;
|
|
119
119
|
/** Removes this branch's keys and every branch under it, at once however many. */
|
|
120
120
|
clear(): Promise<void>;
|
|
121
|
+
/**
|
|
122
|
+
* Seals again with the store's sealing key every value of this branch, and
|
|
123
|
+
* of the branches under it, that an older key sealed, and says how many.
|
|
124
|
+
* Once every value of the bucket is, the older key can leave the store's
|
|
125
|
+
* options. A value sealed with a key the store does not hold stops it with
|
|
126
|
+
* an `InvalidError` that names the key, and a reseal after the key is given
|
|
127
|
+
* goes on.
|
|
128
|
+
*/
|
|
129
|
+
reseal(): Promise<number>;
|
|
121
130
|
/** One page of this branch's own keys, in the byte order of their text: `"10"` before `"9"`. */
|
|
122
131
|
list(options?: {
|
|
123
132
|
limit?: number | undefined;
|
package/dist/kv.js
CHANGED
|
@@ -6,7 +6,7 @@ import { CorruptError, InvalidError } from './errors.js';
|
|
|
6
6
|
import { checkName, handleOn, ownerText, viaLink, } from './handles.js';
|
|
7
7
|
import { dateOf, ms, unixMs } from './time.js';
|
|
8
8
|
import { textOf } from './wire/codec.js';
|
|
9
|
-
import { KvBranch, KvBucketOpen, KvCall, KvCount, KvCountersOpen, KvEntry, KvFound, KvList, KvPage, KvWritten, methods, } from './wire/protocol.js';
|
|
9
|
+
import { KvBranch, KvBucketOpen, KvCall, KvCount, KvCountersOpen, KvEntry, KvFound, KvList, KvPage, KvResealed, KvWritten, methods, } from './wire/protocol.js';
|
|
10
10
|
// a handle's parts, kept out of its public shape, for store.tx
|
|
11
11
|
const parts = new WeakMap();
|
|
12
12
|
export function partsOf(handle) {
|
|
@@ -98,6 +98,19 @@ export class Bucket {
|
|
|
98
98
|
const under = underField(this.#under);
|
|
99
99
|
await this.#via.call(methods['kv.bucket.open'], this.#open, methods['kv.clear'], handle => KvBranch.encode({ handle, under }), 'write');
|
|
100
100
|
}
|
|
101
|
+
/**
|
|
102
|
+
* Seals again with the store's sealing key every value of this branch, and
|
|
103
|
+
* of the branches under it, that an older key sealed, and says how many.
|
|
104
|
+
* Once every value of the bucket is, the older key can leave the store's
|
|
105
|
+
* options. A value sealed with a key the store does not hold stops it with
|
|
106
|
+
* an `InvalidError` that names the key, and a reseal after the key is given
|
|
107
|
+
* goes on.
|
|
108
|
+
*/
|
|
109
|
+
async reseal() {
|
|
110
|
+
const under = underField(this.#under);
|
|
111
|
+
const answer = await this.#via.call(methods['kv.bucket.open'], this.#open, methods['kv.reseal'], handle => KvBranch.encode({ handle, under }), 'write');
|
|
112
|
+
return KvResealed.decode(answer).count ?? 0;
|
|
113
|
+
}
|
|
101
114
|
/** One page of this branch's own keys, in the byte order of their text: `"10"` before `"9"`. */
|
|
102
115
|
async list(options) {
|
|
103
116
|
const under = underField(this.#under);
|
package/dist/logger.d.ts
CHANGED
|
@@ -131,7 +131,12 @@ export declare class Logger {
|
|
|
131
131
|
error(message: string, attrs?: Fields): void;
|
|
132
132
|
/** An event of the stream: a record named for it, with no level and no body. */
|
|
133
133
|
event(name: string, attrs?: Fields): void;
|
|
134
|
-
/**
|
|
134
|
+
/**
|
|
135
|
+
* The logger of the same stream, buffer and console, its context these
|
|
136
|
+
* fields besides its own: a key it had already takes the new value where
|
|
137
|
+
* it stood, as an object spread does, where it prints, in what is kept
|
|
138
|
+
* and for a query's `where`.
|
|
139
|
+
*/
|
|
135
140
|
with(context: Fields): Logger;
|
|
136
141
|
/** Hands what the buffer holds to the server now, as the timer does every second. */
|
|
137
142
|
flush(): Promise<void>;
|
package/dist/logger.js
CHANGED
|
@@ -303,12 +303,15 @@ export class Logger {
|
|
|
303
303
|
}
|
|
304
304
|
this.#emit({ event: name }, attrs);
|
|
305
305
|
}
|
|
306
|
-
/**
|
|
306
|
+
/**
|
|
307
|
+
* The logger of the same stream, buffer and console, its context these
|
|
308
|
+
* fields besides its own: a key it had already takes the new value where
|
|
309
|
+
* it stood, as an object spread does, where it prints, in what is kept
|
|
310
|
+
* and for a query's `where`.
|
|
311
|
+
*/
|
|
307
312
|
with(context) {
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
...encodeFields(context, this.#shared.redact, this.#shared.replace),
|
|
311
|
-
]);
|
|
313
|
+
const added = encodeFields(context, this.#shared.redact, this.#shared.replace);
|
|
314
|
+
return new Logger(this.#shared, onceEach(this.#context, added));
|
|
312
315
|
}
|
|
313
316
|
/** Hands what the buffer holds to the server now, as the timer does every second. */
|
|
314
317
|
async flush() {
|
|
@@ -380,6 +383,27 @@ export class Logger {
|
|
|
380
383
|
}
|
|
381
384
|
}
|
|
382
385
|
}
|
|
386
|
+
/**
|
|
387
|
+
* A context's key, JSON pairs with `added`'s after them, each key once: where
|
|
388
|
+
* it was first given, holding the value given last.
|
|
389
|
+
*
|
|
390
|
+
* ```text
|
|
391
|
+
* component=http user=7, then component=access → component=access user=7
|
|
392
|
+
* ```
|
|
393
|
+
*/
|
|
394
|
+
function onceEach(context, added) {
|
|
395
|
+
const once = [...context];
|
|
396
|
+
for (let i = 0; i < added.length; i += 2) {
|
|
397
|
+
const at = once.findIndex((key, place) => place % 2 === 0 && key === added[i]);
|
|
398
|
+
if (at === -1) {
|
|
399
|
+
once.push(added[i], added[i + 1]);
|
|
400
|
+
}
|
|
401
|
+
else {
|
|
402
|
+
once[at + 1] = added[i + 1];
|
|
403
|
+
}
|
|
404
|
+
}
|
|
405
|
+
return once;
|
|
406
|
+
}
|
|
383
407
|
/** Whether a logger prints a line of level: it has a console, and the level is the level printed or above. */
|
|
384
408
|
function prints(shared, level) {
|
|
385
409
|
return shared.print !== undefined && level >= (shared.printed ?? shared.kept);
|
package/dist/search.d.ts
CHANGED
|
@@ -54,6 +54,12 @@ export interface Query {
|
|
|
54
54
|
vectorWeight?: number | undefined;
|
|
55
55
|
rerank?: boolean | undefined;
|
|
56
56
|
} | undefined;
|
|
57
|
+
/**
|
|
58
|
+
* the last word read as the start of a word too, as a search box types:
|
|
59
|
+
* `crypt` finds `cryptography` as well, at half its field's weight; a
|
|
60
|
+
* word a space follows is whole, and so is one of a single letter
|
|
61
|
+
*/
|
|
62
|
+
prefix?: boolean | undefined;
|
|
57
63
|
/** 10 unless said; offset and limit together 1000 at most */
|
|
58
64
|
limit?: number | undefined;
|
|
59
65
|
offset?: number | undefined;
|
package/dist/search.js
CHANGED
package/dist/sql.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { Files, type FilesOptions } from './blobs.ts';
|
|
1
2
|
import { type Channel, type TxChannel } from './channels.ts';
|
|
2
3
|
import type { Connection, Durability, Link } from './connection.ts';
|
|
3
4
|
import { type Home, type Open, type Via } from './handles.ts';
|
|
@@ -171,6 +172,15 @@ export declare class Database implements Runner, Home {
|
|
|
171
172
|
schema: S;
|
|
172
173
|
}): Queue<StandardSchemaV1.InferOutput<S>>;
|
|
173
174
|
queue<T = unknown>(name: string, options?: QueueOptions): Queue<T>;
|
|
175
|
+
/**
|
|
176
|
+
* A set of files kept in this database's file, as `store.files` keeps one
|
|
177
|
+
* in blobs.db: the same calls, and its writes commit with the rows in
|
|
178
|
+
* `db.tx`, through `tx.with(files)`. A file's bytes past 16 KiB lie in the
|
|
179
|
+
* folder beside the database's file.
|
|
180
|
+
*
|
|
181
|
+
* const packages = db.files('packages', { maxFileSize: '10MiB' })
|
|
182
|
+
*/
|
|
183
|
+
files(name: string, options?: FilesOptions): Files;
|
|
174
184
|
/**
|
|
175
185
|
* An index of a table of this database, kept in its file: its rows found
|
|
176
186
|
* again by their words, vectors or both, each hit's value its row as
|
|
@@ -266,6 +276,7 @@ export declare class SqlTx implements Runner {
|
|
|
266
276
|
with<T>(bucket: Bucket<T>): Bucket<T>;
|
|
267
277
|
with<T>(queue: Queue<T>): TxQueue<T>;
|
|
268
278
|
with<T>(channel: Channel<T>): TxChannel<T>;
|
|
279
|
+
with(files: Files): Files;
|
|
269
280
|
all<T = Row>(...statement: Statement): Promise<T[]>;
|
|
270
281
|
one<T = Row>(...statement: Statement): Promise<T | undefined>;
|
|
271
282
|
scalar<T = SqlValue>(...statement: Statement): Promise<T>;
|
package/dist/sql.js
CHANGED
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
import { AsyncLocalStorage } from 'node:async_hooks';
|
|
7
7
|
import { readdir, readFile } from 'node:fs/promises';
|
|
8
8
|
import { join } from 'node:path';
|
|
9
|
+
import { Files, filesOpen, filesParts } from './blobs.js';
|
|
9
10
|
import { channelParts, txChannel } from './channels.js';
|
|
10
11
|
import { checkDurability, download } from './connection.js';
|
|
11
12
|
import { CancelledError, errorOf, InvalidError, OutcomeUnknownError } from './errors.js';
|
|
@@ -546,6 +547,19 @@ export class Database {
|
|
|
546
547
|
queue(name, options = {}) {
|
|
547
548
|
return openQueue(this.#link, this.#workers, name, options, this);
|
|
548
549
|
}
|
|
550
|
+
/**
|
|
551
|
+
* A set of files kept in this database's file, as `store.files` keeps one
|
|
552
|
+
* in blobs.db: the same calls, and its writes commit with the rows in
|
|
553
|
+
* `db.tx`, through `tx.with(files)`. A file's bytes past 16 KiB lie in the
|
|
554
|
+
* folder beside the database's file.
|
|
555
|
+
*
|
|
556
|
+
* const packages = db.files('packages', { maxFileSize: '10MiB' })
|
|
557
|
+
*/
|
|
558
|
+
files(name, options) {
|
|
559
|
+
const bytes = filesOpen(name, options);
|
|
560
|
+
const open = this.keep(methods['blobs.open'], bytes, async (connection) => filesOpen(name, options, await this.handle(connection)));
|
|
561
|
+
return new Files(this.#link, name, open, [], this);
|
|
562
|
+
}
|
|
549
563
|
/**
|
|
550
564
|
* An index of a table of this database, kept in its file: its rows found
|
|
551
565
|
* again by their words, vectors or both, each hit's value its row as
|
|
@@ -787,7 +801,12 @@ export class SqlTx {
|
|
|
787
801
|
this.#takes(queue.source.home, queue.source.open, `jobs queue ${queue.name}`);
|
|
788
802
|
return queueVia(queue, this.#via);
|
|
789
803
|
}
|
|
790
|
-
|
|
804
|
+
const files = filesParts(handle);
|
|
805
|
+
if (files !== undefined) {
|
|
806
|
+
this.#takes(files.home, files.open, `files ${files.name}`);
|
|
807
|
+
return files.inside(this.#via);
|
|
808
|
+
}
|
|
809
|
+
throw new InvalidError(`${this.describe}: a transaction takes in a bucket, a queue or files opened from its database`);
|
|
791
810
|
}
|
|
792
811
|
async all(...statement) {
|
|
793
812
|
return all(this, statementOf(statement));
|
package/dist/store.d.ts
CHANGED
|
@@ -81,6 +81,14 @@ export interface OpenOptions {
|
|
|
81
81
|
* running keeps the key it was started with.
|
|
82
82
|
*/
|
|
83
83
|
encryptionKeyFile?: string;
|
|
84
|
+
/**
|
|
85
|
+
* The files of the store's older keys, which open the values they sealed
|
|
86
|
+
* and seal nothing new: a key being rotated out, until `reseal` has sealed
|
|
87
|
+
* every value again, or one found after a restore that left it out. Each
|
|
88
|
+
* file must be there. A sidecar found running keeps the keys it was
|
|
89
|
+
* started with.
|
|
90
|
+
*/
|
|
91
|
+
olderEncryptionKeys?: string[];
|
|
84
92
|
/**
|
|
85
93
|
* Makes the store a node: it keeps a log of each file it writes, serves
|
|
86
94
|
* its files to other nodes, and copies the files of the node it takes
|
package/dist/store.js
CHANGED
|
@@ -317,6 +317,7 @@ export async function openWith(runtime, dir, options = {}) {
|
|
|
317
317
|
const started = {
|
|
318
318
|
durability,
|
|
319
319
|
encryptionKeyFile: options.encryptionKeyFile,
|
|
320
|
+
olderEncryptionKeys: options.olderEncryptionKeys,
|
|
320
321
|
node: options.node,
|
|
321
322
|
selfMetrics: options.selfMetrics,
|
|
322
323
|
maxSeries: options.maxSeries,
|
package/dist/wire/protocol.d.ts
CHANGED
|
@@ -56,6 +56,7 @@ export declare const methods: {
|
|
|
56
56
|
readonly 'kv.expire': 264;
|
|
57
57
|
readonly 'kv.clear': 265;
|
|
58
58
|
readonly 'kv.list': 266;
|
|
59
|
+
readonly 'kv.reseal': 267;
|
|
59
60
|
readonly 'kv.counters.open': 272;
|
|
60
61
|
readonly 'kv.counters.add': 273;
|
|
61
62
|
readonly 'kv.counters.get': 274;
|
|
@@ -180,6 +181,8 @@ export declare const BlobsOpen: import("./codec.ts").Message<{
|
|
|
180
181
|
readonly ttl: readonly [2, import("./codec.ts").Codec<number, number>];
|
|
181
182
|
/** the largest file a write may leave */
|
|
182
183
|
readonly maxFileSize: readonly [3, import("./codec.ts").Codec<number, number>];
|
|
184
|
+
/** a database's handle: the files live in its file, and put, head, delete, copy and rename join its transactions; blobs.db when absent */
|
|
185
|
+
readonly database: readonly [4, import("./codec.ts").Codec<number, number>];
|
|
183
186
|
}>;
|
|
184
187
|
/**
|
|
185
188
|
* Where a call stands: the files, a folder's segments, a path within it.
|
|
@@ -529,6 +532,8 @@ export declare const StoreOptions: import("./codec.ts").Message<{
|
|
|
529
532
|
readonly withoutSystem: readonly [14, import("./codec.ts").Codec<boolean, boolean>];
|
|
530
533
|
/** the metrics series the store holds at most; 100,000 when absent */
|
|
531
534
|
readonly maxSeries: readonly [15, import("./codec.ts").Codec<number, number>];
|
|
535
|
+
/** the files of the store's older keys, which open the values they sealed and seal none */
|
|
536
|
+
readonly olderEncryptionKeys: readonly [16, import("./codec.ts").Codec<string[], readonly string[]>];
|
|
532
537
|
}>;
|
|
533
538
|
/**
|
|
534
539
|
* The last frame of a connection.
|
|
@@ -953,6 +958,12 @@ export declare const KvWritten: import("./codec.ts").Message<{
|
|
|
953
958
|
export declare const KvFound: import("./codec.ts").Message<{
|
|
954
959
|
readonly found: readonly [1, import("./codec.ts").Codec<boolean, boolean>];
|
|
955
960
|
}>;
|
|
961
|
+
/**
|
|
962
|
+
* How many values a reseal sealed again with the sealing key.
|
|
963
|
+
*/
|
|
964
|
+
export declare const KvResealed: import("./codec.ts").Message<{
|
|
965
|
+
readonly count: readonly [1, import("./codec.ts").Codec<number, number>];
|
|
966
|
+
}>;
|
|
956
967
|
/**
|
|
957
968
|
* A branch of a handle, for a clear.
|
|
958
969
|
*/
|
|
@@ -2388,6 +2399,8 @@ export declare const SearchQuery: import("./codec.ts").Message<{
|
|
|
2388
2399
|
readonly offset: readonly [7, import("./codec.ts").Codec<number, number>];
|
|
2389
2400
|
/** 10 when absent; offset and limit together 1000 at most */
|
|
2390
2401
|
readonly limit: readonly [8, import("./codec.ts").Codec<number, number>];
|
|
2402
|
+
/** the last word read as the start of a word too, as a search box types */
|
|
2403
|
+
readonly prefix: readonly [9, import("./codec.ts").Codec<boolean, boolean>];
|
|
2391
2404
|
}>;
|
|
2392
2405
|
/**
|
|
2393
2406
|
* A hit: a document and its value, or a table's row, a value a column of
|
package/dist/wire/protocol.js
CHANGED
|
@@ -57,6 +57,7 @@ export const methods = {
|
|
|
57
57
|
'kv.expire': 0x0108,
|
|
58
58
|
'kv.clear': 0x0109,
|
|
59
59
|
'kv.list': 0x010a,
|
|
60
|
+
'kv.reseal': 0x010b,
|
|
60
61
|
'kv.counters.open': 0x0110,
|
|
61
62
|
'kv.counters.add': 0x0111,
|
|
62
63
|
'kv.counters.get': 0x0112,
|
|
@@ -291,6 +292,8 @@ export const BlobsOpen = message('blobs.Open', {
|
|
|
291
292
|
ttl: [2, uint],
|
|
292
293
|
/** the largest file a write may leave */
|
|
293
294
|
maxFileSize: [3, uint],
|
|
295
|
+
/** a database's handle: the files live in its file, and put, head, delete, copy and rename join its transactions; blobs.db when absent */
|
|
296
|
+
database: [4, uint],
|
|
294
297
|
}, {
|
|
295
298
|
write(w, v) {
|
|
296
299
|
const head = w.openMap();
|
|
@@ -310,12 +313,18 @@ export const BlobsOpen = message('blobs.Open', {
|
|
|
310
313
|
w.uint(v.maxFileSize);
|
|
311
314
|
n++;
|
|
312
315
|
}
|
|
316
|
+
if (v.database !== undefined) {
|
|
317
|
+
w.field(4);
|
|
318
|
+
w.uint(v.database);
|
|
319
|
+
n++;
|
|
320
|
+
}
|
|
313
321
|
w.closeMap(head, n);
|
|
314
322
|
},
|
|
315
323
|
read(r) {
|
|
316
324
|
let $name;
|
|
317
325
|
let $ttl;
|
|
318
326
|
let $maxFileSize;
|
|
327
|
+
let $database;
|
|
319
328
|
for (let n = r.message(); n > 0; n--) {
|
|
320
329
|
switch (r.field()) {
|
|
321
330
|
case 1:
|
|
@@ -327,12 +336,15 @@ export const BlobsOpen = message('blobs.Open', {
|
|
|
327
336
|
case 3:
|
|
328
337
|
$maxFileSize = r.uint();
|
|
329
338
|
break;
|
|
339
|
+
case 4:
|
|
340
|
+
$database = r.uint();
|
|
341
|
+
break;
|
|
330
342
|
default:
|
|
331
343
|
r.skip();
|
|
332
344
|
}
|
|
333
345
|
}
|
|
334
346
|
r.leave();
|
|
335
|
-
return { name: $name, ttl: $ttl, maxFileSize: $maxFileSize };
|
|
347
|
+
return { name: $name, ttl: $ttl, maxFileSize: $maxFileSize, database: $database };
|
|
336
348
|
},
|
|
337
349
|
});
|
|
338
350
|
/**
|
|
@@ -2111,6 +2123,8 @@ export const StoreOptions = message('store.Options', {
|
|
|
2111
2123
|
withoutSystem: [14, bool],
|
|
2112
2124
|
/** the metrics series the store holds at most; 100,000 when absent */
|
|
2113
2125
|
maxSeries: [15, uint],
|
|
2126
|
+
/** the files of the store's older keys, which open the values they sealed and seal none */
|
|
2127
|
+
olderEncryptionKeys: [16, list(str)],
|
|
2114
2128
|
}, {
|
|
2115
2129
|
write(w, v) {
|
|
2116
2130
|
const head = w.openMap();
|
|
@@ -2199,6 +2213,14 @@ export const StoreOptions = message('store.Options', {
|
|
|
2199
2213
|
w.uint(v.maxSeries);
|
|
2200
2214
|
n++;
|
|
2201
2215
|
}
|
|
2216
|
+
if (v.olderEncryptionKeys !== undefined) {
|
|
2217
|
+
w.field(16);
|
|
2218
|
+
w.array(v.olderEncryptionKeys.length);
|
|
2219
|
+
for (const item0 of v.olderEncryptionKeys) {
|
|
2220
|
+
w.str(item0);
|
|
2221
|
+
}
|
|
2222
|
+
n++;
|
|
2223
|
+
}
|
|
2202
2224
|
w.closeMap(head, n);
|
|
2203
2225
|
},
|
|
2204
2226
|
read(r) {
|
|
@@ -2217,6 +2239,7 @@ export const StoreOptions = message('store.Options', {
|
|
|
2217
2239
|
let $selfMetrics;
|
|
2218
2240
|
let $withoutSystem;
|
|
2219
2241
|
let $maxSeries;
|
|
2242
|
+
let $olderEncryptionKeys;
|
|
2220
2243
|
for (let n = r.message(); n > 0; n--) {
|
|
2221
2244
|
switch (r.field()) {
|
|
2222
2245
|
case 1:
|
|
@@ -2282,6 +2305,15 @@ export const StoreOptions = message('store.Options', {
|
|
|
2282
2305
|
case 15:
|
|
2283
2306
|
$maxSeries = r.uint();
|
|
2284
2307
|
break;
|
|
2308
|
+
case 16: {
|
|
2309
|
+
const items0 = [];
|
|
2310
|
+
for (let i0 = r.array(); i0 > 0; i0--) {
|
|
2311
|
+
items0.push(r.str());
|
|
2312
|
+
}
|
|
2313
|
+
r.leave();
|
|
2314
|
+
$olderEncryptionKeys = items0;
|
|
2315
|
+
break;
|
|
2316
|
+
}
|
|
2285
2317
|
default:
|
|
2286
2318
|
r.skip();
|
|
2287
2319
|
}
|
|
@@ -2303,6 +2335,7 @@ export const StoreOptions = message('store.Options', {
|
|
|
2303
2335
|
selfMetrics: $selfMetrics,
|
|
2304
2336
|
withoutSystem: $withoutSystem,
|
|
2305
2337
|
maxSeries: $maxSeries,
|
|
2338
|
+
olderEncryptionKeys: $olderEncryptionKeys,
|
|
2306
2339
|
};
|
|
2307
2340
|
},
|
|
2308
2341
|
});
|
|
@@ -4145,6 +4178,37 @@ export const KvFound = message('kv.Found', {
|
|
|
4145
4178
|
return { found: $found };
|
|
4146
4179
|
},
|
|
4147
4180
|
});
|
|
4181
|
+
/**
|
|
4182
|
+
* How many values a reseal sealed again with the sealing key.
|
|
4183
|
+
*/
|
|
4184
|
+
export const KvResealed = message('kv.Resealed', {
|
|
4185
|
+
count: [1, uint],
|
|
4186
|
+
}, {
|
|
4187
|
+
write(w, v) {
|
|
4188
|
+
const head = w.openMap();
|
|
4189
|
+
let n = 0;
|
|
4190
|
+
if (v.count !== undefined) {
|
|
4191
|
+
w.field(1);
|
|
4192
|
+
w.uint(v.count);
|
|
4193
|
+
n++;
|
|
4194
|
+
}
|
|
4195
|
+
w.closeMap(head, n);
|
|
4196
|
+
},
|
|
4197
|
+
read(r) {
|
|
4198
|
+
let $count;
|
|
4199
|
+
for (let n = r.message(); n > 0; n--) {
|
|
4200
|
+
switch (r.field()) {
|
|
4201
|
+
case 1:
|
|
4202
|
+
$count = r.uint();
|
|
4203
|
+
break;
|
|
4204
|
+
default:
|
|
4205
|
+
r.skip();
|
|
4206
|
+
}
|
|
4207
|
+
}
|
|
4208
|
+
r.leave();
|
|
4209
|
+
return { count: $count };
|
|
4210
|
+
},
|
|
4211
|
+
});
|
|
4148
4212
|
/**
|
|
4149
4213
|
* A branch of a handle, for a clear.
|
|
4150
4214
|
*/
|
|
@@ -9424,6 +9488,8 @@ export const SearchQuery = message('search.Query', {
|
|
|
9424
9488
|
offset: [7, uint],
|
|
9425
9489
|
/** 10 when absent; offset and limit together 1000 at most */
|
|
9426
9490
|
limit: [8, uint],
|
|
9491
|
+
/** the last word read as the start of a word too, as a search box types */
|
|
9492
|
+
prefix: [9, bool],
|
|
9427
9493
|
}, {
|
|
9428
9494
|
write(w, v) {
|
|
9429
9495
|
const head = w.openMap();
|
|
@@ -9468,6 +9534,11 @@ export const SearchQuery = message('search.Query', {
|
|
|
9468
9534
|
w.uint(v.limit);
|
|
9469
9535
|
n++;
|
|
9470
9536
|
}
|
|
9537
|
+
if (v.prefix !== undefined) {
|
|
9538
|
+
w.field(9);
|
|
9539
|
+
w.bool(v.prefix);
|
|
9540
|
+
n++;
|
|
9541
|
+
}
|
|
9471
9542
|
w.closeMap(head, n);
|
|
9472
9543
|
},
|
|
9473
9544
|
read(r) {
|
|
@@ -9479,6 +9550,7 @@ export const SearchQuery = message('search.Query', {
|
|
|
9479
9550
|
let $where;
|
|
9480
9551
|
let $offset;
|
|
9481
9552
|
let $limit;
|
|
9553
|
+
let $prefix;
|
|
9482
9554
|
for (let n = r.message(); n > 0; n--) {
|
|
9483
9555
|
switch (r.field()) {
|
|
9484
9556
|
case 1:
|
|
@@ -9505,6 +9577,9 @@ export const SearchQuery = message('search.Query', {
|
|
|
9505
9577
|
case 8:
|
|
9506
9578
|
$limit = r.uint();
|
|
9507
9579
|
break;
|
|
9580
|
+
case 9:
|
|
9581
|
+
$prefix = r.bool();
|
|
9582
|
+
break;
|
|
9508
9583
|
default:
|
|
9509
9584
|
r.skip();
|
|
9510
9585
|
}
|
|
@@ -9519,6 +9594,7 @@ export const SearchQuery = message('search.Query', {
|
|
|
9519
9594
|
where: $where,
|
|
9520
9595
|
offset: $offset,
|
|
9521
9596
|
limit: $limit,
|
|
9597
|
+
prefix: $prefix,
|
|
9522
9598
|
};
|
|
9523
9599
|
},
|
|
9524
9600
|
});
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@tinyshed/tinystore",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.4.1",
|
|
4
4
|
"description": "TinyStore for Bun and Node: kv, jobs, blobs, SQL, records and metrics in one directory, served by a sidecar",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"homepage": "https://tinyshed.org/tinystore",
|
|
@@ -31,12 +31,12 @@
|
|
|
31
31
|
"README.md"
|
|
32
32
|
],
|
|
33
33
|
"optionalDependencies": {
|
|
34
|
-
"@tinyshed/tinystore-linux-x64": "0.
|
|
35
|
-
"@tinyshed/tinystore-linux-arm64": "0.
|
|
36
|
-
"@tinyshed/tinystore-linux-x64-musl": "0.
|
|
37
|
-
"@tinyshed/tinystore-linux-arm64-musl": "0.
|
|
38
|
-
"@tinyshed/tinystore-darwin-arm64": "0.
|
|
39
|
-
"@tinyshed/tinystore-darwin-x64": "0.
|
|
40
|
-
"@tinyshed/tinystore-win32-x64": "0.
|
|
34
|
+
"@tinyshed/tinystore-linux-x64": "0.4.1",
|
|
35
|
+
"@tinyshed/tinystore-linux-arm64": "0.4.1",
|
|
36
|
+
"@tinyshed/tinystore-linux-x64-musl": "0.4.1",
|
|
37
|
+
"@tinyshed/tinystore-linux-arm64-musl": "0.4.1",
|
|
38
|
+
"@tinyshed/tinystore-darwin-arm64": "0.4.1",
|
|
39
|
+
"@tinyshed/tinystore-darwin-x64": "0.4.1",
|
|
40
|
+
"@tinyshed/tinystore-win32-x64": "0.4.1"
|
|
41
41
|
}
|
|
42
42
|
}
|
package/src/blobs.ts
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
|
|
6
6
|
import { type Connection, type Idempotence, type Link, pieceOf } from './connection.ts'
|
|
7
7
|
import { ClosedError, InvalidError, OutcomeUnknownError } from './errors.ts'
|
|
8
|
-
import { checkName, handleOn, type Via, viaLink } from './handles.ts'
|
|
8
|
+
import { checkName, type Home, handleOn, type Open, type Via, viaLink } from './handles.ts'
|
|
9
9
|
import { ConnectionLostError, type Stream, watch } from './session.ts'
|
|
10
10
|
import { type Duration, dateOf, ms, unixMs } from './time.ts'
|
|
11
11
|
import {
|
|
@@ -96,13 +96,35 @@ const nothing = new Uint8Array(0)
|
|
|
96
96
|
|
|
97
97
|
/** Files as `store.files` opens them. */
|
|
98
98
|
export function openFiles(link: Link, name: string, options?: FilesOptions): Files {
|
|
99
|
+
return new Files(link, name, filesOpen(name, options), [])
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/** The open of a set of files, kept in the database of the handle `database` when it is given. */
|
|
103
|
+
export function filesOpen(name: string, options?: FilesOptions, database?: number): Uint8Array {
|
|
99
104
|
checkName(name, 'files')
|
|
100
|
-
|
|
105
|
+
return BlobsOpen.encode({
|
|
101
106
|
name,
|
|
102
107
|
ttl: options?.ttl === undefined ? undefined : ms(options.ttl),
|
|
103
108
|
maxFileSize: options?.maxFileSize === undefined ? undefined : bytesOf(options.maxFileSize),
|
|
109
|
+
database,
|
|
104
110
|
})
|
|
105
|
-
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/** What a transaction needs of a set of files it takes in. */
|
|
114
|
+
export interface FilesParts {
|
|
115
|
+
readonly name: string
|
|
116
|
+
readonly open: Open
|
|
117
|
+
/** the database whose file keeps the files; the store's own when undefined */
|
|
118
|
+
readonly home: Home | undefined
|
|
119
|
+
/** the same files, their calls through the transaction's `via` */
|
|
120
|
+
inside(via: Via): Files
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
// a set of files' parts, kept out of its public shape, for db.tx
|
|
124
|
+
const parts = new WeakMap<object, FilesParts>()
|
|
125
|
+
|
|
126
|
+
export function filesParts(handle: object): FilesParts | undefined {
|
|
127
|
+
return parts.get(handle)
|
|
106
128
|
}
|
|
107
129
|
|
|
108
130
|
/**
|
|
@@ -113,24 +135,49 @@ export class Files {
|
|
|
113
135
|
readonly name: string
|
|
114
136
|
readonly #link: Link
|
|
115
137
|
readonly #via: Via
|
|
116
|
-
readonly #open:
|
|
138
|
+
readonly #open: Open
|
|
117
139
|
readonly #folder: string[]
|
|
140
|
+
readonly #home: Home | undefined
|
|
141
|
+
/** whether its calls go through a database's transaction */
|
|
142
|
+
readonly #inside: boolean
|
|
118
143
|
|
|
119
|
-
/**
|
|
120
|
-
|
|
144
|
+
/**
|
|
145
|
+
* Files come from `store.files` and `db.files`; inside a database's
|
|
146
|
+
* transaction, from `tx.with(files)`, whose writes go through the
|
|
147
|
+
* transaction.
|
|
148
|
+
*/
|
|
149
|
+
constructor(link: Link, name: string, open: Open, folder: string[], home?: Home, via?: Via) {
|
|
121
150
|
this.name = name
|
|
122
151
|
this.#link = link
|
|
123
|
-
this.#via = viaLink(link)
|
|
152
|
+
this.#via = via ?? home?.via(`files ${name}`) ?? viaLink(link)
|
|
124
153
|
this.#open = open
|
|
125
154
|
this.#folder = folder
|
|
155
|
+
this.#home = home
|
|
156
|
+
this.#inside = via !== undefined
|
|
157
|
+
const inside = (through: Via) => new Files(link, name, open, folder, home, through)
|
|
158
|
+
parts.set(this, { name, open, home, inside })
|
|
126
159
|
}
|
|
127
160
|
|
|
128
161
|
/** The folder of these segments, one each: `folder('users', 42)` is users/42, never users/420. */
|
|
129
162
|
folder(...segments: (string | number | bigint)[]): Files {
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
163
|
+
const folder = [...this.#folder, ...segments.map(segmentOf)]
|
|
164
|
+
return new Files(
|
|
165
|
+
this.#link,
|
|
166
|
+
this.name,
|
|
167
|
+
this.#open,
|
|
168
|
+
folder,
|
|
169
|
+
this.#home,
|
|
170
|
+
this.#inside ? this.#via : undefined,
|
|
171
|
+
)
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
/** Refuses what streams a file's bytes inside a transaction, which takes a put's bytes whole. */
|
|
175
|
+
#outside(what: string): void {
|
|
176
|
+
if (this.#inside) {
|
|
177
|
+
throw new InvalidError(
|
|
178
|
+
`files ${this.name}: ${what} inside a transaction: put a file's bytes whole, and read them after its commit`,
|
|
179
|
+
)
|
|
180
|
+
}
|
|
134
181
|
}
|
|
135
182
|
|
|
136
183
|
/**
|
|
@@ -277,6 +324,7 @@ export class Files {
|
|
|
277
324
|
* leaving the block without commit aborts it.
|
|
278
325
|
*/
|
|
279
326
|
async upload(path: string, options?: FileOptions): Promise<Upload> {
|
|
327
|
+
this.#outside('an upload')
|
|
280
328
|
const connection = await this.#link.connection()
|
|
281
329
|
const write = BlobsWrite.encode({
|
|
282
330
|
...this.#fields(path, options),
|
|
@@ -396,6 +444,15 @@ export class Files {
|
|
|
396
444
|
size: options?.size ?? source.size,
|
|
397
445
|
create,
|
|
398
446
|
}
|
|
447
|
+
if (this.#inside) {
|
|
448
|
+
const bytes = await allOf(source)
|
|
449
|
+
const answer = await this.#call(
|
|
450
|
+
'blobs.put',
|
|
451
|
+
handle => BlobsWrite.encode({ ...fields, handle, bytes }),
|
|
452
|
+
'write',
|
|
453
|
+
)
|
|
454
|
+
return writtenOf(answer)
|
|
455
|
+
}
|
|
399
456
|
return this.#link.run(
|
|
400
457
|
'write',
|
|
401
458
|
async connection => {
|
|
@@ -688,6 +745,26 @@ function sourceOf(body: Body): Source {
|
|
|
688
745
|
}
|
|
689
746
|
}
|
|
690
747
|
|
|
748
|
+
/** Every byte of a body at once, as a put inside a transaction sends it. */
|
|
749
|
+
async function allOf(source: Source): Promise<Uint8Array> {
|
|
750
|
+
const chunks: Uint8Array[] = []
|
|
751
|
+
let size = 0
|
|
752
|
+
for await (const chunk of source.chunks()) {
|
|
753
|
+
chunks.push(chunk)
|
|
754
|
+
size += chunk.length
|
|
755
|
+
}
|
|
756
|
+
if (chunks.length === 1) {
|
|
757
|
+
return chunks[0] as Uint8Array
|
|
758
|
+
}
|
|
759
|
+
const all = new Uint8Array(size)
|
|
760
|
+
let at = 0
|
|
761
|
+
for (const chunk of chunks) {
|
|
762
|
+
all.set(chunk, at)
|
|
763
|
+
at += chunk.length
|
|
764
|
+
}
|
|
765
|
+
return all
|
|
766
|
+
}
|
|
767
|
+
|
|
691
768
|
/**
|
|
692
769
|
* The bytes of chunks again in pieces of `size` bytes, the last shorter. A
|
|
693
770
|
* chunk that holds a whole piece goes as it is, without a copy.
|
package/src/connection.ts
CHANGED
|
@@ -293,6 +293,8 @@ export interface Started {
|
|
|
293
293
|
durability?: Durability | undefined
|
|
294
294
|
/** the file of the store's encryption key, when it is not the store's own */
|
|
295
295
|
encryptionKeyFile?: string | undefined
|
|
296
|
+
/** the files of the store's older keys, which open what they sealed */
|
|
297
|
+
olderEncryptionKeys?: string[] | undefined
|
|
296
298
|
/** the node the store is, when it is one */
|
|
297
299
|
node?: NodeOptions | undefined
|
|
298
300
|
/** measures the store into its own metrics, the machine too unless it says system: false */
|
|
@@ -319,8 +321,13 @@ function selfMetricsOf(given: SelfMetrics | undefined): { system: boolean } | un
|
|
|
319
321
|
function storeFlags(started: Started, dir: string): string[] {
|
|
320
322
|
const committing = started.durability === undefined ? [] : ['--durability', started.durability]
|
|
321
323
|
const key = started.encryptionKeyFile
|
|
322
|
-
const
|
|
324
|
+
const sealing =
|
|
323
325
|
key === undefined ? committing : [...committing, '--encryption-key-file', resolve(key)]
|
|
326
|
+
const older = (started.olderEncryptionKeys ?? []).flatMap(file => [
|
|
327
|
+
'--older-encryption-key',
|
|
328
|
+
resolve(file),
|
|
329
|
+
])
|
|
330
|
+
const keyed = [...sealing, ...older]
|
|
324
331
|
const asked = selfMetricsOf(started.selfMetrics)
|
|
325
332
|
const measured =
|
|
326
333
|
asked === undefined
|
|
@@ -743,6 +750,7 @@ export function embedded(
|
|
|
743
750
|
const options = StoreOptions.encode({
|
|
744
751
|
durability: started.durability,
|
|
745
752
|
encryptionKeyFile,
|
|
753
|
+
olderEncryptionKeys: (started.olderEncryptionKeys ?? []).map(file => resolve(file)),
|
|
746
754
|
node: node !== undefined,
|
|
747
755
|
from: node?.from ?? [],
|
|
748
756
|
serve: node?.serve,
|
package/src/kv.ts
CHANGED
|
@@ -27,6 +27,7 @@ import {
|
|
|
27
27
|
KvFound,
|
|
28
28
|
KvList,
|
|
29
29
|
KvPage,
|
|
30
|
+
KvResealed,
|
|
30
31
|
KvWritten,
|
|
31
32
|
type Method,
|
|
32
33
|
methods,
|
|
@@ -239,6 +240,26 @@ export class Bucket<T> {
|
|
|
239
240
|
)
|
|
240
241
|
}
|
|
241
242
|
|
|
243
|
+
/**
|
|
244
|
+
* Seals again with the store's sealing key every value of this branch, and
|
|
245
|
+
* of the branches under it, that an older key sealed, and says how many.
|
|
246
|
+
* Once every value of the bucket is, the older key can leave the store's
|
|
247
|
+
* options. A value sealed with a key the store does not hold stops it with
|
|
248
|
+
* an `InvalidError` that names the key, and a reseal after the key is given
|
|
249
|
+
* goes on.
|
|
250
|
+
*/
|
|
251
|
+
async reseal(): Promise<number> {
|
|
252
|
+
const under = underField(this.#under)
|
|
253
|
+
const answer = await this.#via.call(
|
|
254
|
+
methods['kv.bucket.open'],
|
|
255
|
+
this.#open,
|
|
256
|
+
methods['kv.reseal'],
|
|
257
|
+
handle => KvBranch.encode({ handle, under }),
|
|
258
|
+
'write',
|
|
259
|
+
)
|
|
260
|
+
return KvResealed.decode(answer).count ?? 0
|
|
261
|
+
}
|
|
262
|
+
|
|
242
263
|
/** One page of this branch's own keys, in the byte order of their text: `"10"` before `"9"`. */
|
|
243
264
|
async list(options?: { limit?: number | undefined; after?: Key | undefined }): Promise<Page<T>> {
|
|
244
265
|
const under = underField(this.#under)
|
package/src/logger.ts
CHANGED
|
@@ -460,12 +460,15 @@ export class Logger {
|
|
|
460
460
|
this.#emit({ event: name }, attrs)
|
|
461
461
|
}
|
|
462
462
|
|
|
463
|
-
/**
|
|
463
|
+
/**
|
|
464
|
+
* The logger of the same stream, buffer and console, its context these
|
|
465
|
+
* fields besides its own: a key it had already takes the new value where
|
|
466
|
+
* it stood, as an object spread does, where it prints, in what is kept
|
|
467
|
+
* and for a query's `where`.
|
|
468
|
+
*/
|
|
464
469
|
with(context: Fields): Logger {
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
...encodeFields(context, this.#shared.redact, this.#shared.replace),
|
|
468
|
-
])
|
|
470
|
+
const added = encodeFields(context, this.#shared.redact, this.#shared.replace)
|
|
471
|
+
return new Logger(this.#shared, onceEach(this.#context, added))
|
|
469
472
|
}
|
|
470
473
|
|
|
471
474
|
/** Hands what the buffer holds to the server now, as the timer does every second. */
|
|
@@ -543,6 +546,27 @@ export class Logger {
|
|
|
543
546
|
}
|
|
544
547
|
}
|
|
545
548
|
|
|
549
|
+
/**
|
|
550
|
+
* A context's key, JSON pairs with `added`'s after them, each key once: where
|
|
551
|
+
* it was first given, holding the value given last.
|
|
552
|
+
*
|
|
553
|
+
* ```text
|
|
554
|
+
* component=http user=7, then component=access → component=access user=7
|
|
555
|
+
* ```
|
|
556
|
+
*/
|
|
557
|
+
function onceEach(context: readonly string[], added: readonly string[]): string[] {
|
|
558
|
+
const once = [...context]
|
|
559
|
+
for (let i = 0; i < added.length; i += 2) {
|
|
560
|
+
const at = once.findIndex((key, place) => place % 2 === 0 && key === added[i])
|
|
561
|
+
if (at === -1) {
|
|
562
|
+
once.push(added[i] as string, added[i + 1] as string)
|
|
563
|
+
} else {
|
|
564
|
+
once[at + 1] = added[i + 1] as string
|
|
565
|
+
}
|
|
566
|
+
}
|
|
567
|
+
return once
|
|
568
|
+
}
|
|
569
|
+
|
|
546
570
|
/** Whether a logger prints a line of level: it has a console, and the level is the level printed or above. */
|
|
547
571
|
function prints(shared: Shared, level: number): boolean {
|
|
548
572
|
return shared.print !== undefined && level >= (shared.printed ?? shared.kept)
|
package/src/search.ts
CHANGED
|
@@ -85,6 +85,12 @@ export interface Query {
|
|
|
85
85
|
where?: Where | undefined
|
|
86
86
|
/** this search's way of combining its words and its vector, in place of the index's */
|
|
87
87
|
hybrid?: { vectorWeight?: number | undefined; rerank?: boolean | undefined } | undefined
|
|
88
|
+
/**
|
|
89
|
+
* the last word read as the start of a word too, as a search box types:
|
|
90
|
+
* `crypt` finds `cryptography` as well, at half its field's weight; a
|
|
91
|
+
* word a space follows is whole, and so is one of a single letter
|
|
92
|
+
*/
|
|
93
|
+
prefix?: boolean | undefined
|
|
88
94
|
/** 10 unless said; offset and limit together 1000 at most */
|
|
89
95
|
limit?: number | undefined
|
|
90
96
|
offset?: number | undefined
|
|
@@ -187,6 +193,7 @@ abstract class Searchable<T> {
|
|
|
187
193
|
vector,
|
|
188
194
|
vectorWeight: asked.hybrid?.vectorWeight,
|
|
189
195
|
rerank: asked.hybrid?.rerank,
|
|
196
|
+
prefix: asked.prefix,
|
|
190
197
|
where,
|
|
191
198
|
offset: asked.offset,
|
|
192
199
|
limit: asked.limit,
|
package/src/sql.ts
CHANGED
|
@@ -8,6 +8,7 @@ import { AsyncLocalStorage } from 'node:async_hooks'
|
|
|
8
8
|
import { readdir, readFile } from 'node:fs/promises'
|
|
9
9
|
import { join } from 'node:path'
|
|
10
10
|
|
|
11
|
+
import { Files, type FilesOptions, filesOpen, filesParts } from './blobs.ts'
|
|
11
12
|
import { type Channel, channelParts, type TxChannel, txChannel } from './channels.ts'
|
|
12
13
|
import type { Connection, Durability, Idempotence, Link } from './connection.ts'
|
|
13
14
|
import { checkDurability, download } from './connection.ts'
|
|
@@ -774,6 +775,22 @@ export class Database implements Runner, Home {
|
|
|
774
775
|
return openQueue(this.#link, this.#workers, name, options, this)
|
|
775
776
|
}
|
|
776
777
|
|
|
778
|
+
/**
|
|
779
|
+
* A set of files kept in this database's file, as `store.files` keeps one
|
|
780
|
+
* in blobs.db: the same calls, and its writes commit with the rows in
|
|
781
|
+
* `db.tx`, through `tx.with(files)`. A file's bytes past 16 KiB lie in the
|
|
782
|
+
* folder beside the database's file.
|
|
783
|
+
*
|
|
784
|
+
* const packages = db.files('packages', { maxFileSize: '10MiB' })
|
|
785
|
+
*/
|
|
786
|
+
files(name: string, options?: FilesOptions): Files {
|
|
787
|
+
const bytes = filesOpen(name, options)
|
|
788
|
+
const open = this.keep(methods['blobs.open'], bytes, async connection =>
|
|
789
|
+
filesOpen(name, options, await this.handle(connection)),
|
|
790
|
+
)
|
|
791
|
+
return new Files(this.#link, name, open, [], this)
|
|
792
|
+
}
|
|
793
|
+
|
|
777
794
|
/**
|
|
778
795
|
* An index of a table of this database, kept in its file: its rows found
|
|
779
796
|
* again by their words, vectors or both, each hit's value its row as
|
|
@@ -1059,9 +1076,10 @@ export class SqlTx implements Runner {
|
|
|
1059
1076
|
with<T>(bucket: Bucket<T>): Bucket<T>
|
|
1060
1077
|
with<T>(queue: Queue<T>): TxQueue<T>
|
|
1061
1078
|
with<T>(channel: Channel<T>): TxChannel<T>
|
|
1079
|
+
with(files: Files): Files
|
|
1062
1080
|
with(
|
|
1063
|
-
handle: Bucket<unknown> | Queue<unknown> | Channel<unknown
|
|
1064
|
-
): Bucket<unknown> | TxQueue<unknown> | TxChannel<unknown> {
|
|
1081
|
+
handle: Bucket<unknown> | Queue<unknown> | Channel<unknown> | Files,
|
|
1082
|
+
): Bucket<unknown> | TxQueue<unknown> | TxChannel<unknown> | Files {
|
|
1065
1083
|
const channel = channelParts(handle)
|
|
1066
1084
|
if (channel !== undefined) {
|
|
1067
1085
|
// a message published here is handed over once the transaction commits, at the writer
|
|
@@ -1080,8 +1098,13 @@ export class SqlTx implements Runner {
|
|
|
1080
1098
|
this.#takes(queue.source.home, queue.source.open, `jobs queue ${queue.name}`)
|
|
1081
1099
|
return queueVia(queue, this.#via)
|
|
1082
1100
|
}
|
|
1101
|
+
const files = filesParts(handle)
|
|
1102
|
+
if (files !== undefined) {
|
|
1103
|
+
this.#takes(files.home, files.open, `files ${files.name}`)
|
|
1104
|
+
return files.inside(this.#via)
|
|
1105
|
+
}
|
|
1083
1106
|
throw new InvalidError(
|
|
1084
|
-
`${this.describe}: a transaction takes in a bucket
|
|
1107
|
+
`${this.describe}: a transaction takes in a bucket, a queue or files opened from its database`,
|
|
1085
1108
|
)
|
|
1086
1109
|
}
|
|
1087
1110
|
|
package/src/store.ts
CHANGED
|
@@ -142,6 +142,14 @@ export interface OpenOptions {
|
|
|
142
142
|
* running keeps the key it was started with.
|
|
143
143
|
*/
|
|
144
144
|
encryptionKeyFile?: string
|
|
145
|
+
/**
|
|
146
|
+
* The files of the store's older keys, which open the values they sealed
|
|
147
|
+
* and seal nothing new: a key being rotated out, until `reseal` has sealed
|
|
148
|
+
* every value again, or one found after a restore that left it out. Each
|
|
149
|
+
* file must be there. A sidecar found running keeps the keys it was
|
|
150
|
+
* started with.
|
|
151
|
+
*/
|
|
152
|
+
olderEncryptionKeys?: string[]
|
|
145
153
|
/**
|
|
146
154
|
* Makes the store a node: it keeps a log of each file it writes, serves
|
|
147
155
|
* its files to other nodes, and copies the files of the node it takes
|
|
@@ -573,6 +581,7 @@ export async function openWith(
|
|
|
573
581
|
const started = {
|
|
574
582
|
durability,
|
|
575
583
|
encryptionKeyFile: options.encryptionKeyFile,
|
|
584
|
+
olderEncryptionKeys: options.olderEncryptionKeys,
|
|
576
585
|
node: options.node,
|
|
577
586
|
selfMetrics: options.selfMetrics,
|
|
578
587
|
maxSeries: options.maxSeries,
|
package/src/wire/protocol.ts
CHANGED
|
@@ -78,6 +78,7 @@ export const methods = {
|
|
|
78
78
|
'kv.expire': 0x0108,
|
|
79
79
|
'kv.clear': 0x0109,
|
|
80
80
|
'kv.list': 0x010a,
|
|
81
|
+
'kv.reseal': 0x010b,
|
|
81
82
|
'kv.counters.open': 0x0110,
|
|
82
83
|
'kv.counters.add': 0x0111,
|
|
83
84
|
'kv.counters.get': 0x0112,
|
|
@@ -332,6 +333,8 @@ export const BlobsOpen = message(
|
|
|
332
333
|
ttl: [2, uint],
|
|
333
334
|
/** the largest file a write may leave */
|
|
334
335
|
maxFileSize: [3, uint],
|
|
336
|
+
/** a database's handle: the files live in its file, and put, head, delete, copy and rename join its transactions; blobs.db when absent */
|
|
337
|
+
database: [4, uint],
|
|
335
338
|
},
|
|
336
339
|
{
|
|
337
340
|
write(w, v) {
|
|
@@ -352,12 +355,18 @@ export const BlobsOpen = message(
|
|
|
352
355
|
w.uint(v.maxFileSize)
|
|
353
356
|
n++
|
|
354
357
|
}
|
|
358
|
+
if (v.database !== undefined) {
|
|
359
|
+
w.field(4)
|
|
360
|
+
w.uint(v.database)
|
|
361
|
+
n++
|
|
362
|
+
}
|
|
355
363
|
w.closeMap(head, n)
|
|
356
364
|
},
|
|
357
365
|
read(r) {
|
|
358
366
|
let $name: string | undefined
|
|
359
367
|
let $ttl: number | undefined
|
|
360
368
|
let $maxFileSize: number | undefined
|
|
369
|
+
let $database: number | undefined
|
|
361
370
|
for (let n = r.message(); n > 0; n--) {
|
|
362
371
|
switch (r.field()) {
|
|
363
372
|
case 1:
|
|
@@ -369,12 +378,15 @@ export const BlobsOpen = message(
|
|
|
369
378
|
case 3:
|
|
370
379
|
$maxFileSize = r.uint()
|
|
371
380
|
break
|
|
381
|
+
case 4:
|
|
382
|
+
$database = r.uint()
|
|
383
|
+
break
|
|
372
384
|
default:
|
|
373
385
|
r.skip()
|
|
374
386
|
}
|
|
375
387
|
}
|
|
376
388
|
r.leave()
|
|
377
|
-
return { name: $name, ttl: $ttl, maxFileSize: $maxFileSize }
|
|
389
|
+
return { name: $name, ttl: $ttl, maxFileSize: $maxFileSize, database: $database }
|
|
378
390
|
},
|
|
379
391
|
},
|
|
380
392
|
)
|
|
@@ -2282,6 +2294,8 @@ export const StoreOptions = message(
|
|
|
2282
2294
|
withoutSystem: [14, bool],
|
|
2283
2295
|
/** the metrics series the store holds at most; 100,000 when absent */
|
|
2284
2296
|
maxSeries: [15, uint],
|
|
2297
|
+
/** the files of the store's older keys, which open the values they sealed and seal none */
|
|
2298
|
+
olderEncryptionKeys: [16, list(str)],
|
|
2285
2299
|
},
|
|
2286
2300
|
{
|
|
2287
2301
|
write(w, v) {
|
|
@@ -2371,6 +2385,14 @@ export const StoreOptions = message(
|
|
|
2371
2385
|
w.uint(v.maxSeries)
|
|
2372
2386
|
n++
|
|
2373
2387
|
}
|
|
2388
|
+
if (v.olderEncryptionKeys !== undefined) {
|
|
2389
|
+
w.field(16)
|
|
2390
|
+
w.array(v.olderEncryptionKeys.length)
|
|
2391
|
+
for (const item0 of v.olderEncryptionKeys) {
|
|
2392
|
+
w.str(item0)
|
|
2393
|
+
}
|
|
2394
|
+
n++
|
|
2395
|
+
}
|
|
2374
2396
|
w.closeMap(head, n)
|
|
2375
2397
|
},
|
|
2376
2398
|
read(r) {
|
|
@@ -2389,6 +2411,7 @@ export const StoreOptions = message(
|
|
|
2389
2411
|
let $selfMetrics: boolean | undefined
|
|
2390
2412
|
let $withoutSystem: boolean | undefined
|
|
2391
2413
|
let $maxSeries: number | undefined
|
|
2414
|
+
let $olderEncryptionKeys: string[] | undefined
|
|
2392
2415
|
for (let n = r.message(); n > 0; n--) {
|
|
2393
2416
|
switch (r.field()) {
|
|
2394
2417
|
case 1:
|
|
@@ -2454,6 +2477,15 @@ export const StoreOptions = message(
|
|
|
2454
2477
|
case 15:
|
|
2455
2478
|
$maxSeries = r.uint()
|
|
2456
2479
|
break
|
|
2480
|
+
case 16: {
|
|
2481
|
+
const items0: string[] = []
|
|
2482
|
+
for (let i0 = r.array(); i0 > 0; i0--) {
|
|
2483
|
+
items0.push(r.str())
|
|
2484
|
+
}
|
|
2485
|
+
r.leave()
|
|
2486
|
+
$olderEncryptionKeys = items0
|
|
2487
|
+
break
|
|
2488
|
+
}
|
|
2457
2489
|
default:
|
|
2458
2490
|
r.skip()
|
|
2459
2491
|
}
|
|
@@ -2475,6 +2507,7 @@ export const StoreOptions = message(
|
|
|
2475
2507
|
selfMetrics: $selfMetrics,
|
|
2476
2508
|
withoutSystem: $withoutSystem,
|
|
2477
2509
|
maxSeries: $maxSeries,
|
|
2510
|
+
olderEncryptionKeys: $olderEncryptionKeys,
|
|
2478
2511
|
}
|
|
2479
2512
|
},
|
|
2480
2513
|
},
|
|
@@ -4459,6 +4492,42 @@ export const KvFound = message(
|
|
|
4459
4492
|
},
|
|
4460
4493
|
)
|
|
4461
4494
|
|
|
4495
|
+
/**
|
|
4496
|
+
* How many values a reseal sealed again with the sealing key.
|
|
4497
|
+
*/
|
|
4498
|
+
export const KvResealed = message(
|
|
4499
|
+
'kv.Resealed',
|
|
4500
|
+
{
|
|
4501
|
+
count: [1, uint],
|
|
4502
|
+
},
|
|
4503
|
+
{
|
|
4504
|
+
write(w, v) {
|
|
4505
|
+
const head = w.openMap()
|
|
4506
|
+
let n = 0
|
|
4507
|
+
if (v.count !== undefined) {
|
|
4508
|
+
w.field(1)
|
|
4509
|
+
w.uint(v.count)
|
|
4510
|
+
n++
|
|
4511
|
+
}
|
|
4512
|
+
w.closeMap(head, n)
|
|
4513
|
+
},
|
|
4514
|
+
read(r) {
|
|
4515
|
+
let $count: number | undefined
|
|
4516
|
+
for (let n = r.message(); n > 0; n--) {
|
|
4517
|
+
switch (r.field()) {
|
|
4518
|
+
case 1:
|
|
4519
|
+
$count = r.uint()
|
|
4520
|
+
break
|
|
4521
|
+
default:
|
|
4522
|
+
r.skip()
|
|
4523
|
+
}
|
|
4524
|
+
}
|
|
4525
|
+
r.leave()
|
|
4526
|
+
return { count: $count }
|
|
4527
|
+
},
|
|
4528
|
+
},
|
|
4529
|
+
)
|
|
4530
|
+
|
|
4462
4531
|
/**
|
|
4463
4532
|
* A branch of a handle, for a clear.
|
|
4464
4533
|
*/
|
|
@@ -10170,6 +10239,8 @@ export const SearchQuery = message(
|
|
|
10170
10239
|
offset: [7, uint],
|
|
10171
10240
|
/** 10 when absent; offset and limit together 1000 at most */
|
|
10172
10241
|
limit: [8, uint],
|
|
10242
|
+
/** the last word read as the start of a word too, as a search box types */
|
|
10243
|
+
prefix: [9, bool],
|
|
10173
10244
|
},
|
|
10174
10245
|
{
|
|
10175
10246
|
write(w, v) {
|
|
@@ -10215,6 +10286,11 @@ export const SearchQuery = message(
|
|
|
10215
10286
|
w.uint(v.limit)
|
|
10216
10287
|
n++
|
|
10217
10288
|
}
|
|
10289
|
+
if (v.prefix !== undefined) {
|
|
10290
|
+
w.field(9)
|
|
10291
|
+
w.bool(v.prefix)
|
|
10292
|
+
n++
|
|
10293
|
+
}
|
|
10218
10294
|
w.closeMap(head, n)
|
|
10219
10295
|
},
|
|
10220
10296
|
read(r) {
|
|
@@ -10226,6 +10302,7 @@ export const SearchQuery = message(
|
|
|
10226
10302
|
let $where: string | undefined
|
|
10227
10303
|
let $offset: number | undefined
|
|
10228
10304
|
let $limit: number | undefined
|
|
10305
|
+
let $prefix: boolean | undefined
|
|
10229
10306
|
for (let n = r.message(); n > 0; n--) {
|
|
10230
10307
|
switch (r.field()) {
|
|
10231
10308
|
case 1:
|
|
@@ -10252,6 +10329,9 @@ export const SearchQuery = message(
|
|
|
10252
10329
|
case 8:
|
|
10253
10330
|
$limit = r.uint()
|
|
10254
10331
|
break
|
|
10332
|
+
case 9:
|
|
10333
|
+
$prefix = r.bool()
|
|
10334
|
+
break
|
|
10255
10335
|
default:
|
|
10256
10336
|
r.skip()
|
|
10257
10337
|
}
|
|
@@ -10266,6 +10346,7 @@ export const SearchQuery = message(
|
|
|
10266
10346
|
where: $where,
|
|
10267
10347
|
offset: $offset,
|
|
10268
10348
|
limit: $limit,
|
|
10349
|
+
prefix: $prefix,
|
|
10269
10350
|
}
|
|
10270
10351
|
},
|
|
10271
10352
|
},
|