@tinyshed/tinystore 0.3.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/blobs.d.ts CHANGED
@@ -1,4 +1,5 @@
1
1
  import { type Link } from './connection.ts';
2
+ import { type Home, type Open, type Via } from './handles.ts';
2
3
  import { type Stream } from './session.ts';
3
4
  import { type Duration } from './time.ts';
4
5
  /** A size in bytes: a number, or text in binary units, 64KiB or 2GiB; '2GB' does not compile. */
@@ -51,6 +52,18 @@ export interface FilePage {
51
52
  }
52
53
  /** Files as `store.files` opens them. */
53
54
  export declare function openFiles(link: Link, name: string, options?: FilesOptions): Files;
55
+ /** The open of a set of files, kept in the database of the handle `database` when it is given. */
56
+ export declare function filesOpen(name: string, options?: FilesOptions, database?: number): Uint8Array;
57
+ /** What a transaction needs of a set of files it takes in. */
58
+ export interface FilesParts {
59
+ readonly name: string;
60
+ readonly open: Open;
61
+ /** the database whose file keeps the files; the store's own when undefined */
62
+ readonly home: Home | undefined;
63
+ /** the same files, their calls through the transaction's `via` */
64
+ inside(via: Via): Files;
65
+ }
66
+ export declare function filesParts(handle: object): FilesParts | undefined;
54
67
  /**
55
68
  * A named set of files by path, or a folder of it, whose calls are the
56
69
  * folder's: a path in it is the folder's segments and then its own.
@@ -58,8 +71,12 @@ export declare function openFiles(link: Link, name: string, options?: FilesOptio
58
71
  export declare class Files {
59
72
  #private;
60
73
  readonly name: string;
61
- /** Files come from `store.files`. */
62
- constructor(link: Link, name: string, open: Uint8Array, folder: string[]);
74
+ /**
75
+ * Files come from `store.files` and `db.files`; inside a database's
76
+ * transaction, from `tx.with(files)`, whose writes go through the
77
+ * transaction.
78
+ */
79
+ constructor(link: Link, name: string, open: Open, folder: string[], home?: Home, via?: Via);
63
80
  /** The folder of these segments, one each: `folder('users', 42)` is users/42, never users/420. */
64
81
  folder(...segments: (string | number | bigint)[]): Files;
65
82
  /**
package/dist/blobs.js CHANGED
@@ -13,13 +13,22 @@ const decoder = new TextDecoder();
13
13
  const nothing = new Uint8Array(0);
14
14
  /** Files as `store.files` opens them. */
15
15
  export function openFiles(link, name, options) {
16
+ return new Files(link, name, filesOpen(name, options), []);
17
+ }
18
+ /** The open of a set of files, kept in the database of the handle `database` when it is given. */
19
+ export function filesOpen(name, options, database) {
16
20
  checkName(name, 'files');
17
- const open = BlobsOpen.encode({
21
+ return BlobsOpen.encode({
18
22
  name,
19
23
  ttl: options?.ttl === undefined ? undefined : ms(options.ttl),
20
24
  maxFileSize: options?.maxFileSize === undefined ? undefined : bytesOf(options.maxFileSize),
25
+ database,
21
26
  });
22
- return new Files(link, name, open, []);
27
+ }
28
+ // a set of files' parts, kept out of its public shape, for db.tx
29
+ const parts = new WeakMap();
30
+ export function filesParts(handle) {
31
+ return parts.get(handle);
23
32
  }
24
33
  /**
25
34
  * A named set of files by path, or a folder of it, whose calls are the
@@ -31,20 +40,35 @@ export class Files {
31
40
  #via;
32
41
  #open;
33
42
  #folder;
34
- /** Files come from `store.files`. */
35
- constructor(link, name, open, folder) {
43
+ #home;
44
+ /** whether its calls go through a database's transaction */
45
+ #inside;
46
+ /**
47
+ * Files come from `store.files` and `db.files`; inside a database's
48
+ * transaction, from `tx.with(files)`, whose writes go through the
49
+ * transaction.
50
+ */
51
+ constructor(link, name, open, folder, home, via) {
36
52
  this.name = name;
37
53
  this.#link = link;
38
- this.#via = viaLink(link);
54
+ this.#via = via ?? home?.via(`files ${name}`) ?? viaLink(link);
39
55
  this.#open = open;
40
56
  this.#folder = folder;
57
+ this.#home = home;
58
+ this.#inside = via !== undefined;
59
+ const inside = (through) => new Files(link, name, open, folder, home, through);
60
+ parts.set(this, { name, open, home, inside });
41
61
  }
42
62
  /** The folder of these segments, one each: `folder('users', 42)` is users/42, never users/420. */
43
63
  folder(...segments) {
44
- return new Files(this.#link, this.name, this.#open, [
45
- ...this.#folder,
46
- ...segments.map(segmentOf),
47
- ]);
64
+ const folder = [...this.#folder, ...segments.map(segmentOf)];
65
+ return new Files(this.#link, this.name, this.#open, folder, this.#home, this.#inside ? this.#via : undefined);
66
+ }
67
+ /** Refuses what streams a file's bytes inside a transaction, which takes a put's bytes whole. */
68
+ #outside(what) {
69
+ if (this.#inside) {
70
+ throw new InvalidError(`files ${this.name}: ${what} inside a transaction: put a file's bytes whole, and read them after its commit`);
71
+ }
48
72
  }
49
73
  /**
50
74
  * Writes the file whole and replaces whatever the path held, returning
@@ -146,6 +170,7 @@ export class Files {
146
170
  * leaving the block without commit aborts it.
147
171
  */
148
172
  async upload(path, options) {
173
+ this.#outside('an upload');
149
174
  const connection = await this.#link.connection();
150
175
  const write = BlobsWrite.encode({
151
176
  ...this.#fields(path, options),
@@ -236,6 +261,11 @@ export class Files {
236
261
  size: options?.size ?? source.size,
237
262
  create,
238
263
  };
264
+ if (this.#inside) {
265
+ const bytes = await allOf(source);
266
+ const answer = await this.#call('blobs.put', handle => BlobsWrite.encode({ ...fields, handle, bytes }), 'write');
267
+ return writtenOf(answer);
268
+ }
239
269
  return this.#link.run('write', async (connection) => {
240
270
  const handle = await this.#handle(connection);
241
271
  const piece = pieceOf(connection);
@@ -485,6 +515,25 @@ function sourceOf(body) {
485
515
  whole: async () => undefined,
486
516
  };
487
517
  }
518
+ /** Every byte of a body at once, as a put inside a transaction sends it. */
519
+ async function allOf(source) {
520
+ const chunks = [];
521
+ let size = 0;
522
+ for await (const chunk of source.chunks()) {
523
+ chunks.push(chunk);
524
+ size += chunk.length;
525
+ }
526
+ if (chunks.length === 1) {
527
+ return chunks[0];
528
+ }
529
+ const all = new Uint8Array(size);
530
+ let at = 0;
531
+ for (const chunk of chunks) {
532
+ all.set(chunk, at);
533
+ at += chunk.length;
534
+ }
535
+ return all;
536
+ }
488
537
  /**
489
538
  * The bytes of chunks again in pieces of `size` bytes, the last shorter. A
490
539
  * chunk that holds a whole piece goes as it is, without a copy.
@@ -76,6 +76,8 @@ export interface Started {
76
76
  durability?: Durability | undefined;
77
77
  /** the file of the store's encryption key, when it is not the store's own */
78
78
  encryptionKeyFile?: string | undefined;
79
+ /** the files of the store's older keys, which open what they sealed */
80
+ olderEncryptionKeys?: string[] | undefined;
79
81
  /** the node the store is, when it is one */
80
82
  node?: NodeOptions | undefined;
81
83
  /** measures the store into its own metrics, the machine too unless it says system: false */
@@ -192,7 +192,12 @@ function selfMetricsOf(given) {
192
192
  function storeFlags(started, dir) {
193
193
  const committing = started.durability === undefined ? [] : ['--durability', started.durability];
194
194
  const key = started.encryptionKeyFile;
195
- const keyed = key === undefined ? committing : [...committing, '--encryption-key-file', resolve(key)];
195
+ const sealing = key === undefined ? committing : [...committing, '--encryption-key-file', resolve(key)];
196
+ const older = (started.olderEncryptionKeys ?? []).flatMap(file => [
197
+ '--older-encryption-key',
198
+ resolve(file),
199
+ ]);
200
+ const keyed = [...sealing, ...older];
196
201
  const asked = selfMetricsOf(started.selfMetrics);
197
202
  const measured = asked === undefined
198
203
  ? keyed
@@ -552,6 +557,7 @@ export function embedded(runtime, dir, library, started = {}) {
552
557
  const options = StoreOptions.encode({
553
558
  durability: started.durability,
554
559
  encryptionKeyFile,
560
+ olderEncryptionKeys: (started.olderEncryptionKeys ?? []).map(file => resolve(file)),
555
561
  node: node !== undefined,
556
562
  from: node?.from ?? [],
557
563
  serve: node?.serve,
package/dist/kv.d.ts CHANGED
@@ -118,6 +118,15 @@ export declare class Bucket<T> {
118
118
  expire(key: Key, when: Duration | Date, options?: Pick<WriteOptions, 'ifVersion'>): Promise<boolean>;
119
119
  /** Removes this branch's keys and every branch under it, at once however many. */
120
120
  clear(): Promise<void>;
121
+ /**
122
+ * Seals again with the store's sealing key every value of this branch, and
123
+ * of the branches under it, that an older key sealed, and says how many.
124
+ * Once every value of the bucket is, the older key can leave the store's
125
+ * options. A value sealed with a key the store does not hold stops it with
126
+ * an `InvalidError` that names the key, and a reseal after the key is given
127
+ * goes on.
128
+ */
129
+ reseal(): Promise<number>;
121
130
  /** One page of this branch's own keys, in the byte order of their text: `"10"` before `"9"`. */
122
131
  list(options?: {
123
132
  limit?: number | undefined;
package/dist/kv.js CHANGED
@@ -6,7 +6,7 @@ import { CorruptError, InvalidError } from './errors.js';
6
6
  import { checkName, handleOn, ownerText, viaLink, } from './handles.js';
7
7
  import { dateOf, ms, unixMs } from './time.js';
8
8
  import { textOf } from './wire/codec.js';
9
- import { KvBranch, KvBucketOpen, KvCall, KvCount, KvCountersOpen, KvEntry, KvFound, KvList, KvPage, KvWritten, methods, } from './wire/protocol.js';
9
+ import { KvBranch, KvBucketOpen, KvCall, KvCount, KvCountersOpen, KvEntry, KvFound, KvList, KvPage, KvResealed, KvWritten, methods, } from './wire/protocol.js';
10
10
  // a handle's parts, kept out of its public shape, for store.tx
11
11
  const parts = new WeakMap();
12
12
  export function partsOf(handle) {
@@ -98,6 +98,19 @@ export class Bucket {
98
98
  const under = underField(this.#under);
99
99
  await this.#via.call(methods['kv.bucket.open'], this.#open, methods['kv.clear'], handle => KvBranch.encode({ handle, under }), 'write');
100
100
  }
101
+ /**
102
+ * Seals again with the store's sealing key every value of this branch, and
103
+ * of the branches under it, that an older key sealed, and says how many.
104
+ * Once every value of the bucket is, the older key can leave the store's
105
+ * options. A value sealed with a key the store does not hold stops it with
106
+ * an `InvalidError` that names the key, and a reseal after the key is given
107
+ * goes on.
108
+ */
109
+ async reseal() {
110
+ const under = underField(this.#under);
111
+ const answer = await this.#via.call(methods['kv.bucket.open'], this.#open, methods['kv.reseal'], handle => KvBranch.encode({ handle, under }), 'write');
112
+ return KvResealed.decode(answer).count ?? 0;
113
+ }
101
114
  /** One page of this branch's own keys, in the byte order of their text: `"10"` before `"9"`. */
102
115
  async list(options) {
103
116
  const under = underField(this.#under);
package/dist/sql.d.ts CHANGED
@@ -1,3 +1,4 @@
1
+ import { Files, type FilesOptions } from './blobs.ts';
1
2
  import { type Channel, type TxChannel } from './channels.ts';
2
3
  import type { Connection, Durability, Link } from './connection.ts';
3
4
  import { type Home, type Open, type Via } from './handles.ts';
@@ -171,6 +172,15 @@ export declare class Database implements Runner, Home {
171
172
  schema: S;
172
173
  }): Queue<StandardSchemaV1.InferOutput<S>>;
173
174
  queue<T = unknown>(name: string, options?: QueueOptions): Queue<T>;
175
+ /**
176
+ * A set of files kept in this database's file, as `store.files` keeps one
177
+ * in blobs.db: the same calls, and its writes commit with the rows in
178
+ * `db.tx`, through `tx.with(files)`. A file's bytes past 16 KiB lie in the
179
+ * folder beside the database's file.
180
+ *
181
+ * const packages = db.files('packages', { maxFileSize: '10MiB' })
182
+ */
183
+ files(name: string, options?: FilesOptions): Files;
174
184
  /**
175
185
  * An index of a table of this database, kept in its file: its rows found
176
186
  * again by their words, vectors or both, each hit's value its row as
@@ -266,6 +276,7 @@ export declare class SqlTx implements Runner {
266
276
  with<T>(bucket: Bucket<T>): Bucket<T>;
267
277
  with<T>(queue: Queue<T>): TxQueue<T>;
268
278
  with<T>(channel: Channel<T>): TxChannel<T>;
279
+ with(files: Files): Files;
269
280
  all<T = Row>(...statement: Statement): Promise<T[]>;
270
281
  one<T = Row>(...statement: Statement): Promise<T | undefined>;
271
282
  scalar<T = SqlValue>(...statement: Statement): Promise<T>;
package/dist/sql.js CHANGED
@@ -6,6 +6,7 @@
6
6
  import { AsyncLocalStorage } from 'node:async_hooks';
7
7
  import { readdir, readFile } from 'node:fs/promises';
8
8
  import { join } from 'node:path';
9
+ import { Files, filesOpen, filesParts } from './blobs.js';
9
10
  import { channelParts, txChannel } from './channels.js';
10
11
  import { checkDurability, download } from './connection.js';
11
12
  import { CancelledError, errorOf, InvalidError, OutcomeUnknownError } from './errors.js';
@@ -546,6 +547,19 @@ export class Database {
546
547
  queue(name, options = {}) {
547
548
  return openQueue(this.#link, this.#workers, name, options, this);
548
549
  }
550
+ /**
551
+ * A set of files kept in this database's file, as `store.files` keeps one
552
+ * in blobs.db: the same calls, and its writes commit with the rows in
553
+ * `db.tx`, through `tx.with(files)`. A file's bytes past 16 KiB lie in the
554
+ * folder beside the database's file.
555
+ *
556
+ * const packages = db.files('packages', { maxFileSize: '10MiB' })
557
+ */
558
+ files(name, options) {
559
+ const bytes = filesOpen(name, options);
560
+ const open = this.keep(methods['blobs.open'], bytes, async (connection) => filesOpen(name, options, await this.handle(connection)));
561
+ return new Files(this.#link, name, open, [], this);
562
+ }
549
563
  /**
550
564
  * An index of a table of this database, kept in its file: its rows found
551
565
  * again by their words, vectors or both, each hit's value its row as
@@ -787,7 +801,12 @@ export class SqlTx {
787
801
  this.#takes(queue.source.home, queue.source.open, `jobs queue ${queue.name}`);
788
802
  return queueVia(queue, this.#via);
789
803
  }
790
- throw new InvalidError(`${this.describe}: a transaction takes in a bucket or a queue opened from its database`);
804
+ const files = filesParts(handle);
805
+ if (files !== undefined) {
806
+ this.#takes(files.home, files.open, `files ${files.name}`);
807
+ return files.inside(this.#via);
808
+ }
809
+ throw new InvalidError(`${this.describe}: a transaction takes in a bucket, a queue or files opened from its database`);
791
810
  }
792
811
  async all(...statement) {
793
812
  return all(this, statementOf(statement));
package/dist/store.d.ts CHANGED
@@ -81,6 +81,14 @@ export interface OpenOptions {
81
81
  * running keeps the key it was started with.
82
82
  */
83
83
  encryptionKeyFile?: string;
84
+ /**
85
+ * The files of the store's older keys, which open the values they sealed
86
+ * and seal nothing new: a key being rotated out, until `reseal` has sealed
87
+ * every value again, or one found after a restore that left it out. Each
88
+ * file must be there. A sidecar found running keeps the keys it was
89
+ * started with.
90
+ */
91
+ olderEncryptionKeys?: string[];
84
92
  /**
85
93
  * Makes the store a node: it keeps a log of each file it writes, serves
86
94
  * its files to other nodes, and copies the files of the node it takes
package/dist/store.js CHANGED
@@ -317,6 +317,7 @@ export async function openWith(runtime, dir, options = {}) {
317
317
  const started = {
318
318
  durability,
319
319
  encryptionKeyFile: options.encryptionKeyFile,
320
+ olderEncryptionKeys: options.olderEncryptionKeys,
320
321
  node: options.node,
321
322
  selfMetrics: options.selfMetrics,
322
323
  maxSeries: options.maxSeries,
@@ -56,6 +56,7 @@ export declare const methods: {
56
56
  readonly 'kv.expire': 264;
57
57
  readonly 'kv.clear': 265;
58
58
  readonly 'kv.list': 266;
59
+ readonly 'kv.reseal': 267;
59
60
  readonly 'kv.counters.open': 272;
60
61
  readonly 'kv.counters.add': 273;
61
62
  readonly 'kv.counters.get': 274;
@@ -180,6 +181,8 @@ export declare const BlobsOpen: import("./codec.ts").Message<{
180
181
  readonly ttl: readonly [2, import("./codec.ts").Codec<number, number>];
181
182
  /** the largest file a write may leave */
182
183
  readonly maxFileSize: readonly [3, import("./codec.ts").Codec<number, number>];
184
+ /** a database's handle: the files live in its file, and put, head, delete, copy and rename join its transactions; blobs.db when absent */
185
+ readonly database: readonly [4, import("./codec.ts").Codec<number, number>];
183
186
  }>;
184
187
  /**
185
188
  * Where a call stands: the files, a folder's segments, a path within it.
@@ -529,6 +532,8 @@ export declare const StoreOptions: import("./codec.ts").Message<{
529
532
  readonly withoutSystem: readonly [14, import("./codec.ts").Codec<boolean, boolean>];
530
533
  /** the metrics series the store holds at most; 100,000 when absent */
531
534
  readonly maxSeries: readonly [15, import("./codec.ts").Codec<number, number>];
535
+ /** the files of the store's older keys, which open the values they sealed and seal none */
536
+ readonly olderEncryptionKeys: readonly [16, import("./codec.ts").Codec<string[], readonly string[]>];
532
537
  }>;
533
538
  /**
534
539
  * The last frame of a connection.
@@ -953,6 +958,12 @@ export declare const KvWritten: import("./codec.ts").Message<{
953
958
  export declare const KvFound: import("./codec.ts").Message<{
954
959
  readonly found: readonly [1, import("./codec.ts").Codec<boolean, boolean>];
955
960
  }>;
961
+ /**
962
+ * How many values a reseal sealed again with the sealing key.
963
+ */
964
+ export declare const KvResealed: import("./codec.ts").Message<{
965
+ readonly count: readonly [1, import("./codec.ts").Codec<number, number>];
966
+ }>;
956
967
  /**
957
968
  * A branch of a handle, for a clear.
958
969
  */
@@ -57,6 +57,7 @@ export const methods = {
57
57
  'kv.expire': 0x0108,
58
58
  'kv.clear': 0x0109,
59
59
  'kv.list': 0x010a,
60
+ 'kv.reseal': 0x010b,
60
61
  'kv.counters.open': 0x0110,
61
62
  'kv.counters.add': 0x0111,
62
63
  'kv.counters.get': 0x0112,
@@ -291,6 +292,8 @@ export const BlobsOpen = message('blobs.Open', {
291
292
  ttl: [2, uint],
292
293
  /** the largest file a write may leave */
293
294
  maxFileSize: [3, uint],
295
+ /** a database's handle: the files live in its file, and put, head, delete, copy and rename join its transactions; blobs.db when absent */
296
+ database: [4, uint],
294
297
  }, {
295
298
  write(w, v) {
296
299
  const head = w.openMap();
@@ -310,12 +313,18 @@ export const BlobsOpen = message('blobs.Open', {
310
313
  w.uint(v.maxFileSize);
311
314
  n++;
312
315
  }
316
+ if (v.database !== undefined) {
317
+ w.field(4);
318
+ w.uint(v.database);
319
+ n++;
320
+ }
313
321
  w.closeMap(head, n);
314
322
  },
315
323
  read(r) {
316
324
  let $name;
317
325
  let $ttl;
318
326
  let $maxFileSize;
327
+ let $database;
319
328
  for (let n = r.message(); n > 0; n--) {
320
329
  switch (r.field()) {
321
330
  case 1:
@@ -327,12 +336,15 @@ export const BlobsOpen = message('blobs.Open', {
327
336
  case 3:
328
337
  $maxFileSize = r.uint();
329
338
  break;
339
+ case 4:
340
+ $database = r.uint();
341
+ break;
330
342
  default:
331
343
  r.skip();
332
344
  }
333
345
  }
334
346
  r.leave();
335
- return { name: $name, ttl: $ttl, maxFileSize: $maxFileSize };
347
+ return { name: $name, ttl: $ttl, maxFileSize: $maxFileSize, database: $database };
336
348
  },
337
349
  });
338
350
  /**
@@ -2111,6 +2123,8 @@ export const StoreOptions = message('store.Options', {
2111
2123
  withoutSystem: [14, bool],
2112
2124
  /** the metrics series the store holds at most; 100,000 when absent */
2113
2125
  maxSeries: [15, uint],
2126
+ /** the files of the store's older keys, which open the values they sealed and seal none */
2127
+ olderEncryptionKeys: [16, list(str)],
2114
2128
  }, {
2115
2129
  write(w, v) {
2116
2130
  const head = w.openMap();
@@ -2199,6 +2213,14 @@ export const StoreOptions = message('store.Options', {
2199
2213
  w.uint(v.maxSeries);
2200
2214
  n++;
2201
2215
  }
2216
+ if (v.olderEncryptionKeys !== undefined) {
2217
+ w.field(16);
2218
+ w.array(v.olderEncryptionKeys.length);
2219
+ for (const item0 of v.olderEncryptionKeys) {
2220
+ w.str(item0);
2221
+ }
2222
+ n++;
2223
+ }
2202
2224
  w.closeMap(head, n);
2203
2225
  },
2204
2226
  read(r) {
@@ -2217,6 +2239,7 @@ export const StoreOptions = message('store.Options', {
2217
2239
  let $selfMetrics;
2218
2240
  let $withoutSystem;
2219
2241
  let $maxSeries;
2242
+ let $olderEncryptionKeys;
2220
2243
  for (let n = r.message(); n > 0; n--) {
2221
2244
  switch (r.field()) {
2222
2245
  case 1:
@@ -2282,6 +2305,15 @@ export const StoreOptions = message('store.Options', {
2282
2305
  case 15:
2283
2306
  $maxSeries = r.uint();
2284
2307
  break;
2308
+ case 16: {
2309
+ const items0 = [];
2310
+ for (let i0 = r.array(); i0 > 0; i0--) {
2311
+ items0.push(r.str());
2312
+ }
2313
+ r.leave();
2314
+ $olderEncryptionKeys = items0;
2315
+ break;
2316
+ }
2285
2317
  default:
2286
2318
  r.skip();
2287
2319
  }
@@ -2303,6 +2335,7 @@ export const StoreOptions = message('store.Options', {
2303
2335
  selfMetrics: $selfMetrics,
2304
2336
  withoutSystem: $withoutSystem,
2305
2337
  maxSeries: $maxSeries,
2338
+ olderEncryptionKeys: $olderEncryptionKeys,
2306
2339
  };
2307
2340
  },
2308
2341
  });
@@ -4145,6 +4178,37 @@ export const KvFound = message('kv.Found', {
4145
4178
  return { found: $found };
4146
4179
  },
4147
4180
  });
4181
+ /**
4182
+ * How many values a reseal sealed again with the sealing key.
4183
+ */
4184
+ export const KvResealed = message('kv.Resealed', {
4185
+ count: [1, uint],
4186
+ }, {
4187
+ write(w, v) {
4188
+ const head = w.openMap();
4189
+ let n = 0;
4190
+ if (v.count !== undefined) {
4191
+ w.field(1);
4192
+ w.uint(v.count);
4193
+ n++;
4194
+ }
4195
+ w.closeMap(head, n);
4196
+ },
4197
+ read(r) {
4198
+ let $count;
4199
+ for (let n = r.message(); n > 0; n--) {
4200
+ switch (r.field()) {
4201
+ case 1:
4202
+ $count = r.uint();
4203
+ break;
4204
+ default:
4205
+ r.skip();
4206
+ }
4207
+ }
4208
+ r.leave();
4209
+ return { count: $count };
4210
+ },
4211
+ });
4148
4212
  /**
4149
4213
  * A branch of a handle, for a clear.
4150
4214
  */
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@tinyshed/tinystore",
3
- "version": "0.3.0",
3
+ "version": "0.4.0",
4
4
  "description": "TinyStore for Bun and Node: kv, jobs, blobs, SQL, records and metrics in one directory, served by a sidecar",
5
5
  "license": "Apache-2.0",
6
6
  "homepage": "https://tinyshed.org/tinystore",
@@ -31,12 +31,12 @@
31
31
  "README.md"
32
32
  ],
33
33
  "optionalDependencies": {
34
- "@tinyshed/tinystore-linux-x64": "0.3.0",
35
- "@tinyshed/tinystore-linux-arm64": "0.3.0",
36
- "@tinyshed/tinystore-linux-x64-musl": "0.3.0",
37
- "@tinyshed/tinystore-linux-arm64-musl": "0.3.0",
38
- "@tinyshed/tinystore-darwin-arm64": "0.3.0",
39
- "@tinyshed/tinystore-darwin-x64": "0.3.0",
40
- "@tinyshed/tinystore-win32-x64": "0.3.0"
34
+ "@tinyshed/tinystore-linux-x64": "0.4.0",
35
+ "@tinyshed/tinystore-linux-arm64": "0.4.0",
36
+ "@tinyshed/tinystore-linux-x64-musl": "0.4.0",
37
+ "@tinyshed/tinystore-linux-arm64-musl": "0.4.0",
38
+ "@tinyshed/tinystore-darwin-arm64": "0.4.0",
39
+ "@tinyshed/tinystore-darwin-x64": "0.4.0",
40
+ "@tinyshed/tinystore-win32-x64": "0.4.0"
41
41
  }
42
42
  }
package/src/blobs.ts CHANGED
@@ -5,7 +5,7 @@
5
5
 
6
6
  import { type Connection, type Idempotence, type Link, pieceOf } from './connection.ts'
7
7
  import { ClosedError, InvalidError, OutcomeUnknownError } from './errors.ts'
8
- import { checkName, handleOn, type Via, viaLink } from './handles.ts'
8
+ import { checkName, type Home, handleOn, type Open, type Via, viaLink } from './handles.ts'
9
9
  import { ConnectionLostError, type Stream, watch } from './session.ts'
10
10
  import { type Duration, dateOf, ms, unixMs } from './time.ts'
11
11
  import {
@@ -96,13 +96,35 @@ const nothing = new Uint8Array(0)
96
96
 
97
97
  /** Files as `store.files` opens them. */
98
98
  export function openFiles(link: Link, name: string, options?: FilesOptions): Files {
99
+ return new Files(link, name, filesOpen(name, options), [])
100
+ }
101
+
102
+ /** The open of a set of files, kept in the database of the handle `database` when it is given. */
103
+ export function filesOpen(name: string, options?: FilesOptions, database?: number): Uint8Array {
99
104
  checkName(name, 'files')
100
- const open = BlobsOpen.encode({
105
+ return BlobsOpen.encode({
101
106
  name,
102
107
  ttl: options?.ttl === undefined ? undefined : ms(options.ttl),
103
108
  maxFileSize: options?.maxFileSize === undefined ? undefined : bytesOf(options.maxFileSize),
109
+ database,
104
110
  })
105
- return new Files(link, name, open, [])
111
+ }
112
+
113
+ /** What a transaction needs of a set of files it takes in. */
114
+ export interface FilesParts {
115
+ readonly name: string
116
+ readonly open: Open
117
+ /** the database whose file keeps the files; the store's own when undefined */
118
+ readonly home: Home | undefined
119
+ /** the same files, their calls through the transaction's `via` */
120
+ inside(via: Via): Files
121
+ }
122
+
123
+ // a set of files' parts, kept out of its public shape, for db.tx
124
+ const parts = new WeakMap<object, FilesParts>()
125
+
126
+ export function filesParts(handle: object): FilesParts | undefined {
127
+ return parts.get(handle)
106
128
  }
107
129
 
108
130
  /**
@@ -113,24 +135,49 @@ export class Files {
113
135
  readonly name: string
114
136
  readonly #link: Link
115
137
  readonly #via: Via
116
- readonly #open: Uint8Array
138
+ readonly #open: Open
117
139
  readonly #folder: string[]
140
+ readonly #home: Home | undefined
141
+ /** whether its calls go through a database's transaction */
142
+ readonly #inside: boolean
118
143
 
119
- /** Files come from `store.files`. */
120
- constructor(link: Link, name: string, open: Uint8Array, folder: string[]) {
144
+ /**
145
+ * Files come from `store.files` and `db.files`; inside a database's
146
+ * transaction, from `tx.with(files)`, whose writes go through the
147
+ * transaction.
148
+ */
149
+ constructor(link: Link, name: string, open: Open, folder: string[], home?: Home, via?: Via) {
121
150
  this.name = name
122
151
  this.#link = link
123
- this.#via = viaLink(link)
152
+ this.#via = via ?? home?.via(`files ${name}`) ?? viaLink(link)
124
153
  this.#open = open
125
154
  this.#folder = folder
155
+ this.#home = home
156
+ this.#inside = via !== undefined
157
+ const inside = (through: Via) => new Files(link, name, open, folder, home, through)
158
+ parts.set(this, { name, open, home, inside })
126
159
  }
127
160
 
128
161
  /** The folder of these segments, one each: `folder('users', 42)` is users/42, never users/420. */
129
162
  folder(...segments: (string | number | bigint)[]): Files {
130
- return new Files(this.#link, this.name, this.#open, [
131
- ...this.#folder,
132
- ...segments.map(segmentOf),
133
- ])
163
+ const folder = [...this.#folder, ...segments.map(segmentOf)]
164
+ return new Files(
165
+ this.#link,
166
+ this.name,
167
+ this.#open,
168
+ folder,
169
+ this.#home,
170
+ this.#inside ? this.#via : undefined,
171
+ )
172
+ }
173
+
174
+ /** Refuses what streams a file's bytes inside a transaction, which takes a put's bytes whole. */
175
+ #outside(what: string): void {
176
+ if (this.#inside) {
177
+ throw new InvalidError(
178
+ `files ${this.name}: ${what} inside a transaction: put a file's bytes whole, and read them after its commit`,
179
+ )
180
+ }
134
181
  }
135
182
 
136
183
  /**
@@ -277,6 +324,7 @@ export class Files {
277
324
  * leaving the block without commit aborts it.
278
325
  */
279
326
  async upload(path: string, options?: FileOptions): Promise<Upload> {
327
+ this.#outside('an upload')
280
328
  const connection = await this.#link.connection()
281
329
  const write = BlobsWrite.encode({
282
330
  ...this.#fields(path, options),
@@ -396,6 +444,15 @@ export class Files {
396
444
  size: options?.size ?? source.size,
397
445
  create,
398
446
  }
447
+ if (this.#inside) {
448
+ const bytes = await allOf(source)
449
+ const answer = await this.#call(
450
+ 'blobs.put',
451
+ handle => BlobsWrite.encode({ ...fields, handle, bytes }),
452
+ 'write',
453
+ )
454
+ return writtenOf(answer)
455
+ }
399
456
  return this.#link.run(
400
457
  'write',
401
458
  async connection => {
@@ -688,6 +745,26 @@ function sourceOf(body: Body): Source {
688
745
  }
689
746
  }
690
747
 
748
+ /** Every byte of a body at once, as a put inside a transaction sends it. */
749
+ async function allOf(source: Source): Promise<Uint8Array> {
750
+ const chunks: Uint8Array[] = []
751
+ let size = 0
752
+ for await (const chunk of source.chunks()) {
753
+ chunks.push(chunk)
754
+ size += chunk.length
755
+ }
756
+ if (chunks.length === 1) {
757
+ return chunks[0] as Uint8Array
758
+ }
759
+ const all = new Uint8Array(size)
760
+ let at = 0
761
+ for (const chunk of chunks) {
762
+ all.set(chunk, at)
763
+ at += chunk.length
764
+ }
765
+ return all
766
+ }
767
+
691
768
  /**
692
769
  * The bytes of chunks again in pieces of `size` bytes, the last shorter. A
693
770
  * chunk that holds a whole piece goes as it is, without a copy.
package/src/connection.ts CHANGED
@@ -293,6 +293,8 @@ export interface Started {
293
293
  durability?: Durability | undefined
294
294
  /** the file of the store's encryption key, when it is not the store's own */
295
295
  encryptionKeyFile?: string | undefined
296
+ /** the files of the store's older keys, which open what they sealed */
297
+ olderEncryptionKeys?: string[] | undefined
296
298
  /** the node the store is, when it is one */
297
299
  node?: NodeOptions | undefined
298
300
  /** measures the store into its own metrics, the machine too unless it says system: false */
@@ -319,8 +321,13 @@ function selfMetricsOf(given: SelfMetrics | undefined): { system: boolean } | un
319
321
  function storeFlags(started: Started, dir: string): string[] {
320
322
  const committing = started.durability === undefined ? [] : ['--durability', started.durability]
321
323
  const key = started.encryptionKeyFile
322
- const keyed =
324
+ const sealing =
323
325
  key === undefined ? committing : [...committing, '--encryption-key-file', resolve(key)]
326
+ const older = (started.olderEncryptionKeys ?? []).flatMap(file => [
327
+ '--older-encryption-key',
328
+ resolve(file),
329
+ ])
330
+ const keyed = [...sealing, ...older]
324
331
  const asked = selfMetricsOf(started.selfMetrics)
325
332
  const measured =
326
333
  asked === undefined
@@ -743,6 +750,7 @@ export function embedded(
743
750
  const options = StoreOptions.encode({
744
751
  durability: started.durability,
745
752
  encryptionKeyFile,
753
+ olderEncryptionKeys: (started.olderEncryptionKeys ?? []).map(file => resolve(file)),
746
754
  node: node !== undefined,
747
755
  from: node?.from ?? [],
748
756
  serve: node?.serve,
package/src/kv.ts CHANGED
@@ -27,6 +27,7 @@ import {
27
27
  KvFound,
28
28
  KvList,
29
29
  KvPage,
30
+ KvResealed,
30
31
  KvWritten,
31
32
  type Method,
32
33
  methods,
@@ -239,6 +240,26 @@ export class Bucket<T> {
239
240
  )
240
241
  }
241
242
 
243
+ /**
244
+ * Seals again with the store's sealing key every value of this branch, and
245
+ * of the branches under it, that an older key sealed, and says how many.
246
+ * Once every value of the bucket is, the older key can leave the store's
247
+ * options. A value sealed with a key the store does not hold stops it with
248
+ * an `InvalidError` that names the key, and a reseal after the key is given
249
+ * goes on.
250
+ */
251
+ async reseal(): Promise<number> {
252
+ const under = underField(this.#under)
253
+ const answer = await this.#via.call(
254
+ methods['kv.bucket.open'],
255
+ this.#open,
256
+ methods['kv.reseal'],
257
+ handle => KvBranch.encode({ handle, under }),
258
+ 'write',
259
+ )
260
+ return KvResealed.decode(answer).count ?? 0
261
+ }
262
+
242
263
  /** One page of this branch's own keys, in the byte order of their text: `"10"` before `"9"`. */
243
264
  async list(options?: { limit?: number | undefined; after?: Key | undefined }): Promise<Page<T>> {
244
265
  const under = underField(this.#under)
package/src/sql.ts CHANGED
@@ -8,6 +8,7 @@ import { AsyncLocalStorage } from 'node:async_hooks'
8
8
  import { readdir, readFile } from 'node:fs/promises'
9
9
  import { join } from 'node:path'
10
10
 
11
+ import { Files, type FilesOptions, filesOpen, filesParts } from './blobs.ts'
11
12
  import { type Channel, channelParts, type TxChannel, txChannel } from './channels.ts'
12
13
  import type { Connection, Durability, Idempotence, Link } from './connection.ts'
13
14
  import { checkDurability, download } from './connection.ts'
@@ -774,6 +775,22 @@ export class Database implements Runner, Home {
774
775
  return openQueue(this.#link, this.#workers, name, options, this)
775
776
  }
776
777
 
778
+ /**
779
+ * A set of files kept in this database's file, as `store.files` keeps one
780
+ * in blobs.db: the same calls, and its writes commit with the rows in
781
+ * `db.tx`, through `tx.with(files)`. A file's bytes past 16 KiB lie in the
782
+ * folder beside the database's file.
783
+ *
784
+ * const packages = db.files('packages', { maxFileSize: '10MiB' })
785
+ */
786
+ files(name: string, options?: FilesOptions): Files {
787
+ const bytes = filesOpen(name, options)
788
+ const open = this.keep(methods['blobs.open'], bytes, async connection =>
789
+ filesOpen(name, options, await this.handle(connection)),
790
+ )
791
+ return new Files(this.#link, name, open, [], this)
792
+ }
793
+
777
794
  /**
778
795
  * An index of a table of this database, kept in its file: its rows found
779
796
  * again by their words, vectors or both, each hit's value its row as
@@ -1059,9 +1076,10 @@ export class SqlTx implements Runner {
1059
1076
  with<T>(bucket: Bucket<T>): Bucket<T>
1060
1077
  with<T>(queue: Queue<T>): TxQueue<T>
1061
1078
  with<T>(channel: Channel<T>): TxChannel<T>
1079
+ with(files: Files): Files
1062
1080
  with(
1063
- handle: Bucket<unknown> | Queue<unknown> | Channel<unknown>,
1064
- ): Bucket<unknown> | TxQueue<unknown> | TxChannel<unknown> {
1081
+ handle: Bucket<unknown> | Queue<unknown> | Channel<unknown> | Files,
1082
+ ): Bucket<unknown> | TxQueue<unknown> | TxChannel<unknown> | Files {
1065
1083
  const channel = channelParts(handle)
1066
1084
  if (channel !== undefined) {
1067
1085
  // a message published here is handed over once the transaction commits, at the writer
@@ -1080,8 +1098,13 @@ export class SqlTx implements Runner {
1080
1098
  this.#takes(queue.source.home, queue.source.open, `jobs queue ${queue.name}`)
1081
1099
  return queueVia(queue, this.#via)
1082
1100
  }
1101
+ const files = filesParts(handle)
1102
+ if (files !== undefined) {
1103
+ this.#takes(files.home, files.open, `files ${files.name}`)
1104
+ return files.inside(this.#via)
1105
+ }
1083
1106
  throw new InvalidError(
1084
- `${this.describe}: a transaction takes in a bucket or a queue opened from its database`,
1107
+ `${this.describe}: a transaction takes in a bucket, a queue or files opened from its database`,
1085
1108
  )
1086
1109
  }
1087
1110
 
package/src/store.ts CHANGED
@@ -142,6 +142,14 @@ export interface OpenOptions {
142
142
  * running keeps the key it was started with.
143
143
  */
144
144
  encryptionKeyFile?: string
145
+ /**
146
+ * The files of the store's older keys, which open the values they sealed
147
+ * and seal nothing new: a key being rotated out, until `reseal` has sealed
148
+ * every value again, or one found after a restore that left it out. Each
149
+ * file must be there. A sidecar found running keeps the keys it was
150
+ * started with.
151
+ */
152
+ olderEncryptionKeys?: string[]
145
153
  /**
146
154
  * Makes the store a node: it keeps a log of each file it writes, serves
147
155
  * its files to other nodes, and copies the files of the node it takes
@@ -573,6 +581,7 @@ export async function openWith(
573
581
  const started = {
574
582
  durability,
575
583
  encryptionKeyFile: options.encryptionKeyFile,
584
+ olderEncryptionKeys: options.olderEncryptionKeys,
576
585
  node: options.node,
577
586
  selfMetrics: options.selfMetrics,
578
587
  maxSeries: options.maxSeries,
@@ -78,6 +78,7 @@ export const methods = {
78
78
  'kv.expire': 0x0108,
79
79
  'kv.clear': 0x0109,
80
80
  'kv.list': 0x010a,
81
+ 'kv.reseal': 0x010b,
81
82
  'kv.counters.open': 0x0110,
82
83
  'kv.counters.add': 0x0111,
83
84
  'kv.counters.get': 0x0112,
@@ -332,6 +333,8 @@ export const BlobsOpen = message(
332
333
  ttl: [2, uint],
333
334
  /** the largest file a write may leave */
334
335
  maxFileSize: [3, uint],
336
+ /** a database's handle: the files live in its file, and put, head, delete, copy and rename join its transactions; blobs.db when absent */
337
+ database: [4, uint],
335
338
  },
336
339
  {
337
340
  write(w, v) {
@@ -352,12 +355,18 @@ export const BlobsOpen = message(
352
355
  w.uint(v.maxFileSize)
353
356
  n++
354
357
  }
358
+ if (v.database !== undefined) {
359
+ w.field(4)
360
+ w.uint(v.database)
361
+ n++
362
+ }
355
363
  w.closeMap(head, n)
356
364
  },
357
365
  read(r) {
358
366
  let $name: string | undefined
359
367
  let $ttl: number | undefined
360
368
  let $maxFileSize: number | undefined
369
+ let $database: number | undefined
361
370
  for (let n = r.message(); n > 0; n--) {
362
371
  switch (r.field()) {
363
372
  case 1:
@@ -369,12 +378,15 @@ export const BlobsOpen = message(
369
378
  case 3:
370
379
  $maxFileSize = r.uint()
371
380
  break
381
+ case 4:
382
+ $database = r.uint()
383
+ break
372
384
  default:
373
385
  r.skip()
374
386
  }
375
387
  }
376
388
  r.leave()
377
- return { name: $name, ttl: $ttl, maxFileSize: $maxFileSize }
389
+ return { name: $name, ttl: $ttl, maxFileSize: $maxFileSize, database: $database }
378
390
  },
379
391
  },
380
392
  )
@@ -2282,6 +2294,8 @@ export const StoreOptions = message(
2282
2294
  withoutSystem: [14, bool],
2283
2295
  /** the metrics series the store holds at most; 100,000 when absent */
2284
2296
  maxSeries: [15, uint],
2297
+ /** the files of the store's older keys, which open the values they sealed and seal none */
2298
+ olderEncryptionKeys: [16, list(str)],
2285
2299
  },
2286
2300
  {
2287
2301
  write(w, v) {
@@ -2371,6 +2385,14 @@ export const StoreOptions = message(
2371
2385
  w.uint(v.maxSeries)
2372
2386
  n++
2373
2387
  }
2388
+ if (v.olderEncryptionKeys !== undefined) {
2389
+ w.field(16)
2390
+ w.array(v.olderEncryptionKeys.length)
2391
+ for (const item0 of v.olderEncryptionKeys) {
2392
+ w.str(item0)
2393
+ }
2394
+ n++
2395
+ }
2374
2396
  w.closeMap(head, n)
2375
2397
  },
2376
2398
  read(r) {
@@ -2389,6 +2411,7 @@ export const StoreOptions = message(
2389
2411
  let $selfMetrics: boolean | undefined
2390
2412
  let $withoutSystem: boolean | undefined
2391
2413
  let $maxSeries: number | undefined
2414
+ let $olderEncryptionKeys: string[] | undefined
2392
2415
  for (let n = r.message(); n > 0; n--) {
2393
2416
  switch (r.field()) {
2394
2417
  case 1:
@@ -2454,6 +2477,15 @@ export const StoreOptions = message(
2454
2477
  case 15:
2455
2478
  $maxSeries = r.uint()
2456
2479
  break
2480
+ case 16: {
2481
+ const items0: string[] = []
2482
+ for (let i0 = r.array(); i0 > 0; i0--) {
2483
+ items0.push(r.str())
2484
+ }
2485
+ r.leave()
2486
+ $olderEncryptionKeys = items0
2487
+ break
2488
+ }
2457
2489
  default:
2458
2490
  r.skip()
2459
2491
  }
@@ -2475,6 +2507,7 @@ export const StoreOptions = message(
2475
2507
  selfMetrics: $selfMetrics,
2476
2508
  withoutSystem: $withoutSystem,
2477
2509
  maxSeries: $maxSeries,
2510
+ olderEncryptionKeys: $olderEncryptionKeys,
2478
2511
  }
2479
2512
  },
2480
2513
  },
@@ -4459,6 +4492,42 @@ export const KvFound = message(
4459
4492
  },
4460
4493
  )
4461
4494
 
4495
+ /**
4496
+ * How many values a reseal sealed again with the sealing key.
4497
+ */
4498
+ export const KvResealed = message(
4499
+ 'kv.Resealed',
4500
+ {
4501
+ count: [1, uint],
4502
+ },
4503
+ {
4504
+ write(w, v) {
4505
+ const head = w.openMap()
4506
+ let n = 0
4507
+ if (v.count !== undefined) {
4508
+ w.field(1)
4509
+ w.uint(v.count)
4510
+ n++
4511
+ }
4512
+ w.closeMap(head, n)
4513
+ },
4514
+ read(r) {
4515
+ let $count: number | undefined
4516
+ for (let n = r.message(); n > 0; n--) {
4517
+ switch (r.field()) {
4518
+ case 1:
4519
+ $count = r.uint()
4520
+ break
4521
+ default:
4522
+ r.skip()
4523
+ }
4524
+ }
4525
+ r.leave()
4526
+ return { count: $count }
4527
+ },
4528
+ },
4529
+ )
4530
+
4462
4531
  /**
4463
4532
  * A branch of a handle, for a clear.
4464
4533
  */