@hold-baby/bucket-db 0.3.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (103) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +503 -19
  3. package/dist/bucket-db.d.ts +25 -0
  4. package/dist/bucket-db.js +36 -0
  5. package/dist/cli.d.ts +2 -0
  6. package/dist/cli.js +133 -0
  7. package/dist/collection.d.ts +78 -0
  8. package/dist/collection.js +321 -0
  9. package/dist/doctor.d.ts +7 -0
  10. package/dist/doctor.js +100 -0
  11. package/dist/errors.d.ts +59 -0
  12. package/dist/errors.js +57 -0
  13. package/dist/index.d.ts +13 -0
  14. package/dist/index.js +6 -0
  15. package/dist/keys.d.ts +12 -0
  16. package/dist/keys.js +31 -0
  17. package/dist/query.d.ts +16 -0
  18. package/dist/query.js +133 -0
  19. package/dist/snapshot.d.ts +16 -0
  20. package/dist/snapshot.js +40 -0
  21. package/dist/storage/adapter.d.ts +75 -0
  22. package/dist/storage/adapter.js +27 -0
  23. package/dist/storage/fs.d.ts +28 -0
  24. package/dist/storage/fs.js +175 -0
  25. package/dist/storage/memory.d.ts +18 -0
  26. package/dist/storage/memory.js +49 -0
  27. package/dist/storage/s3/adapters.d.ts +38 -0
  28. package/dist/storage/s3/adapters.js +65 -0
  29. package/dist/storage/s3/client.d.ts +52 -0
  30. package/dist/storage/s3/client.js +231 -0
  31. package/dist/storage/s3/credentials.d.ts +17 -0
  32. package/dist/storage/s3/credentials.js +42 -0
  33. package/dist/storage/s3/sigv4.d.ts +32 -0
  34. package/dist/storage/s3/sigv4.js +75 -0
  35. package/dist/types.d.ts +55 -0
  36. package/dist/types.js +1 -0
  37. package/dist/url.d.ts +16 -0
  38. package/dist/url.js +66 -0
  39. package/llms.txt +142 -0
  40. package/package.json +20 -7
  41. package/src/cache/lru-cache.ts +0 -131
  42. package/src/core/bucket-db.ts +0 -47
  43. package/src/core/collection.ts +0 -278
  44. package/src/core/wal.ts +0 -106
  45. package/src/index/shard-manager.ts +0 -197
  46. package/src/index.ts +0 -37
  47. package/src/query/evaluator.ts +0 -143
  48. package/src/storage/ali-oss.d.ts +0 -43
  49. package/src/storage/fs-adapter.ts +0 -248
  50. package/src/storage/memory-adapter.ts +0 -105
  51. package/src/storage/oss-adapter.ts +0 -164
  52. package/src/storage/s3-adapter.ts +0 -200
  53. package/src/types/collection.ts +0 -43
  54. package/src/types/document.ts +0 -22
  55. package/src/types/errors.ts +0 -49
  56. package/src/types/index.ts +0 -37
  57. package/src/types/query.ts +0 -50
  58. package/src/types/storage.ts +0 -46
  59. package/src/utils/hash.ts +0 -24
  60. package/src/utils/path-validator.ts +0 -119
  61. package/tests/core/bucket-db.test.d.ts +0 -2
  62. package/tests/core/bucket-db.test.d.ts.map +0 -1
  63. package/tests/core/bucket-db.test.js +0 -34
  64. package/tests/core/bucket-db.test.js.map +0 -1
  65. package/tests/core/bucket-db.test.ts +0 -50
  66. package/tests/core/collection.test.d.ts +0 -2
  67. package/tests/core/collection.test.d.ts.map +0 -1
  68. package/tests/core/collection.test.js +0 -148
  69. package/tests/core/collection.test.js.map +0 -1
  70. package/tests/core/collection.test.ts +0 -195
  71. package/tests/core/dbpath.test.ts +0 -52
  72. package/tests/index/shard-manager.test.d.ts +0 -2
  73. package/tests/index/shard-manager.test.d.ts.map +0 -1
  74. package/tests/index/shard-manager.test.js +0 -106
  75. package/tests/index/shard-manager.test.js.map +0 -1
  76. package/tests/index/shard-manager.test.ts +0 -129
  77. package/tests/query/evaluator.test.d.ts +0 -2
  78. package/tests/query/evaluator.test.d.ts.map +0 -1
  79. package/tests/query/evaluator.test.js +0 -87
  80. package/tests/query/evaluator.test.js.map +0 -1
  81. package/tests/query/evaluator.test.ts +0 -104
  82. package/tests/storage/fs-adapter.test.ts +0 -111
  83. package/tests/storage/memory-adapter.test.d.ts +0 -2
  84. package/tests/storage/memory-adapter.test.d.ts.map +0 -1
  85. package/tests/storage/memory-adapter.test.js +0 -65
  86. package/tests/storage/memory-adapter.test.js.map +0 -1
  87. package/tests/storage/memory-adapter.test.ts +0 -84
  88. package/tests/storage/oss-adapter.test.d.ts +0 -2
  89. package/tests/storage/oss-adapter.test.d.ts.map +0 -1
  90. package/tests/storage/oss-adapter.test.js +0 -34
  91. package/tests/storage/oss-adapter.test.js.map +0 -1
  92. package/tests/storage/oss-adapter.test.ts +0 -40
  93. package/tests/storage/s3-adapter.test.d.ts +0 -2
  94. package/tests/storage/s3-adapter.test.d.ts.map +0 -1
  95. package/tests/storage/s3-adapter.test.js +0 -51
  96. package/tests/storage/s3-adapter.test.js.map +0 -1
  97. package/tests/storage/s3-adapter.test.ts +0 -62
  98. package/tests/utils/hash.test.d.ts +0 -2
  99. package/tests/utils/hash.test.d.ts.map +0 -1
  100. package/tests/utils/hash.test.js +0 -36
  101. package/tests/utils/hash.test.js.map +0 -1
  102. package/tests/utils/hash.test.ts +0 -42
  103. package/tsconfig.json +0 -12
package/dist/cli.js ADDED
@@ -0,0 +1,133 @@
1
+ #!/usr/bin/env node
2
+ import { readFileSync } from 'node:fs';
3
+ import { parseArgs } from 'node:util';
4
+ import { BucketDB } from './bucket-db.js';
5
+ import { BucketDBError, ValidationError } from './errors.js';
6
+ import { docsPrefix, validateName } from './keys.js';
7
+ import { doctor } from './doctor.js';
8
+ import { openUrl } from './url.js';
9
+ const USAGE = `Usage: bucket-db <command> [arguments] [--url <url>]
10
+
11
+ get <collection> <id> print a document (exit 2 if it does not exist)
12
+ put <collection> <id> [json] create or replace a document; JSON from the argument or stdin
13
+ find <collection> [filter] print matching documents as a JSON array
14
+ [--sort field[:desc]]... [--limit n] [--offset n]
15
+ count <collection> [filter] print the number of (matching) documents
16
+ delete <collection> <id> delete a document (no error if it does not exist)
17
+ export <collection> print every document as JSON Lines
18
+ import <collection> [file] write JSON Lines from a file or stdin; documents keep id and timestamps
19
+ compact <collection> rebuild the query snapshot
20
+ doctor check what the storage service guarantees
21
+
22
+ The URL comes from --url or BUCKET_DB_URL: s3://bucket/prefix?region=…,
23
+ oss://bucket/prefix?region=…, file:///dir. Credentials come from the
24
+ environment (AWS_* / OSS_*), never the URL.
25
+ Exit codes: 0 success, 1 error, 2 not found.`;
26
+ const encoder = new TextEncoder();
27
+ const print = (value) => process.stdout.write(`${JSON.stringify(value)}\n`);
28
+ function parseJson(text, what) {
29
+ try {
30
+ return JSON.parse(text);
31
+ }
32
+ catch {
33
+ throw new ValidationError(`${what} is not valid JSON`);
34
+ }
35
+ }
36
+ function object(value, what) {
37
+ if (typeof value !== 'object' || value === null || Array.isArray(value))
38
+ throw new ValidationError(`${what} must be a JSON object`);
39
+ return value;
40
+ }
41
+ function required(value, name) {
42
+ if (value === undefined)
43
+ throw new ValidationError(`Missing <${name}>\n\n${USAGE}`);
44
+ return value;
45
+ }
46
+ async function main() {
47
+ const { values, positionals } = parseArgs({
48
+ allowPositionals: true,
49
+ options: {
50
+ url: { type: 'string' },
51
+ sort: { type: 'string', multiple: true },
52
+ limit: { type: 'string' },
53
+ offset: { type: 'string' },
54
+ help: { type: 'boolean', short: 'h' },
55
+ },
56
+ });
57
+ const [command, name, ...args] = positionals;
58
+ if (values.help || !command) {
59
+ console.log(USAGE);
60
+ return command || values.help ? 0 : 1;
61
+ }
62
+ const { adapter, dbPath } = openUrl(values.url);
63
+ if (command === 'doctor')
64
+ return (await doctor(adapter, dbPath, line => console.log(line))) > 0 ? 1 : 0;
65
+ const db = new BucketDB(adapter, dbPath, { logger: ({ message }) => console.error(`bucket-db: ${message}`) });
66
+ const collection = db.collection(required(name, 'collection'));
67
+ const filter = () => (args[0] === undefined ? {} : object(parseJson(args[0], 'filter'), 'filter'));
68
+ const stdinOr = (value) => value ?? readFileSync(0, 'utf8');
69
+ switch (command) {
70
+ case 'get': {
71
+ const document = await collection.findById(required(args[0], 'id'));
72
+ if (!document) {
73
+ console.error(`bucket-db: not found: ${args[0]}`);
74
+ return 2;
75
+ }
76
+ print(document);
77
+ return 0;
78
+ }
79
+ case 'put':
80
+ print(await collection.put(required(args[0], 'id'), object(parseJson(stdinOr(args[1]), 'document'), 'document')));
81
+ return 0;
82
+ case 'find': {
83
+ const sort = Object.fromEntries((values.sort ?? []).map(spec => {
84
+ const [field, direction = 'asc'] = spec.split(':');
85
+ if (direction !== 'asc' && direction !== 'desc')
86
+ throw new ValidationError(`--sort ${spec}: use field, field:asc or field:desc`);
87
+ return [field, direction === 'asc' ? 1 : -1];
88
+ }));
89
+ const number = (text) => (text === undefined ? undefined : Number(text));
90
+ print(await collection.find(filter(), { sort, limit: number(values.limit), offset: number(values.offset) }));
91
+ return 0;
92
+ }
93
+ case 'count':
94
+ print(await collection.count(filter()));
95
+ return 0;
96
+ case 'delete':
97
+ await collection.delete(required(args[0], 'id'));
98
+ return 0;
99
+ case 'export':
100
+ for (const { _etag, ...document } of await collection.find({}))
101
+ process.stdout.write(`${JSON.stringify(document)}\n`);
102
+ return 0;
103
+ case 'import': {
104
+ // Written in the storage format directly so a restored backup keeps ids and timestamps.
105
+ const prefix = docsPrefix(dbPath, validateName('collection name', name));
106
+ const lines = stdinOr(args[0] === undefined ? undefined : readFileSync(args[0], 'utf8'))
107
+ .split('\n')
108
+ .filter(line => line.trim());
109
+ const documents = lines.map((line, index) => object(parseJson(line, `line ${index + 1}`), `line ${index + 1}`));
110
+ for (let start = 0; start < documents.length; start += 16) {
111
+ await Promise.all(documents.slice(start, start + 16).map(({ _etag, ...document }) => {
112
+ if (typeof document.id !== 'string')
113
+ return collection.insert(document);
114
+ const { id, ...body } = document;
115
+ return adapter.put(`${prefix}${validateName('document ID', id)}.json`, encoder.encode(JSON.stringify({ id, ...body })));
116
+ }));
117
+ }
118
+ print(documents.length);
119
+ return 0;
120
+ }
121
+ case 'compact':
122
+ await collection.compact();
123
+ return 0;
124
+ default:
125
+ throw new ValidationError(`Unknown command: ${command}\n\n${USAGE}`);
126
+ }
127
+ }
128
+ main().then(code => {
129
+ process.exitCode = code;
130
+ }, (error) => {
131
+ console.error(`bucket-db: ${error instanceof BucketDBError || error instanceof TypeError ? error.message : String(error)}`);
132
+ process.exitCode = 1;
133
+ });
@@ -0,0 +1,78 @@
1
+ import type { Document, InsertDocument, InsertOptions, UpdateDocument, QueryFilter, QueryOptions, UpdateOptions, Logger } from './types.js';
2
+ import type { StorageAdapter } from './storage/adapter.js';
3
+ export interface CollectionOptions {
4
+ /** Receives snapshot write failures and envelope warnings. Default: console.warn. */
5
+ logger?: Logger;
6
+ }
7
+ /**
8
+ * A set of JSON documents stored one object per document at
9
+ * `<dbPath>/<name>/docs/<id>.json`. Every write is a single storage request.
10
+ */
11
+ export declare class Collection<T extends Document> {
12
+ private readonly adapter;
13
+ private readonly prefix;
14
+ private readonly snapshotKey;
15
+ private readonly logger;
16
+ /** Cleared after a permission error, e.g. read-only credentials. */
17
+ private snapshotWritable;
18
+ private warnedEnvelope;
19
+ constructor(adapter: StorageAdapter, dbPath: string, name: string, options?: CollectionOptions);
20
+ private keyOf;
21
+ /** The key decides the id and the service decides the etag, whatever the body says. */
22
+ private decode;
23
+ private encode;
24
+ /**
25
+ * Creates a document. One conditional PUT. With `options.id` the call fails
26
+ * with DocumentExistsError when that id is taken; otherwise an id is generated.
27
+ */
28
+ insert(data: InsertDocument<T>, options?: InsertOptions): Promise<T>;
29
+ /**
30
+ * Creates or replaces the whole document under `id`. One unconditional PUT, so
31
+ * repeating it is harmless; `_createdAt` is reset, the old document is not read.
32
+ */
33
+ put(id: string, data: InsertDocument<T>): Promise<T>;
34
+ /** Returns `null` only when the document does not exist; other failures throw. One GET. */
35
+ findById(id: string): Promise<T | null>;
36
+ /**
37
+ * Lists the collection, reads the documents (from the snapshot where it is
38
+ * current) and filters, sorts and pages in memory. An `id` equality filter
39
+ * reads a single document instead.
40
+ */
41
+ find(filter?: QueryFilter<T>, options?: QueryOptions): Promise<T[]>;
42
+ /** Without a filter this only lists the collection; with one it is `find(filter).length`. */
43
+ count(filter?: QueryFilter<T>): Promise<number>;
44
+ /**
45
+ * Deletes every document matching `filter` and returns how many. Not atomic:
46
+ * documents are deleted one by one, 16 at a time.
47
+ */
48
+ deleteMany(filter: QueryFilter<T>): Promise<number>;
49
+ private idOf;
50
+ private readSnapshot;
51
+ /** Lost, overwritten or failed snapshot writes only cost speed, so failures are logged, not thrown. */
52
+ private writeSnapshot;
53
+ private checkEnvelope;
54
+ /** Every document key in the collection with its etag, in key order. */
55
+ private listDocuments;
56
+ /**
57
+ * Lists the collection, then takes each document from the snapshot when its
58
+ * etag still matches and GETs the rest. The list is the authority: the
59
+ * snapshot never adds, removes or changes a document in the result.
60
+ */
61
+ private readAll;
62
+ /**
63
+ * Rewrites the snapshot from the documents as they are now, removing the
64
+ * contents of deleted documents from it. No-op on local backends.
65
+ */
66
+ compact(): Promise<void>;
67
+ /**
68
+ * Merges `data` into the stored document.
69
+ *
70
+ * With `etag`: fails with ConcurrentUpdateError unless the document still has
71
+ * that etag. Without: on backends that support compare-and-swap, conflicting
72
+ * writers are retried so concurrent partial updates all apply; otherwise the
73
+ * last write wins.
74
+ */
75
+ update(id: string, data: UpdateDocument<T>, options?: UpdateOptions): Promise<T>;
76
+ /** Deletes the document if it exists. One DELETE. */
77
+ delete(id: string): Promise<void>;
78
+ }
@@ -0,0 +1,321 @@
1
+ import { setTimeout as sleep } from 'node:timers/promises';
2
+ import { getMany } from './storage/adapter.js';
3
+ import { ConcurrentUpdateError, DocumentExistsError, DocumentNotFoundError, StorageError, UnsupportedError, ValidationError, } from './errors.js';
4
+ import { docsPrefix, isValidName, validateName } from './keys.js';
5
+ import { evaluateFilter, sortDocuments, validateFilter } from './query.js';
6
+ import { SNAPSHOT_NAME, encodeSnapshot, parseSnapshot } from './snapshot.js';
7
+ /** Conflicting writes an etag-less update retries before giving up. */
8
+ const UPDATE_RETRIES = 5;
9
+ const encoder = new TextEncoder();
10
+ const decoder = new TextDecoder();
11
+ /** Drops system fields a caller may have passed in at runtime. */
12
+ function withoutSystemFields(data) {
13
+ const { id: _id, _etag, _createdAt, _updatedAt, ...rest } = data;
14
+ return rest;
15
+ }
16
+ /** Documents fetched or dropped beyond which a query rewrites the snapshot. */
17
+ const SNAPSHOT_REWRITE_THRESHOLD = 16;
18
+ /** Collections beyond this are outside the documented envelope. */
19
+ const ENVELOPE_DOCUMENTS = 10_000;
20
+ const ENVELOPE_BYTES = 10 * 1024 * 1024;
21
+ const defaultLogger = ({ message, error }) => console.warn(`[bucket-db] ${message}`, ...(error ? [error] : []));
22
+ /**
23
+ * A set of JSON documents stored one object per document at
24
+ * `<dbPath>/<name>/docs/<id>.json`. Every write is a single storage request.
25
+ */
26
+ export class Collection {
27
+ adapter;
28
+ prefix;
29
+ snapshotKey;
30
+ logger;
31
+ /** Cleared after a permission error, e.g. read-only credentials. */
32
+ snapshotWritable = true;
33
+ warnedEnvelope = false;
34
+ constructor(adapter, dbPath, name, options = {}) {
35
+ this.adapter = adapter;
36
+ this.prefix = docsPrefix(dbPath, validateName('collection name', name));
37
+ this.snapshotKey = `${this.prefix.slice(0, -'docs/'.length)}${SNAPSHOT_NAME}`;
38
+ this.logger = options.logger ?? defaultLogger;
39
+ }
40
+ keyOf(id) {
41
+ return `${this.prefix}${validateName('document ID', id)}.json`;
42
+ }
43
+ /** The key decides the id and the service decides the etag, whatever the body says. */
44
+ decode(key, id, body, etag) {
45
+ let data;
46
+ try {
47
+ data = JSON.parse(decoder.decode(body));
48
+ }
49
+ catch (error) {
50
+ throw new StorageError(`${key} is not valid JSON`, { key, cause: error });
51
+ }
52
+ if (typeof data !== 'object' || data === null || Array.isArray(data)) {
53
+ throw new StorageError(`${key} is not a JSON object`, { key });
54
+ }
55
+ return { ...data, id, _etag: etag };
56
+ }
57
+ encode(document) {
58
+ const { _etag, ...body } = document;
59
+ return encoder.encode(JSON.stringify(body));
60
+ }
61
+ /**
62
+ * Creates a document. One conditional PUT. With `options.id` the call fails
63
+ * with DocumentExistsError when that id is taken; otherwise an id is generated.
64
+ */
65
+ async insert(data, options = {}) {
66
+ const id = options.id ?? crypto.randomUUID();
67
+ const key = this.keyOf(id);
68
+ const atomic = this.adapter.capabilities.createIfAbsent !== 'none';
69
+ if (options.id !== undefined && !atomic) {
70
+ throw new UnsupportedError('This storage backend cannot create a document only if it is absent');
71
+ }
72
+ const timestamp = new Date().toISOString();
73
+ const document = { ...withoutSystemFields(data), id, _createdAt: timestamp, _updatedAt: timestamp };
74
+ try {
75
+ const { etag } = await this.adapter.put(key, this.encode(document), atomic ? { ifNoneMatch: '*' } : undefined);
76
+ return { ...document, _etag: etag };
77
+ }
78
+ catch (error) {
79
+ if (error instanceof StorageError && error.kind === 'conflict')
80
+ throw new DocumentExistsError(id);
81
+ throw error;
82
+ }
83
+ }
84
+ /**
85
+ * Creates or replaces the whole document under `id`. One unconditional PUT, so
86
+ * repeating it is harmless; `_createdAt` is reset, the old document is not read.
87
+ */
88
+ async put(id, data) {
89
+ const timestamp = new Date().toISOString();
90
+ const document = { ...withoutSystemFields(data), id, _createdAt: timestamp, _updatedAt: timestamp };
91
+ const { etag } = await this.adapter.put(this.keyOf(id), this.encode(document));
92
+ return { ...document, _etag: etag };
93
+ }
94
+ /** Returns `null` only when the document does not exist; other failures throw. One GET. */
95
+ async findById(id) {
96
+ const key = this.keyOf(id);
97
+ try {
98
+ const { body, etag } = await this.adapter.get(key);
99
+ return this.decode(key, id, body, etag);
100
+ }
101
+ catch (error) {
102
+ if (error instanceof StorageError && error.kind === 'not-found')
103
+ return null;
104
+ throw error;
105
+ }
106
+ }
107
+ /**
108
+ * Lists the collection, reads the documents (from the snapshot where it is
109
+ * current) and filters, sorts and pages in memory. An `id` equality filter
110
+ * reads a single document instead.
111
+ */
112
+ async find(filter = {}, options = {}) {
113
+ validateFilter(filter);
114
+ sortDocuments([], options.sort); // validates the sort spec before any request
115
+ const { offset = 0, limit, sort } = options;
116
+ if (!Number.isInteger(offset) || offset < 0 || (limit !== undefined && (!Number.isInteger(limit) || limit < 0))) {
117
+ throw new ValidationError('offset and limit must be non-negative integers');
118
+ }
119
+ let documents;
120
+ if (typeof filter.id === 'string') {
121
+ const document = isValidName(filter.id) ? await this.findById(filter.id) : null;
122
+ documents = document ? [document] : [];
123
+ }
124
+ else {
125
+ documents = await this.readAll();
126
+ }
127
+ const matched = documents.filter(document => evaluateFilter(document, filter));
128
+ sortDocuments(matched, sort);
129
+ return matched.slice(offset, limit === undefined ? undefined : offset + limit);
130
+ }
131
+ /** Without a filter this only lists the collection; with one it is `find(filter).length`. */
132
+ async count(filter = {}) {
133
+ if (Object.keys(filter).length > 0)
134
+ return (await this.find(filter)).length;
135
+ return (await this.listDocuments()).length;
136
+ }
137
+ /**
138
+ * Deletes every document matching `filter` and returns how many. Not atomic:
139
+ * documents are deleted one by one, 16 at a time.
140
+ */
141
+ async deleteMany(filter) {
142
+ const matched = await this.find(filter);
143
+ for (let start = 0; start < matched.length; start += 16) {
144
+ await Promise.all(matched.slice(start, start + 16).map(document => this.delete(document.id)));
145
+ }
146
+ return matched.length;
147
+ }
148
+ idOf(key) {
149
+ const name = key.slice(this.prefix.length);
150
+ const id = name.endsWith('.json') ? name.slice(0, -'.json'.length) : '';
151
+ if (!isValidName(id)) {
152
+ throw new StorageError(`${key} is not a document key (expected <id>.json)`, { key });
153
+ }
154
+ return id;
155
+ }
156
+ async readSnapshot() {
157
+ try {
158
+ const { body } = await this.adapter.get(this.snapshotKey);
159
+ const entries = parseSnapshot(body);
160
+ if (!entries)
161
+ this.logger({ level: 'warn', message: `${this.snapshotKey} is not a valid snapshot; rebuilding it` });
162
+ return entries ?? new Map();
163
+ }
164
+ catch (error) {
165
+ // a missing snapshot is rebuilt; any other read failure fails the query
166
+ if (error instanceof StorageError && error.kind === 'not-found')
167
+ return new Map();
168
+ throw error;
169
+ }
170
+ }
171
+ /** Lost, overwritten or failed snapshot writes only cost speed, so failures are logged, not thrown. */
172
+ async writeSnapshot(documents, throwOnError = false) {
173
+ const body = encodeSnapshot(documents);
174
+ this.checkEnvelope(documents.length, body.length);
175
+ if (!this.snapshotWritable && !throwOnError)
176
+ return;
177
+ try {
178
+ await this.adapter.put(this.snapshotKey, body);
179
+ }
180
+ catch (error) {
181
+ if (throwOnError)
182
+ throw error;
183
+ if (error instanceof StorageError && (error.status === 401 || error.status === 403))
184
+ this.snapshotWritable = false;
185
+ this.logger({
186
+ level: 'error',
187
+ message: `Failed to write ${this.snapshotKey}; queries stay correct but slower`,
188
+ error,
189
+ });
190
+ }
191
+ }
192
+ checkEnvelope(documents, bytes) {
193
+ if (this.warnedEnvelope || (documents <= ENVELOPE_DOCUMENTS && bytes <= ENVELOPE_BYTES))
194
+ return;
195
+ this.warnedEnvelope = true;
196
+ this.logger({
197
+ level: 'warn',
198
+ message: `Collection ${this.prefix.slice(0, -1)} has ${documents} documents (${Math.round(bytes / 1024)} kB); BucketDB is designed for at most ${ENVELOPE_DOCUMENTS} documents and ${ENVELOPE_BYTES / 1024 / 1024} MB per collection, and queries will keep getting slower`,
199
+ });
200
+ }
201
+ /** Every document key in the collection with its etag, in key order. */
202
+ async listDocuments() {
203
+ const listed = [];
204
+ let cursor;
205
+ do {
206
+ const page = await this.adapter.list(this.prefix, { cursor });
207
+ for (const { key, etag } of page.entries) {
208
+ if (key.slice(this.prefix.length).startsWith('.'))
209
+ continue; // hidden files (e.g. .DS_Store) are never documents
210
+ listed.push({ key, etag, id: this.idOf(key) });
211
+ }
212
+ cursor = page.cursor;
213
+ } while (cursor);
214
+ return listed;
215
+ }
216
+ /**
217
+ * Lists the collection, then takes each document from the snapshot when its
218
+ * etag still matches and GETs the rest. The list is the authority: the
219
+ * snapshot never adds, removes or changes a document in the result.
220
+ */
221
+ async readAll(useSnapshot = this.adapter.remote === true) {
222
+ const listed = await this.listDocuments();
223
+ const snapshot = useSnapshot ? await this.readSnapshot() : new Map();
224
+ const documents = new Array(listed.length);
225
+ const missing = [];
226
+ let listedInSnapshot = 0;
227
+ listed.forEach(({ etag, id }, index) => {
228
+ const cached = snapshot.get(id);
229
+ if (cached)
230
+ listedInSnapshot++;
231
+ if (cached && cached.etag === etag)
232
+ documents[index] = { ...cached.body, id, _etag: etag };
233
+ else
234
+ missing.push(index);
235
+ });
236
+ const fetched = await getMany(this.adapter, missing.map(index => listed[index].key));
237
+ fetched.forEach((result, position) => {
238
+ if ('error' in result) {
239
+ if (result.error.kind === 'not-found')
240
+ return; // deleted after the list
241
+ throw result.error;
242
+ }
243
+ const index = missing[position];
244
+ documents[index] = this.decode(result.key, listed[index].id, result.body, result.etag);
245
+ });
246
+ const present = documents.filter((document) => document !== undefined);
247
+ // documents read individually (new or changed) plus snapshot entries of deleted documents
248
+ const deleted = snapshot.size - listedInSnapshot;
249
+ if (useSnapshot && missing.length + deleted > SNAPSHOT_REWRITE_THRESHOLD) {
250
+ await this.writeSnapshot(present);
251
+ }
252
+ else {
253
+ this.checkEnvelope(present.length, 0);
254
+ }
255
+ return present;
256
+ }
257
+ /**
258
+ * Rewrites the snapshot from the documents as they are now, removing the
259
+ * contents of deleted documents from it. No-op on local backends.
260
+ */
261
+ async compact() {
262
+ if (this.adapter.remote !== true)
263
+ return;
264
+ await this.writeSnapshot(await this.readAll(false), true);
265
+ }
266
+ /**
267
+ * Merges `data` into the stored document.
268
+ *
269
+ * With `etag`: fails with ConcurrentUpdateError unless the document still has
270
+ * that etag. Without: on backends that support compare-and-swap, conflicting
271
+ * writers are retried so concurrent partial updates all apply; otherwise the
272
+ * last write wins.
273
+ */
274
+ async update(id, data, options = {}) {
275
+ const key = this.keyOf(id);
276
+ const compareAndSwap = this.adapter.capabilities.compareAndSwap !== 'none';
277
+ if (options.etag !== undefined && !compareAndSwap) {
278
+ throw new UnsupportedError('This storage backend cannot enforce an etag on update');
279
+ }
280
+ const changes = withoutSystemFields(data);
281
+ for (let attempt = 0;; attempt++) {
282
+ const current = await this.findById(id);
283
+ if (!current)
284
+ throw new DocumentNotFoundError(id);
285
+ if (options.etag !== undefined && current._etag !== options.etag) {
286
+ throw new ConcurrentUpdateError(`ETag mismatch for document ${id}`);
287
+ }
288
+ const updated = {
289
+ ...current,
290
+ ...changes,
291
+ id,
292
+ _createdAt: current._createdAt,
293
+ _updatedAt: new Date().toISOString(),
294
+ };
295
+ try {
296
+ const condition = compareAndSwap ? { ifMatch: current._etag ?? '' } : undefined;
297
+ const { etag } = await this.adapter.put(key, this.encode(updated), condition);
298
+ return { ...updated, _etag: etag };
299
+ }
300
+ catch (error) {
301
+ if (!(error instanceof StorageError))
302
+ throw error;
303
+ if (error.kind === 'not-found')
304
+ throw new DocumentNotFoundError(id);
305
+ if (error.kind !== 'conflict')
306
+ throw error;
307
+ if (options.etag !== undefined) {
308
+ throw new ConcurrentUpdateError(`Document ${id} was modified by another writer`);
309
+ }
310
+ if (attempt === UPDATE_RETRIES) {
311
+ throw new ConcurrentUpdateError(`Document ${id} kept changing; gave up after ${attempt + 1} attempts`);
312
+ }
313
+ await sleep(Math.random() * Math.min(1000, 25 * 2 ** attempt));
314
+ }
315
+ }
316
+ }
317
+ /** Deletes the document if it exists. One DELETE. */
318
+ async delete(id) {
319
+ await this.adapter.delete(this.keyOf(id));
320
+ }
321
+ }
@@ -0,0 +1,7 @@
1
+ import type { StorageAdapter } from './storage/adapter.js';
2
+ /**
3
+ * Checks what a storage service actually guarantees, writing only under
4
+ * `<dbPath>/_doctor-<time>/` and removing it afterwards. Prints one line per
5
+ * check; returns the number of failed checks.
6
+ */
7
+ export declare function doctor(adapter: StorageAdapter, dbPath: string, print: (line: string) => void): Promise<number>;
package/dist/doctor.js ADDED
@@ -0,0 +1,100 @@
1
+ import { StorageError } from './errors.js';
2
+ const encoder = new TextEncoder();
3
+ const decoder = new TextDecoder();
4
+ /** Concurrent writers on one key; returns how many succeeded and what the rest failed with. */
5
+ async function race(writes) {
6
+ const results = await Promise.allSettled(writes.map(write => write()));
7
+ const wins = results.filter(result => result.status === 'fulfilled').length;
8
+ const other = results.find(result => result.status === 'rejected' && !(result.reason instanceof StorageError && result.reason.kind === 'conflict'));
9
+ return { wins, other: other?.status === 'rejected' ? String(other.reason) : undefined };
10
+ }
11
+ /**
12
+ * Checks what a storage service actually guarantees, writing only under
13
+ * `<dbPath>/_doctor-<time>/` and removing it afterwards. Prints one line per
14
+ * check; returns the number of failed checks.
15
+ */
16
+ export async function doctor(adapter, dbPath, print) {
17
+ const base = `${dbPath ? `${dbPath}/` : ''}_doctor-${Date.now()}/`;
18
+ const { createIfAbsent, compareAndSwap } = adapter.capabilities;
19
+ const listKeys = async (prefix) => (await adapter.list(prefix)).entries;
20
+ const checks = [
21
+ [
22
+ 'write, read and delete an object',
23
+ async () => {
24
+ const { etag } = await adapter.put(`${base}a.json`, encoder.encode('{"a":1}'));
25
+ const read = await adapter.get(`${base}a.json`);
26
+ if (decoder.decode(read.body) !== '{"a":1}' || read.etag !== etag)
27
+ return 'read back different bytes or etag';
28
+ await adapter.delete(`${base}a.json`);
29
+ const gone = await adapter.get(`${base}a.json`).then(() => false, (error) => error instanceof StorageError && error.kind === 'not-found');
30
+ return gone || 'object still readable after delete';
31
+ },
32
+ ],
33
+ [
34
+ 'list reflects writes, overwrites and deletes immediately, with matching etags',
35
+ async () => {
36
+ const keys = [0, 1, 2, 3, 4].map(n => `${base}list/${n}.json`);
37
+ const etags = await Promise.all(keys.map(key => adapter.put(key, encoder.encode(key)).then(result => result.etag)));
38
+ let listed = await listKeys(`${base}list/`);
39
+ if (listed.map(entry => entry.key).join() !== keys.join())
40
+ return 'a new object was missing from the list';
41
+ if (listed.some((entry, index) => entry.etag !== etags[index]))
42
+ return 'list etags differ from write etags';
43
+ const { etag } = await adapter.put(keys[0], encoder.encode('changed'));
44
+ await adapter.delete(keys[1]);
45
+ listed = await listKeys(`${base}list/`);
46
+ if (listed[0]?.etag !== etag)
47
+ return 'list showed an old etag after an overwrite';
48
+ return listed.length === 4 || 'list still showed a deleted object';
49
+ },
50
+ ],
51
+ [
52
+ `create-if-absent (declared: ${createIfAbsent})`,
53
+ async () => {
54
+ if (createIfAbsent === 'none')
55
+ return 'skip: not supported by this backend';
56
+ const key = `${base}create.json`;
57
+ const { wins, other } = await race(Array.from({ length: 10 }, (_, n) => () => adapter.put(key, encoder.encode(`${n}`), { ifNoneMatch: '*' })));
58
+ return other ?? (wins === 1 || `${wins} of 10 concurrent creates succeeded (a versioned OSS bucket does this)`);
59
+ },
60
+ ],
61
+ [
62
+ `compare-and-swap (declared: ${compareAndSwap})`,
63
+ async () => {
64
+ if (compareAndSwap === 'none')
65
+ return 'skip: not supported by this backend; etag updates are refused';
66
+ const key = `${base}swap.json`;
67
+ const { etag } = await adapter.put(key, encoder.encode('base'));
68
+ const { wins, other } = await race(Array.from({ length: 10 }, (_, n) => () => adapter.put(key, encoder.encode(`${n}`), { ifMatch: etag })));
69
+ if (other)
70
+ return other;
71
+ if (wins !== 1)
72
+ return `${wins} of 10 concurrent conditional updates succeeded`;
73
+ const stale = await race([() => adapter.put(key, encoder.encode('stale'), { ifMatch: etag })]);
74
+ return stale.wins === 0 || 'a write with a stale etag succeeded';
75
+ },
76
+ ],
77
+ ];
78
+ let failed = 0;
79
+ try {
80
+ for (const [name, check] of checks) {
81
+ const outcome = await check().catch((error) => `error: ${error instanceof Error ? error.message : String(error)}`);
82
+ if (outcome !== true && !outcome.startsWith('skip'))
83
+ failed++;
84
+ print(outcome === true
85
+ ? `ok ${name}`
86
+ : outcome.startsWith('skip')
87
+ ? `skip ${name}: ${outcome.slice(6)}`
88
+ : `FAIL ${name}: ${outcome}`);
89
+ }
90
+ }
91
+ finally {
92
+ let leftovers = await listKeys(base);
93
+ while (leftovers.length > 0) {
94
+ await Promise.all(leftovers.map(entry => adapter.delete(entry.key)));
95
+ leftovers = await listKeys(base);
96
+ }
97
+ }
98
+ print(failed === 0 ? 'all checks passed' : `${failed} check(s) failed`);
99
+ return failed;
100
+ }