@hold-baby/bucket-db 0.3.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (103) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +503 -19
  3. package/dist/bucket-db.d.ts +25 -0
  4. package/dist/bucket-db.js +36 -0
  5. package/dist/cli.d.ts +2 -0
  6. package/dist/cli.js +133 -0
  7. package/dist/collection.d.ts +78 -0
  8. package/dist/collection.js +321 -0
  9. package/dist/doctor.d.ts +7 -0
  10. package/dist/doctor.js +100 -0
  11. package/dist/errors.d.ts +59 -0
  12. package/dist/errors.js +57 -0
  13. package/dist/index.d.ts +13 -0
  14. package/dist/index.js +6 -0
  15. package/dist/keys.d.ts +12 -0
  16. package/dist/keys.js +31 -0
  17. package/dist/query.d.ts +16 -0
  18. package/dist/query.js +133 -0
  19. package/dist/snapshot.d.ts +16 -0
  20. package/dist/snapshot.js +40 -0
  21. package/dist/storage/adapter.d.ts +75 -0
  22. package/dist/storage/adapter.js +27 -0
  23. package/dist/storage/fs.d.ts +28 -0
  24. package/dist/storage/fs.js +175 -0
  25. package/dist/storage/memory.d.ts +18 -0
  26. package/dist/storage/memory.js +49 -0
  27. package/dist/storage/s3/adapters.d.ts +38 -0
  28. package/dist/storage/s3/adapters.js +65 -0
  29. package/dist/storage/s3/client.d.ts +52 -0
  30. package/dist/storage/s3/client.js +231 -0
  31. package/dist/storage/s3/credentials.d.ts +17 -0
  32. package/dist/storage/s3/credentials.js +42 -0
  33. package/dist/storage/s3/sigv4.d.ts +32 -0
  34. package/dist/storage/s3/sigv4.js +75 -0
  35. package/dist/types.d.ts +55 -0
  36. package/dist/types.js +1 -0
  37. package/dist/url.d.ts +16 -0
  38. package/dist/url.js +66 -0
  39. package/llms.txt +142 -0
  40. package/package.json +20 -7
  41. package/src/cache/lru-cache.ts +0 -131
  42. package/src/core/bucket-db.ts +0 -47
  43. package/src/core/collection.ts +0 -278
  44. package/src/core/wal.ts +0 -106
  45. package/src/index/shard-manager.ts +0 -197
  46. package/src/index.ts +0 -37
  47. package/src/query/evaluator.ts +0 -143
  48. package/src/storage/ali-oss.d.ts +0 -43
  49. package/src/storage/fs-adapter.ts +0 -248
  50. package/src/storage/memory-adapter.ts +0 -105
  51. package/src/storage/oss-adapter.ts +0 -164
  52. package/src/storage/s3-adapter.ts +0 -200
  53. package/src/types/collection.ts +0 -43
  54. package/src/types/document.ts +0 -22
  55. package/src/types/errors.ts +0 -49
  56. package/src/types/index.ts +0 -37
  57. package/src/types/query.ts +0 -50
  58. package/src/types/storage.ts +0 -46
  59. package/src/utils/hash.ts +0 -24
  60. package/src/utils/path-validator.ts +0 -119
  61. package/tests/core/bucket-db.test.d.ts +0 -2
  62. package/tests/core/bucket-db.test.d.ts.map +0 -1
  63. package/tests/core/bucket-db.test.js +0 -34
  64. package/tests/core/bucket-db.test.js.map +0 -1
  65. package/tests/core/bucket-db.test.ts +0 -50
  66. package/tests/core/collection.test.d.ts +0 -2
  67. package/tests/core/collection.test.d.ts.map +0 -1
  68. package/tests/core/collection.test.js +0 -148
  69. package/tests/core/collection.test.js.map +0 -1
  70. package/tests/core/collection.test.ts +0 -195
  71. package/tests/core/dbpath.test.ts +0 -52
  72. package/tests/index/shard-manager.test.d.ts +0 -2
  73. package/tests/index/shard-manager.test.d.ts.map +0 -1
  74. package/tests/index/shard-manager.test.js +0 -106
  75. package/tests/index/shard-manager.test.js.map +0 -1
  76. package/tests/index/shard-manager.test.ts +0 -129
  77. package/tests/query/evaluator.test.d.ts +0 -2
  78. package/tests/query/evaluator.test.d.ts.map +0 -1
  79. package/tests/query/evaluator.test.js +0 -87
  80. package/tests/query/evaluator.test.js.map +0 -1
  81. package/tests/query/evaluator.test.ts +0 -104
  82. package/tests/storage/fs-adapter.test.ts +0 -111
  83. package/tests/storage/memory-adapter.test.d.ts +0 -2
  84. package/tests/storage/memory-adapter.test.d.ts.map +0 -1
  85. package/tests/storage/memory-adapter.test.js +0 -65
  86. package/tests/storage/memory-adapter.test.js.map +0 -1
  87. package/tests/storage/memory-adapter.test.ts +0 -84
  88. package/tests/storage/oss-adapter.test.d.ts +0 -2
  89. package/tests/storage/oss-adapter.test.d.ts.map +0 -1
  90. package/tests/storage/oss-adapter.test.js +0 -34
  91. package/tests/storage/oss-adapter.test.js.map +0 -1
  92. package/tests/storage/oss-adapter.test.ts +0 -40
  93. package/tests/storage/s3-adapter.test.d.ts +0 -2
  94. package/tests/storage/s3-adapter.test.d.ts.map +0 -1
  95. package/tests/storage/s3-adapter.test.js +0 -51
  96. package/tests/storage/s3-adapter.test.js.map +0 -1
  97. package/tests/storage/s3-adapter.test.ts +0 -62
  98. package/tests/utils/hash.test.d.ts +0 -2
  99. package/tests/utils/hash.test.d.ts.map +0 -1
  100. package/tests/utils/hash.test.js +0 -36
  101. package/tests/utils/hash.test.js.map +0 -1
  102. package/tests/utils/hash.test.ts +0 -42
  103. package/tsconfig.json +0 -12
package/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 holdbaby5483
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
package/README.md CHANGED
@@ -1,38 +1,522 @@
1
- # @bucket-db/core
1
+ # BucketDB
2
2
 
3
- Core database engine for BucketDB.
3
+ [![npm version](https://img.shields.io/npm/v/@hold-baby/bucket-db.svg)](https://www.npmjs.com/package/@hold-baby/bucket-db)
4
+ [![npm downloads](https://img.shields.io/npm/dm/@hold-baby/bucket-db.svg)](https://www.npmjs.com/package/@hold-baby/bucket-db)
5
+ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
4
6
 
5
- See the [main README](../../README.md) for usage and documentation.
7
+ A TypeScript document database built on cloud object storage (AWS S3 / Alibaba Cloud OSS) or local file system. No database servers needed.
6
8
 
7
- ## API
9
+ Each document is one JSON object in your bucket: `<dbPath>/<collection>/docs/<id>.json`. There is no index and no log; every write is a single storage request. On S3 and OSS, queries also keep a disposable snapshot, `<dbPath>/<collection>/_snapshot.v1.json`, so a query does not download every document ([storage format](https://github.com/holdbaby5483/bucket-db/blob/main/docs/STORAGE-FORMAT.md)).
8
10
 
9
- ### BucketDB
11
+ > **0.5.0** adds connection URLs (`BucketDB.connect`), the `bucket-db` CLI with `doctor`, `put`, `insert` with a chosen id, `sort`, `count` and `deleteMany`, and settles query semantics. Upgrading from 0.4? Read [MIGRATION-0.5.md](https://github.com/holdbaby5483/bucket-db/blob/main/docs/MIGRATION-0.5.md); from 0.3.x, [MIGRATION-0.4.md](https://github.com/holdbaby5483/bucket-db/blob/main/docs/MIGRATION-0.4.md) first.
10
12
 
11
- Main database class.
13
+ ## Status and limitations
14
+
15
+ What each backend guarantees:
16
+
17
+ | | Memory | FileSystem | S3 | OSS |
18
+ |---|---|---|---|---|
19
+ | Single write is atomic | yes | yes | yes | yes |
20
+ | Create-if-absent (`insert`) | yes | yes (across processes) | yes | yes (non-versioned buckets only) |
21
+ | Conditional update (`etag`) | yes | single process only | yes | no — `UnsupportedError` |
22
+ | Concurrent partial updates all apply | yes | single process | yes | no (last writer wins) |
23
+
24
+ Multiple writer processes are supported for `insert` and `delete` on every backend, and for `update` on S3 (on Memory and FileSystem, within one process). In a test, 4 processes each inserting 25 documents left all 100 visible on FileSystem, S3 and OSS. [Guarantees and limits](https://github.com/holdbaby5483/bucket-db/blob/main/docs/GUARANTEES.md) names the test behind each row.
25
+
26
+ Remaining limits:
27
+
28
+ - **No transactions.** Each write touches one document. `deleteMany` deletes its matches one by one and is not atomic.
29
+ - **Envelope: at most 10,000 documents and 10 MB of snapshot per collection.** Past either, queries keep working but get slower, and the collection logs one warning. At 10,000 documents a `find` takes about 1.35 s on OSS and 3.2 s on S3, and about 0.5 s on a local FileSystem ([benchmarks](https://github.com/holdbaby5483/bucket-db/blob/main/docs/BENCHMARKS.md)). BucketDB is meant for small collections and a few writes per second.
30
+ - **Queries list the whole collection.** `find` lists every document key, then filters, sorts and pages in memory; `sort`, `limit` and `offset` do not reduce reads. On S3 / OSS document bodies come from the snapshot, plus a GET for each document changed since it was written; Memory and FileSystem read every file. Only a `{ id: '...' }` filter reads a single document, and only `count()` without a filter reads no documents.
31
+ - **Results are not a point-in-time snapshot.** Every returned document is a version that was committed, but documents written while a query runs may or may not be included.
32
+ - **The snapshot holds document contents.** A deleted document's contents stay in `_snapshot.v1.json` until the snapshot is rewritten or `compact()` runs. Call `compact()` after deleting privacy-sensitive data.
33
+ - **OSS has no conditional update.** `update` with `etag` throws `UnsupportedError`; concurrent `update`s without `etag` are last-writer-wins.
34
+ - **FileSystem conditional updates are single-process.** `etag` checks and the merging of concurrent `update`s hold only among writers in one process.
35
+ - **0.3.x and 0.4 or later must not write the same collection at the same time.** 0.3 reads only its own index and does not see later writes.
36
+
37
+ ## Installation
38
+
39
+ ```bash
40
+ bun add @hold-baby/bucket-db
41
+ # or
42
+ npm install @hold-baby/bucket-db
43
+ ```
44
+
45
+ Requires Node.js >= 20 or Bun. No runtime dependencies: the S3 / OSS client is built in.
46
+
47
+ ## Command line
48
+
49
+ The package includes `bucket-db`, for shells and CI jobs:
50
+
51
+ ```bash
52
+ export BUCKET_DB_URL='s3://my-bucket/ci-data?region=us-east-1' # credentials from AWS_* variables
53
+
54
+ npx @hold-baby/bucket-db doctor # check what the bucket guarantees
55
+ echo '{"ms":1234,"ok":true}' | npx @hold-baby/bucket-db put runs run-42 # create or replace
56
+ npx @hold-baby/bucket-db get runs run-42 | jq .ms
57
+ npx @hold-baby/bucket-db find runs '{"ok":true}' --sort ms:desc --limit 5
58
+ npx @hold-baby/bucket-db count runs
59
+ npx @hold-baby/bucket-db export runs > runs.jsonl # backup; `import` restores
60
+ ```
61
+
62
+ Commands: `get`, `put`, `find` (`--sort field[:desc]`, `--limit`, `--offset`), `count`, `delete`, `export`, `import`, `compact`, `doctor`. stdout is one JSON value per line, errors go to stderr; exit code 0 on success, 1 on error, 2 when `get` finds nothing. Full reference: [CLI.md](https://github.com/holdbaby5483/bucket-db/blob/main/docs/CLI.md). Recipes for keeping CI data in a bucket with OIDC credentials: [examples/ci](https://github.com/holdbaby5483/bucket-db/tree/main/examples/ci).
63
+
64
+ ## Connection URLs
65
+
66
+ `BucketDB.connect(url?)`, the CLI and the MCP server open a database from one URL, by default the `BUCKET_DB_URL` environment variable:
67
+
68
+ | URL | Backend | `dbPath` |
69
+ |---|---|---|
70
+ | `s3://bucket/prefix?region=…&endpoint=…&forcePathStyle=true&timeoutMs=…` | `S3Adapter` | `prefix` |
71
+ | `oss://bucket/prefix?region=…&endpoint=…&timeoutMs=…` | `OSSAdapter` | `prefix` |
72
+ | `file:///absolute/dir`, `file://./relative/dir` | `FileSystemAdapter` | `''` |
73
+ | `memory://prefix` | `MemoryStorageAdapter` | `prefix` |
74
+
75
+ ```typescript
76
+ const db = BucketDB.connect('s3://my-bucket/my-app?region=us-east-1');
77
+ const fromEnv = BucketDB.connect(); // BUCKET_DB_URL
78
+ ```
79
+
80
+ Every query parameter is optional and falls back to the adapter's environment default; unknown parameters throw `ValidationError`. Credentials never go in the URL (a URL with `user:secret@` is refused) but come from `AWS_*`, `OSS_*` or `ALIBABA_CLOUD_*` variables (see [Storage Adapters](#storage-adapters)). Error messages never repeat the URL, so it is safe to log. `new BucketDB(adapter, dbPath)` remains available, for example to pass a credential provider.
81
+
82
+ ## Exports
83
+
84
+ ```typescript
85
+ // Core classes
86
+ import { BucketDB, Collection } from '@hold-baby/bucket-db';
87
+
88
+ // Storage adapters
89
+ import {
90
+ MemoryStorageAdapter, // in-memory (tests)
91
+ FileSystemAdapter, // local file system
92
+ S3Adapter, // AWS S3 and S3-compatible services
93
+ OSSAdapter, // Alibaba Cloud OSS
94
+ } from '@hold-baby/bucket-db';
95
+
96
+ // Types
97
+ import type {
98
+ BucketDBOptions,
99
+ CollectionOptions,
100
+ Logger,
101
+ LogEvent,
102
+ Document,
103
+ InsertDocument,
104
+ UpdateDocument,
105
+ QueryOperator,
106
+ QueryValue,
107
+ QueryFilter,
108
+ QueryOptions,
109
+ InsertOptions,
110
+ UpdateOptions,
111
+ FileSystemAdapterConfig,
112
+ S3AdapterOptions,
113
+ OSSAdapterOptions,
114
+ Credentials,
115
+ CredentialProvider,
116
+ // for custom adapters
117
+ StorageAdapter,
118
+ Capabilities,
119
+ Guarantee,
120
+ WriteCondition,
121
+ ObjectEntry,
122
+ ListPage,
123
+ ListOptions,
124
+ StorageErrorKind,
125
+ StorageErrorOptions,
126
+ } from '@hold-baby/bucket-db';
127
+
128
+ // Error classes
129
+ import {
130
+ BucketDBError,
131
+ ValidationError,
132
+ DocumentNotFoundError,
133
+ DocumentExistsError,
134
+ ConcurrentUpdateError,
135
+ UnsupportedError,
136
+ StorageError,
137
+ } from '@hold-baby/bucket-db';
138
+ ```
139
+
140
+ ## Document Type
141
+
142
+ Every document stored in BucketDB has these system fields:
143
+
144
+ ```typescript
145
+ interface Document {
146
+ id: string; // generated UUID unless you choose it (put, insert with { id }); taken from the object key
147
+ _etag?: string; // version tag from the storage service, for optimistic locking
148
+ _createdAt?: string; // ISO 8601 timestamp
149
+ _updatedAt?: string; // ISO 8601 timestamp
150
+ }
151
+
152
+ // Extend Document to define your own schema
153
+ interface User extends Document {
154
+ name: string;
155
+ age: number;
156
+ email: string;
157
+ }
158
+ ```
159
+
160
+ `InsertDocument<T>` and `UpdateDocument<T>` both omit `id`, `_etag`, `_createdAt` and `_updatedAt`; `UpdateDocument<T>` also makes every field optional. If system fields are passed at runtime anyway, `insert`, `put` and `update` drop them.
161
+
162
+ `_etag` is not stored in the document body: on every read, `id` comes from the object key and `_etag` from the storage service.
163
+
164
+ ## Quick Start
165
+
166
+ ```typescript
167
+ import { BucketDB } from '@hold-baby/bucket-db';
168
+ import type { Document } from '@hold-baby/bucket-db';
169
+
170
+ interface User extends Document {
171
+ name: string;
172
+ age: number;
173
+ email: string;
174
+ status: 'active' | 'inactive';
175
+ }
176
+
177
+ // Or s3://bucket/prefix?region=…, oss://…, memory://; new BucketDB(adapter, dbPath) also works
178
+ const db = BucketDB.connect('file://./data');
179
+
180
+ const users = db.collection<User>('users');
181
+
182
+ // Insert: one conditional PUT
183
+ const user = await users.insert({ name: 'Alice', age: 25, email: 'alice@example.com', status: 'active' });
184
+ // user.id, user._etag, user._createdAt, user._updatedAt are set automatically
185
+
186
+ // Find by ID: one GET; null if the document does not exist
187
+ const found = await users.findById(user.id);
188
+
189
+ // Query: filters in memory (on S3 / OSS, document bodies come from the query snapshot)
190
+ const activeUsers = await users.find({ status: 'active' });
191
+ const adults = await users.find({ age: { $gte: 18 } }, { sort: { age: -1 }, limit: 10 });
192
+ const total = await users.count();
193
+
194
+ // Create or replace under an ID you choose: one unconditional PUT, safe to repeat
195
+ await users.put('alice', { name: 'Alice', age: 25, email: 'alice@example.com', status: 'active' });
196
+
197
+ // Create only if the ID is free: throws DocumentExistsError otherwise
198
+ await users.insert({ name: 'Bob', age: 31, email: 'bob@example.com', status: 'active' }, { id: 'bob' });
199
+
200
+ // Update with optimistic locking (not available on OSS, see above)
201
+ const updated = await users.update(user.id, { age: 26 }, { etag: user._etag });
202
+
203
+ // Delete: one DELETE; no error if the document is already gone
204
+ await users.delete(user.id);
205
+
206
+ // Delete every match, one DELETE each (not atomic); returns how many
207
+ await users.deleteMany({ status: 'inactive' });
208
+ ```
209
+
210
+ ## Operations and cost
211
+
212
+ | Operation | Requests |
213
+ |---|---|
214
+ | `insert(data, { id? })` | 1 conditional PUT |
215
+ | `put(id, data)` | 1 PUT |
216
+ | `findById(id)` | 1 GET |
217
+ | `find({ id: '...' })` | 1 GET |
218
+ | `find(filter)` on S3 / OSS | ⌈N / 1000⌉ LIST + 1 GET (snapshot), plus 1 GET per document changed since the snapshot (up to 16, in parallel) or, when more changed, those GETs and 1 PUT rewriting the snapshot |
219
+ | `find(filter)` on Memory / FileSystem | ⌈N / 1000⌉ LIST + N GET (16 in parallel) |
220
+ | `update(id, data, options?)` | 1 GET + 1 conditional PUT (more on conflict, see below) |
221
+ | `delete(id)` | 1 DELETE |
222
+ | `count()` | ⌈N / 1000⌉ LIST |
223
+ | `count(filter)` | same as `find(filter)` |
224
+ | `deleteMany(filter)` | same as `find(filter)`, plus 1 DELETE per match (16 in parallel) |
225
+ | `compact()` | S3 / OSS: ⌈N / 1000⌉ LIST + N GET + 1 PUT; nothing on Memory / FileSystem |
226
+
227
+ N = documents in the collection. Measured latencies at 100, 1,000 and 10,000 documents are in [BENCHMARKS.md](https://github.com/holdbaby5483/bucket-db/blob/main/docs/BENCHMARKS.md).
228
+
229
+ Without `sort`, `find` returns documents by `_createdAt`, then `id`: roughly creation order, and stable for paging. `sort: { field: 1 | -1 }` sorts by the given fields first; missing and `null` values sort last in either direction. `sort`, `offset` and `limit` apply after filtering, and `limit: 0` returns `[]`. There is no result cache: every `find` lists storage first.
230
+
231
+ `put` replaces the whole document and sets both timestamps to now (so it moves to the end of the default order); `update` merges fields and keeps `_createdAt`. `insert` with `{ id }` creates only if the ID is free; of several concurrent inserts with one ID, exactly one succeeds.
232
+
233
+ `update` merges `data` into the stored document:
234
+
235
+ - **With `etag`**: throws `ConcurrentUpdateError` if the document no longer has that etag, or if another writer changes it before the write lands. No retry.
236
+ - **Without `etag`**: on backends with compare-and-swap (Memory, FileSystem within one process, S3), a write that collides with another writer re-reads and retries (up to 5 retries), so concurrent partial updates all apply. On OSS the last writer wins.
237
+
238
+ ## Query snapshot (S3 / OSS)
239
+
240
+ On remote backends (`S3Adapter`, `OSSAdapter`, custom adapters with `remote: true`), each collection keeps `<dbPath>/<collection>/_snapshot.v1.json`: a copy of every document with its etag. A `find`:
241
+
242
+ 1. lists `docs/` (one request per 1,000 documents), which says which documents exist and their current etags;
243
+ 2. GETs the snapshot and uses an entry only if its etag equals the listed one;
244
+ 3. GETs every other listed document, 16 in parallel;
245
+ 4. rewrites the snapshot (one unconditional PUT) when more than 16 documents had to be read individually, counting snapshot entries of deleted documents too.
246
+
247
+ The list is the authority. A missing, corrupt, stale or concurrently overwritten snapshot changes only how many requests a query makes, never its result; a missing or unparsable one is rebuilt. A transient or permission error while *reading* the snapshot fails the query with `StorageError`. A failed snapshot *write* does not fail the query; it is reported to the logger, and after a 401 / 403 (for example read-only credentials) the collection stops trying to write it.
248
+
249
+ Deleted documents' contents stay in the snapshot until it is rewritten. `compact()` rebuilds it from the current documents right away:
250
+
251
+ ```typescript
252
+ await users.delete(id);
253
+ await users.compact(); // S3 / OSS: LIST + GET every document + 1 PUT; throws on failure. No-op on Memory / FileSystem.
254
+ ```
255
+
256
+ Memory and FileSystem keep no snapshot; reading local files is cheap. Bucket lifecycle rules that expire objects under `docs/` act as a TTL: expired documents disappear from LIST and from results.
257
+
258
+ ### Logger
259
+
260
+ Problems that do not fail the current call go to a logger, `console.warn` by default: a snapshot that could not be written (level `error`), a snapshot that could not be parsed and is rebuilt (`warn`), and a collection past the 10,000-document / 10 MB envelope (`warn`, once per collection).
12
261
 
13
262
  ```typescript
14
- const db = new BucketDB(adapter, dbPath, options);
263
+ import type { LogEvent } from '@hold-baby/bucket-db';
264
+
265
+ const db = new BucketDB(adapter, 'my-app', {
266
+ logger: (event: LogEvent) => myLogger[event.level](event.message, event.error),
267
+ });
268
+ ```
269
+
270
+ ## Query Operators
271
+
272
+ ```typescript
273
+ // Equality (shorthand)
274
+ await users.find({ status: 'active' }); // { status: { $eq: 'active' } }
275
+
276
+ // Comparison
277
+ await users.find({ age: { $gt: 18 } });
278
+ await users.find({ age: { $gte: 18 } });
279
+ await users.find({ age: { $lt: 65 } });
280
+ await users.find({ age: { $lte: 65 } });
281
+ await users.find({ status: { $ne: 'inactive' } });
282
+
283
+ // Array membership
284
+ await users.find({ status: { $in: ['active', 'pending'] } });
285
+ await users.find({ status: { $nin: ['banned', 'deleted'] } });
286
+
287
+ // Sorting and pagination
288
+ await users.find({ status: 'active' }, { sort: { age: -1, name: 1 }, limit: 20, offset: 40 });
15
289
  ```
16
290
 
17
- ### Collection
291
+ Conditions on several fields are combined with AND. A field missing from a document evaluates as `undefined`:
18
292
 
19
- Type-safe collection interface.
293
+ | Condition | Field missing | Value of another type (`'30'` vs `30`) |
294
+ |---|---|---|
295
+ | plain value, `$eq` | no match | no match |
296
+ | `$ne` | match | match |
297
+ | `$gt`, `$gte`, `$lt`, `$lte` | no match | no match |
298
+ | `$in` | no match | no match |
299
+ | `$nin` | match | match |
300
+
301
+ Range operators compare only numbers with numbers and strings with strings; anything else is no match, never an error. `$in: []` matches nothing and `$nin: []` excludes nothing. A malformed filter — an unknown operator, a non-array `$in` / `$nin` operand, or a reserved field name such as `constructor` or `__proto__` — throws `ValidationError` before any request, as do an invalid `sort` and a negative or fractional `limit` / `offset`.
302
+
303
+ ## Error Handling
20
304
 
21
305
  ```typescript
22
- const collection = db.collection<T>(name);
306
+ import {
307
+ DocumentNotFoundError,
308
+ ConcurrentUpdateError,
309
+ DocumentExistsError,
310
+ UnsupportedError,
311
+ ValidationError,
312
+ StorageError,
313
+ } from '@hold-baby/bucket-db';
314
+
315
+ // findById returns null only when the document does not exist.
316
+ // Any other storage failure throws StorageError.
317
+ const user = await users.findById('nonexistent-id');
318
+ if (!user) {
319
+ // Document does not exist
320
+ }
23
321
 
24
- await collection.insert(data);
25
- await collection.findById(id);
26
- await collection.find(filter, options);
27
- await collection.update(id, data, options);
28
- await collection.delete(id);
322
+ try {
323
+ await users.update(id, { age: 30 }, { etag: staleEtag });
324
+ } catch (err) {
325
+ if (err instanceof ConcurrentUpdateError) {
326
+ // Stale etag, or another writer won the race — re-read and retry
327
+ } else if (err instanceof DocumentNotFoundError) {
328
+ // Document does not exist
329
+ } else if (err instanceof UnsupportedError) {
330
+ // The backend cannot enforce an etag (OSS)
331
+ } else if (err instanceof StorageError) {
332
+ console.error(err.kind, err.key, err.status);
333
+ }
334
+ }
29
335
  ```
30
336
 
31
- ### Storage Adapters
337
+ | Error class | When thrown |
338
+ |---|---|
339
+ | `ValidationError` | Invalid collection name, document ID, `dbPath`, filter, `sort`, `limit` or `offset`; invalid or missing connection URL; invalid adapter options; missing credentials |
340
+ | `DocumentNotFoundError` | `update` on a missing document |
341
+ | `DocumentExistsError` | `insert` with an `id` that is already taken |
342
+ | `ConcurrentUpdateError` | `update` with a stale `etag`, or a write that lost a race (with `etag`, or after 5 retries without) |
343
+ | `UnsupportedError` | The backend lacks the needed capability: `update` with `etag` on OSS |
344
+ | `StorageError` | Storage failure. `kind` is `'not-found'`, `'conflict'`, `'transient'` (still failing after retries) or `'permanent'`; `key` is the object key, `status` the HTTP status |
345
+
346
+ All of them extend `BucketDBError`.
347
+
348
+ A file under `docs/` that is not a JSON object, or whose name is not `<id>.json`, makes `find` throw `StorageError` with that file's `key`. Files whose name starts with `.` (for example `.DS_Store`) are ignored.
349
+
350
+ ### Collection names, document IDs and `dbPath`
351
+
352
+ Collection names, document IDs and every `/`-separated segment of `dbPath` must be 1–255 letters, digits, `_`, `-` or `.`, and must not start with `.`. Anything else throws `ValidationError`; names are never rewritten.
353
+
354
+ - `new BucketDB(adapter, dbPath)` and `db.collection(name)` throw synchronously.
355
+ - `findById` / `update` / `delete` reject with `ValidationError` for an invalid ID.
356
+ - `dbPath` may contain `/` (`'apps/tenant-1'`); `''` stores collections at the bucket root.
357
+
358
+ ## Storage Adapters
359
+
360
+ ### MemoryStorageAdapter
361
+
362
+ In-process, nothing persisted. For tests.
363
+
364
+ ```typescript
365
+ import { BucketDB, MemoryStorageAdapter } from '@hold-baby/bucket-db';
366
+
367
+ const db = new BucketDB(new MemoryStorageAdapter(), 'test-db');
368
+ ```
369
+
370
+ ### FileSystemAdapter
371
+
372
+ ```typescript
373
+ import { FileSystemAdapter } from '@hold-baby/bucket-db';
374
+
375
+ const adapter = new FileSystemAdapter({ basePath: './data' });
376
+ ```
377
+
378
+ Each object is a file under `basePath`. Writes go to a temporary file that is then linked (create) or renamed (replace) into place, so a reader never sees a partial file. Create-if-absent works across processes; `etag` checks work within one process.
379
+
380
+ ### S3Adapter
381
+
382
+ ```typescript
383
+ import { S3Adapter } from '@hold-baby/bucket-db';
384
+
385
+ // Region, endpoint and credentials from AWS_* environment variables
386
+ const adapter = new S3Adapter({ bucket: 'my-bucket' });
387
+
388
+ // Or explicitly
389
+ const explicit = new S3Adapter({
390
+ bucket: 'my-bucket',
391
+ region: 'us-east-1',
392
+ credentials: {
393
+ accessKeyId: process.env.MY_KEY_ID!,
394
+ secretAccessKey: process.env.MY_SECRET!,
395
+ },
396
+ });
397
+ ```
398
+
399
+ | Option | Default |
400
+ |---|---|
401
+ | `bucket` | required |
402
+ | `region` | `AWS_REGION`, then `AWS_DEFAULT_REGION` |
403
+ | `endpoint` | `AWS_ENDPOINT_URL_S3`, then `AWS_ENDPOINT_URL`, then `https://s3.<region>.amazonaws.com` |
404
+ | `credentials` | `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_SESSION_TOKEN` |
405
+ | `forcePathStyle` | `true` when the endpoint host is an IP address or `localhost` |
406
+ | `timeoutMs` | `10000` (per attempt) |
407
+
408
+ Any S3-compatible service can be used through `endpoint` (MinIO, Cloudflare R2, …), but only AWS S3 and Alibaba Cloud OSS are tested.
409
+
410
+ ### OSSAdapter
411
+
412
+ ```typescript
413
+ import { OSSAdapter } from '@hold-baby/bucket-db';
414
+
415
+ // Region and credentials from OSS_* environment variables
416
+ const adapter = new OSSAdapter({ bucket: 'my-bucket' });
417
+
418
+ // Or explicitly
419
+ const explicit = new OSSAdapter({
420
+ bucket: 'my-bucket',
421
+ region: 'cn-hangzhou',
422
+ credentials: {
423
+ accessKeyId: process.env.OSS_ACCESS_KEY_ID!,
424
+ secretAccessKey: process.env.OSS_ACCESS_KEY_SECRET!,
425
+ },
426
+ });
427
+ ```
428
+
429
+ | Option | Default |
430
+ |---|---|
431
+ | `bucket` | required |
432
+ | `region` | `OSS_REGION`. Write `cn-hangzhou`; the 0.3 form `oss-cn-hangzhou` is also accepted |
433
+ | `endpoint` | `https://s3.oss-<region>.aliyuncs.com` (OSS's S3-compatible endpoint). `OSS_ENDPOINT` is not read |
434
+ | `credentials` | `OSS_ACCESS_KEY_ID`, `OSS_ACCESS_KEY_SECRET`, `OSS_SESSION_TOKEN`; if unset, `ALIBABA_CLOUD_ACCESS_KEY_ID`, `ALIBABA_CLOUD_ACCESS_KEY_SECRET`, `ALIBABA_CLOUD_SECURITY_TOKEN` (what `aliyun/configure-aliyun-credentials-action` exports after OIDC) |
435
+ | `timeoutMs` | `10000` (per attempt) |
436
+
437
+ New OSS users in mainland China may have to access buckets through a custom domain, passed as `endpoint`. S3-compatible access through a custom domain has not been verified.
438
+
439
+ OSS does not support conditional updates: `update` with `etag` throws `UnsupportedError`. `insert` relies on `x-oss-forbid-overwrite`, which OSS ignores on versioned buckets.
440
+
441
+ ### Credentials
442
+
443
+ `credentials` is either an object or an async function:
444
+
445
+ ```typescript
446
+ interface Credentials {
447
+ accessKeyId: string;
448
+ secretAccessKey: string;
449
+ sessionToken?: string;
450
+ expiresAt?: Date;
451
+ }
452
+ type CredentialProvider = () => Promise<Credentials>;
453
+
454
+ const adapter = new S3Adapter({
455
+ bucket: 'my-bucket',
456
+ region: 'us-east-1',
457
+ credentials: async () => {
458
+ const sts = await fetchTemporaryCredentials(); // your code
459
+ return {
460
+ accessKeyId: sts.accessKeyId,
461
+ secretAccessKey: sts.secretAccessKey,
462
+ sessionToken: sts.sessionToken,
463
+ expiresAt: new Date(sts.expiration),
464
+ };
465
+ },
466
+ });
467
+ ```
468
+
469
+ The function's result is cached; it is called again 5 minutes before `expiresAt` (never, if `expiresAt` is absent). A failed call is not cached. Without the option, missing environment variables throw `ValidationError` on the first request.
470
+
471
+ ### Retries
472
+
473
+ `S3Adapter` and `OSSAdapter` make up to 3 attempts per request, with jittered exponential backoff, on network errors, timeouts, 5xx, 429, `SlowDown` and `RequestTimeout`. Other 4xx responses are not retried. A failure that persists becomes `StorageError` with `kind: 'transient'`. If a conditional write hits a conflict after a transient failure, the adapter reads the object back and compares it with what it wrote, so an earlier attempt that did succeed is not reported as a conflict.
474
+
475
+ ## BucketDB
476
+
477
+ ```typescript
478
+ new BucketDB(adapter: StorageAdapter, dbPath: string, options?: BucketDBOptions)
479
+ BucketDB.connect(url?: string, options?: BucketDBOptions): BucketDB // see Connection URLs; default BUCKET_DB_URL
480
+ db.collection<T extends Document>(name: string): Collection<T>
481
+
482
+ interface BucketDBOptions {
483
+ logger?: Logger; // default: console.warn
484
+ }
485
+ type Logger = (event: LogEvent) => void;
486
+ interface LogEvent {
487
+ level: 'warn' | 'error';
488
+ message: string;
489
+ error?: unknown;
490
+ }
491
+ ```
492
+
493
+ | Parameter | Description |
494
+ |---|---|
495
+ | `adapter` | Storage backend |
496
+ | `dbPath` | Key prefix for this database (e.g. `'my-app'`, `'tenants/123'`); `''` for the bucket root |
497
+ | `options.logger` | Receives snapshot write failures and envelope warnings; see [Logger](#logger) |
498
+ | `name` | Collection name; the same name returns the same `Collection` instance |
499
+
500
+ A custom `StorageAdapter` can set `readonly remote = true` to get the query snapshot; the built-in `S3Adapter` and `OSSAdapter` do.
501
+
502
+ ## Upgrading
503
+
504
+ **From 0.4.** The storage format is unchanged. Queries changed: `$ne` / `$nin` now match documents without the field, range comparisons between mismatched types no longer throw, `find` without `sort` orders by `_createdAt` then `id`, and `limit: 0` returns `[]`. The MCP server is configured by `BUCKET_DB_URL`. Details: [MIGRATION-0.5.md](https://github.com/holdbaby5483/bucket-db/blob/main/docs/MIGRATION-0.5.md).
505
+
506
+ **From 0.3.x.**
507
+
508
+ 0.4 reads 0.3 collections as they are: `docs/<id>.json` is unchanged. Besides `docs/`, 0.4 reads only its own `_snapshot.v1.json` (S3 / OSS), so the index and log directories 0.3 kept next to `docs/`, and 0.3's `*.meta` files, are ignored and can be deleted. On S3 / OSS, queries create a `_snapshot.v1.json` next to `docs/` in the collections they read (once more than 16 documents had to be fetched); with read-only credentials it is simply not written. Stop every 0.3 writer before 0.4 starts writing. The API changes are listed in the [migration guide](https://github.com/holdbaby5483/bucket-db/blob/main/docs/MIGRATION-0.4.md) and the [changelog](https://github.com/holdbaby5483/bucket-db/blob/main/CHANGELOG.md).
509
+
510
+ ## Documentation
511
+
512
+ 📚 https://bucket-db.vercel.app
32
513
 
33
- - `MemoryStorageAdapter` - In-memory storage for testing
34
- - `S3Adapter` - AWS S3 storage
35
- - `OSSAdapter` - Alibaba Cloud OSS storage
514
+ - [API reference](https://github.com/holdbaby5483/bucket-db/blob/main/docs/API.md)
515
+ - [CLI reference](https://github.com/holdbaby5483/bucket-db/blob/main/docs/CLI.md) and [CI recipes](https://github.com/holdbaby5483/bucket-db/tree/main/examples/ci)
516
+ - [Guarantees and limits](https://github.com/holdbaby5483/bucket-db/blob/main/docs/GUARANTEES.md): what each backend guarantees and the test that checks it
517
+ - [Storage format](https://github.com/holdbaby5483/bucket-db/blob/main/docs/STORAGE-FORMAT.md): the keys and JSON BucketDB writes
518
+ - [Benchmarks](https://github.com/holdbaby5483/bucket-db/blob/main/docs/BENCHMARKS.md): requests and latency at 100 to 10,000 documents on OSS, S3 and FileSystem
519
+ - [Migrating from 0.4](https://github.com/holdbaby5483/bucket-db/blob/main/docs/MIGRATION-0.5.md) and [from 0.3](https://github.com/holdbaby5483/bucket-db/blob/main/docs/MIGRATION-0.4.md)
36
520
 
37
521
  ## License
38
522
 
@@ -0,0 +1,25 @@
1
+ import type { Document, Logger } from './types.js';
2
+ import type { StorageAdapter } from './storage/adapter.js';
3
+ import { Collection } from './collection.js';
4
+ export interface BucketDBOptions {
5
+ /** Receives snapshot write failures and envelope warnings. Default: console.warn. */
6
+ logger?: Logger;
7
+ }
8
+ /**
9
+ * Entry point: a storage backend plus a key prefix (`dbPath`, `''` for the bucket root).
10
+ */
11
+ export declare class BucketDB {
12
+ private readonly adapter;
13
+ private readonly dbPath;
14
+ private readonly options;
15
+ private readonly collections;
16
+ constructor(adapter: StorageAdapter, dbPath: string, options?: BucketDBOptions);
17
+ /**
18
+ * Opens a database from a connection URL, by default `BUCKET_DB_URL`:
19
+ * `s3://bucket/prefix?region=…`, `oss://bucket/prefix?region=…`,
20
+ * `file:///dir` or `memory://`. Credentials come from the environment, never the URL.
21
+ */
22
+ static connect(url?: string, options?: BucketDBOptions): BucketDB;
23
+ /** Returns the same Collection instance for the same name. */
24
+ collection<T extends Document>(name: string): Collection<T>;
25
+ }
@@ -0,0 +1,36 @@
1
+ import { Collection } from './collection.js';
2
+ import { validateDbPath } from './keys.js';
3
+ import { openUrl } from './url.js';
4
+ /**
5
+ * Entry point: a storage backend plus a key prefix (`dbPath`, `''` for the bucket root).
6
+ */
7
+ export class BucketDB {
8
+ adapter;
9
+ dbPath;
10
+ options;
11
+ collections = new Map();
12
+ constructor(adapter, dbPath, options = {}) {
13
+ this.adapter = adapter;
14
+ this.dbPath = dbPath;
15
+ this.options = options;
16
+ validateDbPath(dbPath);
17
+ }
18
+ /**
19
+ * Opens a database from a connection URL, by default `BUCKET_DB_URL`:
20
+ * `s3://bucket/prefix?region=…`, `oss://bucket/prefix?region=…`,
21
+ * `file:///dir` or `memory://`. Credentials come from the environment, never the URL.
22
+ */
23
+ static connect(url, options = {}) {
24
+ const { adapter, dbPath } = openUrl(url);
25
+ return new BucketDB(adapter, dbPath, options);
26
+ }
27
+ /** Returns the same Collection instance for the same name. */
28
+ collection(name) {
29
+ let collection = this.collections.get(name);
30
+ if (!collection) {
31
+ collection = new Collection(this.adapter, this.dbPath, name, { logger: this.options.logger });
32
+ this.collections.set(name, collection);
33
+ }
34
+ return collection;
35
+ }
36
+ }
package/dist/cli.d.ts ADDED
@@ -0,0 +1,2 @@
1
+ #!/usr/bin/env node
2
+ export {};