@hold-baby/bucket-db 0.3.1 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (114) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +331 -62
  3. package/dist/bucket-db.d.ts +25 -0
  4. package/dist/bucket-db.js +36 -0
  5. package/dist/cli.d.ts +2 -0
  6. package/dist/cli.js +133 -0
  7. package/dist/collection.d.ts +78 -0
  8. package/dist/collection.js +321 -0
  9. package/dist/doctor.d.ts +7 -0
  10. package/dist/doctor.js +100 -0
  11. package/dist/errors.d.ts +59 -0
  12. package/dist/errors.js +57 -0
  13. package/dist/index.d.ts +13 -9
  14. package/dist/index.js +6 -10
  15. package/dist/keys.d.ts +12 -0
  16. package/dist/keys.js +31 -0
  17. package/dist/query.d.ts +16 -0
  18. package/dist/query.js +133 -0
  19. package/dist/snapshot.d.ts +16 -0
  20. package/dist/snapshot.js +40 -0
  21. package/dist/storage/adapter.d.ts +75 -0
  22. package/dist/storage/adapter.js +27 -0
  23. package/dist/storage/fs.d.ts +28 -0
  24. package/dist/storage/fs.js +175 -0
  25. package/dist/storage/memory.d.ts +18 -0
  26. package/dist/storage/memory.js +49 -0
  27. package/dist/storage/s3/adapters.d.ts +38 -0
  28. package/dist/storage/s3/adapters.js +65 -0
  29. package/dist/storage/s3/client.d.ts +52 -0
  30. package/dist/storage/s3/client.js +231 -0
  31. package/dist/storage/s3/credentials.d.ts +17 -0
  32. package/dist/storage/s3/credentials.js +42 -0
  33. package/dist/storage/s3/sigv4.d.ts +32 -0
  34. package/dist/storage/s3/sigv4.js +75 -0
  35. package/dist/types.d.ts +55 -0
  36. package/dist/types.js +1 -0
  37. package/dist/url.d.ts +16 -0
  38. package/dist/url.js +66 -0
  39. package/llms.txt +116 -20
  40. package/package.json +7 -5
  41. package/dist/cache/lru-cache.d.ts +0 -47
  42. package/dist/cache/lru-cache.d.ts.map +0 -1
  43. package/dist/cache/lru-cache.js +0 -110
  44. package/dist/cache/lru-cache.js.map +0 -1
  45. package/dist/core/bucket-db.d.ts +0 -20
  46. package/dist/core/bucket-db.d.ts.map +0 -1
  47. package/dist/core/bucket-db.js +0 -32
  48. package/dist/core/bucket-db.js.map +0 -1
  49. package/dist/core/collection.d.ts +0 -26
  50. package/dist/core/collection.d.ts.map +0 -1
  51. package/dist/core/collection.js +0 -213
  52. package/dist/core/collection.js.map +0 -1
  53. package/dist/core/wal.d.ts +0 -45
  54. package/dist/core/wal.d.ts.map +0 -1
  55. package/dist/core/wal.js +0 -80
  56. package/dist/core/wal.js.map +0 -1
  57. package/dist/index/shard-manager.d.ts +0 -36
  58. package/dist/index/shard-manager.d.ts.map +0 -1
  59. package/dist/index/shard-manager.js +0 -177
  60. package/dist/index/shard-manager.js.map +0 -1
  61. package/dist/index.d.ts.map +0 -1
  62. package/dist/index.js.map +0 -1
  63. package/dist/query/evaluator.d.ts +0 -10
  64. package/dist/query/evaluator.d.ts.map +0 -1
  65. package/dist/query/evaluator.js +0 -127
  66. package/dist/query/evaluator.js.map +0 -1
  67. package/dist/storage/fs-adapter.d.ts +0 -24
  68. package/dist/storage/fs-adapter.d.ts.map +0 -1
  69. package/dist/storage/fs-adapter.js +0 -208
  70. package/dist/storage/fs-adapter.js.map +0 -1
  71. package/dist/storage/memory-adapter.d.ts +0 -22
  72. package/dist/storage/memory-adapter.d.ts.map +0 -1
  73. package/dist/storage/memory-adapter.js +0 -78
  74. package/dist/storage/memory-adapter.js.map +0 -1
  75. package/dist/storage/oss-adapter.d.ts +0 -26
  76. package/dist/storage/oss-adapter.d.ts.map +0 -1
  77. package/dist/storage/oss-adapter.js +0 -145
  78. package/dist/storage/oss-adapter.js.map +0 -1
  79. package/dist/storage/s3-adapter.d.ts +0 -26
  80. package/dist/storage/s3-adapter.d.ts.map +0 -1
  81. package/dist/storage/s3-adapter.js +0 -170
  82. package/dist/storage/s3-adapter.js.map +0 -1
  83. package/dist/types/collection.d.ts +0 -40
  84. package/dist/types/collection.d.ts.map +0 -1
  85. package/dist/types/collection.js +0 -2
  86. package/dist/types/collection.js.map +0 -1
  87. package/dist/types/document.d.ts +0 -18
  88. package/dist/types/document.d.ts.map +0 -1
  89. package/dist/types/document.js +0 -2
  90. package/dist/types/document.js.map +0 -1
  91. package/dist/types/errors.d.ts +0 -32
  92. package/dist/types/errors.d.ts.map +0 -1
  93. package/dist/types/errors.js +0 -48
  94. package/dist/types/errors.js.map +0 -1
  95. package/dist/types/index.d.ts +0 -6
  96. package/dist/types/index.d.ts.map +0 -1
  97. package/dist/types/index.js +0 -3
  98. package/dist/types/index.js.map +0 -1
  99. package/dist/types/query.d.ts +0 -37
  100. package/dist/types/query.d.ts.map +0 -1
  101. package/dist/types/query.js +0 -2
  102. package/dist/types/query.js.map +0 -1
  103. package/dist/types/storage.d.ts +0 -45
  104. package/dist/types/storage.d.ts.map +0 -1
  105. package/dist/types/storage.js +0 -2
  106. package/dist/types/storage.js.map +0 -1
  107. package/dist/utils/hash.d.ts +0 -13
  108. package/dist/utils/hash.d.ts.map +0 -1
  109. package/dist/utils/hash.js +0 -23
  110. package/dist/utils/hash.js.map +0 -1
  111. package/dist/utils/path-validator.d.ts +0 -46
  112. package/dist/utils/path-validator.d.ts.map +0 -1
  113. package/dist/utils/path-validator.js +0 -99
  114. package/dist/utils/path-validator.js.map +0 -1
package/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 holdbaby5483
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
package/README.md CHANGED
@@ -6,6 +6,34 @@
6
6
 
7
7
  A TypeScript document database built on cloud object storage (AWS S3 / Alibaba Cloud OSS) or local file system. No database servers needed.
8
8
 
9
+ Each document is one JSON object in your bucket: `<dbPath>/<collection>/docs/<id>.json`. There is no index and no log; every write is a single storage request. On S3 and OSS, queries also keep a disposable snapshot, `<dbPath>/<collection>/_snapshot.v1.json`, so a query does not download every document ([storage format](https://github.com/holdbaby5483/bucket-db/blob/main/docs/STORAGE-FORMAT.md)).
10
+
11
+ > **0.5.0** adds connection URLs (`BucketDB.connect`), the `bucket-db` CLI with `doctor`, `put`, `insert` with a chosen id, `sort`, `count` and `deleteMany`, and settles query semantics. Upgrading from 0.4? Read [MIGRATION-0.5.md](https://github.com/holdbaby5483/bucket-db/blob/main/docs/MIGRATION-0.5.md); from 0.3.x, [MIGRATION-0.4.md](https://github.com/holdbaby5483/bucket-db/blob/main/docs/MIGRATION-0.4.md) first.
12
+
13
+ ## Status and limitations
14
+
15
+ What each backend guarantees:
16
+
17
+ | | Memory | FileSystem | S3 | OSS |
18
+ |---|---|---|---|---|
19
+ | Single write is atomic | yes | yes | yes | yes |
20
+ | Create-if-absent (`insert`) | yes | yes (across processes) | yes | yes (non-versioned buckets only) |
21
+ | Conditional update (`etag`) | yes | single process only | yes | no — `UnsupportedError` |
22
+ | Concurrent partial updates all apply | yes | single process | yes | no (last writer wins) |
23
+
24
+ Multiple writer processes are supported for `insert` and `delete` on every backend, and for `update` on S3 (on Memory and FileSystem, within one process). In a test, 4 processes each inserting 25 documents left all 100 visible on FileSystem, S3 and OSS. [Guarantees and limits](https://github.com/holdbaby5483/bucket-db/blob/main/docs/GUARANTEES.md) names the test behind each row.
25
+
26
+ Remaining limits:
27
+
28
+ - **No transactions.** Each write touches one document. `deleteMany` deletes its matches one by one and is not atomic.
29
+ - **Envelope: at most 10,000 documents and 10 MB of snapshot per collection.** Past either, queries keep working but get slower, and the collection logs one warning. At 10,000 documents a `find` takes about 1.35 s on OSS and 3.2 s on S3, and about 0.5 s on a local FileSystem ([benchmarks](https://github.com/holdbaby5483/bucket-db/blob/main/docs/BENCHMARKS.md)). BucketDB is meant for small collections and a few writes per second.
30
+ - **Queries list the whole collection.** `find` lists every document key, then filters, sorts and pages in memory; `sort`, `limit` and `offset` do not reduce reads. On S3 / OSS document bodies come from the snapshot, plus a GET for each document changed since it was written; Memory and FileSystem read every file. Only a `{ id: '...' }` filter reads a single document, and only `count()` without a filter reads no documents.
31
+ - **Results are not a point-in-time snapshot.** Every returned document is a version that was committed, but documents written while a query runs may or may not be included.
32
+ - **The snapshot holds document contents.** A deleted document's contents stay in `_snapshot.v1.json` until the snapshot is rewritten or `compact()` runs. Call `compact()` after deleting privacy-sensitive data.
33
+ - **OSS has no conditional update.** `update` with `etag` throws `UnsupportedError`; concurrent `update`s without `etag` are last-writer-wins.
34
+ - **FileSystem conditional updates are single-process.** `etag` checks and the merging of concurrent `update`s hold only among writers in one process.
35
+ - **0.3.x and 0.4 or later must not write the same collection at the same time.** 0.3 reads only its own index and does not see later writes.
36
+
9
37
  ## Installation
10
38
 
11
39
  ```bash
@@ -14,6 +42,43 @@ bun add @hold-baby/bucket-db
14
42
  npm install @hold-baby/bucket-db
15
43
  ```
16
44
 
45
+ Requires Node.js >= 20 or Bun. No runtime dependencies: the S3 / OSS client is built in.
46
+
47
+ ## Command line
48
+
49
+ The package includes `bucket-db`, for shells and CI jobs:
50
+
51
+ ```bash
52
+ export BUCKET_DB_URL='s3://my-bucket/ci-data?region=us-east-1' # credentials from AWS_* variables
53
+
54
+ npx @hold-baby/bucket-db doctor # check what the bucket guarantees
55
+ echo '{"ms":1234,"ok":true}' | npx @hold-baby/bucket-db put runs run-42 # create or replace
56
+ npx @hold-baby/bucket-db get runs run-42 | jq .ms
57
+ npx @hold-baby/bucket-db find runs '{"ok":true}' --sort ms:desc --limit 5
58
+ npx @hold-baby/bucket-db count runs
59
+ npx @hold-baby/bucket-db export runs > runs.jsonl # backup; `import` restores
60
+ ```
61
+
62
+ Commands: `get`, `put`, `find` (`--sort field[:desc]`, `--limit`, `--offset`), `count`, `delete`, `export`, `import`, `compact`, `doctor`. stdout is one JSON value per line, errors go to stderr; exit code 0 on success, 1 on error, 2 when `get` finds nothing. Full reference: [CLI.md](https://github.com/holdbaby5483/bucket-db/blob/main/docs/CLI.md). Recipes for keeping CI data in a bucket with OIDC credentials: [examples/ci](https://github.com/holdbaby5483/bucket-db/tree/main/examples/ci).
63
+
64
+ ## Connection URLs
65
+
66
+ `BucketDB.connect(url?)`, the CLI and the MCP server open a database from one URL, by default the `BUCKET_DB_URL` environment variable:
67
+
68
+ | URL | Backend | `dbPath` |
69
+ |---|---|---|
70
+ | `s3://bucket/prefix?region=…&endpoint=…&forcePathStyle=true&timeoutMs=…` | `S3Adapter` | `prefix` |
71
+ | `oss://bucket/prefix?region=…&endpoint=…&timeoutMs=…` | `OSSAdapter` | `prefix` |
72
+ | `file:///absolute/dir`, `file://./relative/dir` | `FileSystemAdapter` | `''` |
73
+ | `memory://prefix` | `MemoryStorageAdapter` | `prefix` |
74
+
75
+ ```typescript
76
+ const db = BucketDB.connect('s3://my-bucket/my-app?region=us-east-1');
77
+ const fromEnv = BucketDB.connect(); // BUCKET_DB_URL
78
+ ```
79
+
80
+ Every query parameter is optional and falls back to the adapter's environment default; unknown parameters throw `ValidationError`. Credentials never go in the URL (a URL with `user:secret@` is refused) but come from `AWS_*`, `OSS_*` or `ALIBABA_CLOUD_*` variables (see [Storage Adapters](#storage-adapters)). Error messages never repeat the URL, so it is safe to log. `new BucketDB(adapter, dbPath)` remains available, for example to pass a credential provider.
81
+
17
82
  ## Exports
18
83
 
19
84
  ```typescript
@@ -22,29 +87,52 @@ import { BucketDB, Collection } from '@hold-baby/bucket-db';
22
87
 
23
88
  // Storage adapters
24
89
  import {
25
- MemoryStorageAdapter, // in-memory (testing)
26
- FileSystemAdapter, // local file system (development)
27
- S3Adapter, // AWS S3 (production)
28
- OSSAdapter, // Alibaba Cloud OSS (production)
90
+ MemoryStorageAdapter, // in-memory (tests)
91
+ FileSystemAdapter, // local file system
92
+ S3Adapter, // AWS S3 and S3-compatible services
93
+ OSSAdapter, // Alibaba Cloud OSS
29
94
  } from '@hold-baby/bucket-db';
30
95
 
31
96
  // Types
32
97
  import type {
98
+ BucketDBOptions,
99
+ CollectionOptions,
100
+ Logger,
101
+ LogEvent,
33
102
  Document,
34
103
  InsertDocument,
35
104
  UpdateDocument,
105
+ QueryOperator,
106
+ QueryValue,
36
107
  QueryFilter,
37
108
  QueryOptions,
109
+ InsertOptions,
38
110
  UpdateOptions,
111
+ FileSystemAdapterConfig,
112
+ S3AdapterOptions,
113
+ OSSAdapterOptions,
114
+ Credentials,
115
+ CredentialProvider,
116
+ // for custom adapters
39
117
  StorageAdapter,
118
+ Capabilities,
119
+ Guarantee,
120
+ WriteCondition,
121
+ ObjectEntry,
122
+ ListPage,
123
+ ListOptions,
124
+ StorageErrorKind,
125
+ StorageErrorOptions,
40
126
  } from '@hold-baby/bucket-db';
41
127
 
42
128
  // Error classes
43
129
  import {
44
130
  BucketDBError,
131
+ ValidationError,
45
132
  DocumentNotFoundError,
133
+ DocumentExistsError,
46
134
  ConcurrentUpdateError,
47
- ValidationError,
135
+ UnsupportedError,
48
136
  StorageError,
49
137
  } from '@hold-baby/bucket-db';
50
138
  ```
@@ -55,8 +143,8 @@ Every document stored in BucketDB has these system fields:
55
143
 
56
144
  ```typescript
57
145
  interface Document {
58
- id: string; // auto-generated UUID
59
- _etag?: string; // version tag for optimistic locking
146
+ id: string; // generated UUID unless you choose it (put, insert with { id }); taken from the object key
147
+ _etag?: string; // version tag from the storage service, for optimistic locking
60
148
  _createdAt?: string; // ISO 8601 timestamp
61
149
  _updatedAt?: string; // ISO 8601 timestamp
62
150
  }
@@ -69,14 +157,14 @@ interface User extends Document {
69
157
  }
70
158
  ```
71
159
 
72
- `InsertDocument<T>` omits `id`, `_etag`, `_createdAt`, `_updatedAt` — pass only your fields when inserting.
160
+ `InsertDocument<T>` and `UpdateDocument<T>` both omit `id`, `_etag`, `_createdAt` and `_updatedAt`; `UpdateDocument<T>` also makes every field optional. If system fields are passed at runtime anyway, `insert`, `put` and `update` drop them.
73
161
 
74
- `UpdateDocument<T>` is `Partial<Omit<T, 'id'>>` — all fields optional except `id` is excluded.
162
+ `_etag` is not stored in the document body: on every read, `id` comes from the object key and `_etag` from the storage service.
75
163
 
76
164
  ## Quick Start
77
165
 
78
166
  ```typescript
79
- import { BucketDB, FileSystemAdapter } from '@hold-baby/bucket-db';
167
+ import { BucketDB } from '@hold-baby/bucket-db';
80
168
  import type { Document } from '@hold-baby/bucket-db';
81
169
 
82
170
  interface User extends Document {
@@ -86,29 +174,97 @@ interface User extends Document {
86
174
  status: 'active' | 'inactive';
87
175
  }
88
176
 
89
- const db = new BucketDB(
90
- new FileSystemAdapter({ basePath: './data' }),
91
- 'my-app'
92
- );
177
+ // Or s3://bucket/prefix?region=…, oss://…, memory://; new BucketDB(adapter, dbPath) also works
178
+ const db = BucketDB.connect('file://./data');
93
179
 
94
180
  const users = db.collection<User>('users');
95
181
 
96
- // Insert
182
+ // Insert: one conditional PUT
97
183
  const user = await users.insert({ name: 'Alice', age: 25, email: 'alice@example.com', status: 'active' });
98
- // user.id, user._etag, user._createdAt are set automatically
184
+ // user.id, user._etag, user._createdAt, user._updatedAt are set automatically
99
185
 
100
- // Find by ID
186
+ // Find by ID: one GET; null if the document does not exist
101
187
  const found = await users.findById(user.id);
102
188
 
103
- // Query
189
+ // Query: filters in memory (on S3 / OSS, document bodies come from the query snapshot)
104
190
  const activeUsers = await users.find({ status: 'active' });
105
- const adults = await users.find({ age: { $gte: 18 } }, { limit: 10, offset: 0 });
191
+ const adults = await users.find({ age: { $gte: 18 } }, { sort: { age: -1 }, limit: 10 });
192
+ const total = await users.count();
106
193
 
107
- // Update (with optimistic locking via etag)
194
+ // Create or replace under an ID you choose: one unconditional PUT, safe to repeat
195
+ await users.put('alice', { name: 'Alice', age: 25, email: 'alice@example.com', status: 'active' });
196
+
197
+ // Create only if the ID is free: throws DocumentExistsError otherwise
198
+ await users.insert({ name: 'Bob', age: 31, email: 'bob@example.com', status: 'active' }, { id: 'bob' });
199
+
200
+ // Update with optimistic locking (not available on OSS, see above)
108
201
  const updated = await users.update(user.id, { age: 26 }, { etag: user._etag });
109
202
 
110
- // Delete
203
+ // Delete: one DELETE; no error if the document is already gone
111
204
  await users.delete(user.id);
205
+
206
+ // Delete every match, one DELETE each (not atomic); returns how many
207
+ await users.deleteMany({ status: 'inactive' });
208
+ ```
209
+
210
+ ## Operations and cost
211
+
212
+ | Operation | Requests |
213
+ |---|---|
214
+ | `insert(data, { id? })` | 1 conditional PUT |
215
+ | `put(id, data)` | 1 PUT |
216
+ | `findById(id)` | 1 GET |
217
+ | `find({ id: '...' })` | 1 GET |
218
+ | `find(filter)` on S3 / OSS | ⌈N / 1000⌉ LIST + 1 GET (snapshot), plus 1 GET per document changed since the snapshot (up to 16, in parallel) or, when more changed, those GETs and 1 PUT rewriting the snapshot |
219
+ | `find(filter)` on Memory / FileSystem | ⌈N / 1000⌉ LIST + N GET (16 in parallel) |
220
+ | `update(id, data, options?)` | 1 GET + 1 conditional PUT (more on conflict, see below) |
221
+ | `delete(id)` | 1 DELETE |
222
+ | `count()` | ⌈N / 1000⌉ LIST |
223
+ | `count(filter)` | same as `find(filter)` |
224
+ | `deleteMany(filter)` | same as `find(filter)`, plus 1 DELETE per match (16 in parallel) |
225
+ | `compact()` | S3 / OSS: ⌈N / 1000⌉ LIST + N GET + 1 PUT; nothing on Memory / FileSystem |
226
+
227
+ N = documents in the collection. Measured latencies at 100, 1,000 and 10,000 documents are in [BENCHMARKS.md](https://github.com/holdbaby5483/bucket-db/blob/main/docs/BENCHMARKS.md).
228
+
229
+ Without `sort`, `find` returns documents by `_createdAt`, then `id`: roughly creation order, and stable for paging. `sort: { field: 1 | -1 }` sorts by the given fields first; missing and `null` values sort last in either direction. `sort`, `offset` and `limit` apply after filtering, and `limit: 0` returns `[]`. There is no result cache: every `find` lists storage first.
230
+
231
+ `put` replaces the whole document and sets both timestamps to now (so it moves to the end of the default order); `update` merges fields and keeps `_createdAt`. `insert` with `{ id }` creates only if the ID is free; of several concurrent inserts with one ID, exactly one succeeds.
232
+
233
+ `update` merges `data` into the stored document:
234
+
235
+ - **With `etag`**: throws `ConcurrentUpdateError` if the document no longer has that etag, or if another writer changes it before the write lands. No retry.
236
+ - **Without `etag`**: on backends with compare-and-swap (Memory, FileSystem within one process, S3), a write that collides with another writer re-reads and retries (up to 5 retries), so concurrent partial updates all apply. On OSS the last writer wins.
237
+
238
+ ## Query snapshot (S3 / OSS)
239
+
240
+ On remote backends (`S3Adapter`, `OSSAdapter`, custom adapters with `remote: true`), each collection keeps `<dbPath>/<collection>/_snapshot.v1.json`: a copy of every document with its etag. A `find`:
241
+
242
+ 1. lists `docs/` (one request per 1,000 documents), which says which documents exist and their current etags;
243
+ 2. GETs the snapshot and uses an entry only if its etag equals the listed one;
244
+ 3. GETs every other listed document, 16 in parallel;
245
+ 4. rewrites the snapshot (one unconditional PUT) when more than 16 documents had to be read individually, counting snapshot entries of deleted documents too.
246
+
247
+ The list is the authority. A missing, corrupt, stale or concurrently overwritten snapshot changes only how many requests a query makes, never its result; a missing or unparsable one is rebuilt. A transient or permission error while *reading* the snapshot fails the query with `StorageError`. A failed snapshot *write* does not fail the query; it is reported to the logger, and after a 401 / 403 (for example read-only credentials) the collection stops trying to write it.
248
+
249
+ Deleted documents' contents stay in the snapshot until it is rewritten. `compact()` rebuilds it from the current documents right away:
250
+
251
+ ```typescript
252
+ await users.delete(id);
253
+ await users.compact(); // S3 / OSS: LIST + GET every document + 1 PUT; throws on failure. No-op on Memory / FileSystem.
254
+ ```
255
+
256
+ Memory and FileSystem keep no snapshot; reading local files is cheap. Bucket lifecycle rules that expire objects under `docs/` act as a TTL: expired documents disappear from LIST and from results.
257
+
258
+ ### Logger
259
+
260
+ Problems that do not fail the current call go to a logger, `console.warn` by default: a snapshot that could not be written (level `error`), a snapshot that could not be parsed and is rebuilt (`warn`), and a collection past the 10,000-document / 10 MB envelope (`warn`, once per collection).
261
+
262
+ ```typescript
263
+ import type { LogEvent } from '@hold-baby/bucket-db';
264
+
265
+ const db = new BucketDB(adapter, 'my-app', {
266
+ logger: (event: LogEvent) => myLogger[event.level](event.message, event.error),
267
+ });
112
268
  ```
113
269
 
114
270
  ## Query Operators
@@ -128,47 +284,82 @@ await users.find({ status: { $ne: 'inactive' } });
128
284
  await users.find({ status: { $in: ['active', 'pending'] } });
129
285
  await users.find({ status: { $nin: ['banned', 'deleted'] } });
130
286
 
131
- // Pagination
132
- await users.find({}, { limit: 20, offset: 40 });
287
+ // Sorting and pagination
288
+ await users.find({ status: 'active' }, { sort: { age: -1, name: 1 }, limit: 20, offset: 40 });
133
289
  ```
134
290
 
291
+ Conditions on several fields are combined with AND. A field missing from a document evaluates as `undefined`:
292
+
293
+ | Condition | Field missing | Value of another type (`'30'` vs `30`) |
294
+ |---|---|---|
295
+ | plain value, `$eq` | no match | no match |
296
+ | `$ne` | match | match |
297
+ | `$gt`, `$gte`, `$lt`, `$lte` | no match | no match |
298
+ | `$in` | no match | no match |
299
+ | `$nin` | match | match |
300
+
301
+ Range operators compare only numbers with numbers and strings with strings; anything else is no match, never an error. `$in: []` matches nothing and `$nin: []` excludes nothing. A malformed filter — an unknown operator, a non-array `$in` / `$nin` operand, or a reserved field name such as `constructor` or `__proto__` — throws `ValidationError` before any request, as do an invalid `sort` and a negative or fractional `limit` / `offset`.
302
+
135
303
  ## Error Handling
136
304
 
137
305
  ```typescript
138
306
  import {
139
307
  DocumentNotFoundError,
140
308
  ConcurrentUpdateError,
309
+ DocumentExistsError,
310
+ UnsupportedError,
141
311
  ValidationError,
142
312
  StorageError,
143
313
  } from '@hold-baby/bucket-db';
144
314
 
145
- try {
146
- const user = await users.findById('nonexistent-id');
147
- } catch (err) {
148
- if (err instanceof DocumentNotFoundError) {
149
- // Document does not exist
150
- }
315
+ // findById returns null only when the document does not exist.
316
+ // Any other storage failure throws StorageError.
317
+ const user = await users.findById('nonexistent-id');
318
+ if (!user) {
319
+ // Document does not exist
151
320
  }
152
321
 
153
322
  try {
154
323
  await users.update(id, { age: 30 }, { etag: staleEtag });
155
324
  } catch (err) {
156
325
  if (err instanceof ConcurrentUpdateError) {
157
- // Another write happened — re-fetch and retry
326
+ // Stale etag, or another writer won the race — re-read and retry
327
+ } else if (err instanceof DocumentNotFoundError) {
328
+ // Document does not exist
329
+ } else if (err instanceof UnsupportedError) {
330
+ // The backend cannot enforce an etag (OSS)
331
+ } else if (err instanceof StorageError) {
332
+ console.error(err.kind, err.key, err.status);
158
333
  }
159
334
  }
160
335
  ```
161
336
 
162
337
  | Error class | When thrown |
163
338
  |---|---|
164
- | `DocumentNotFoundError` | `findById` or `update`/`delete` on a missing document |
165
- | `ConcurrentUpdateError` | `update`/`delete` with a stale `etag` |
166
- | `ValidationError` | Invalid input (e.g. missing required fields) |
167
- | `StorageError` | Underlying storage failure (S3, OSS, filesystem I/O) |
339
+ | `ValidationError` | Invalid collection name, document ID, `dbPath`, filter, `sort`, `limit` or `offset`; invalid or missing connection URL; invalid adapter options; missing credentials |
340
+ | `DocumentNotFoundError` | `update` on a missing document |
341
+ | `DocumentExistsError` | `insert` with an `id` that is already taken |
342
+ | `ConcurrentUpdateError` | `update` with a stale `etag`, or a write that lost a race (with `etag`, or after 5 retries without) |
343
+ | `UnsupportedError` | The backend lacks the needed capability: `update` with `etag` on OSS |
344
+ | `StorageError` | Storage failure. `kind` is `'not-found'`, `'conflict'`, `'transient'` (still failing after retries) or `'permanent'`; `key` is the object key, `status` the HTTP status |
345
+
346
+ All of them extend `BucketDBError`.
347
+
348
+ A file under `docs/` that is not a JSON object, or whose name is not `<id>.json`, makes `find` throw `StorageError` with that file's `key`. Files whose name starts with `.` (for example `.DS_Store`) are ignored.
349
+
350
+ ### Collection names, document IDs and `dbPath`
351
+
352
+ Collection names, document IDs and every `/`-separated segment of `dbPath` must be 1–255 letters, digits, `_`, `-` or `.`, and must not start with `.`. Anything else throws `ValidationError`; names are never rewritten.
353
+
354
+ - `new BucketDB(adapter, dbPath)` and `db.collection(name)` throw synchronously.
355
+ - `findById` / `update` / `delete` reject with `ValidationError` for an invalid ID.
356
+ - `dbPath` may contain `/` (`'apps/tenant-1'`); `''` stores collections at the bucket root.
168
357
 
169
358
  ## Storage Adapters
170
359
 
171
- ### MemoryStorageAdapter (Testing)
360
+ ### MemoryStorageAdapter
361
+
362
+ In-process, nothing persisted. For tests.
172
363
 
173
364
  ```typescript
174
365
  import { BucketDB, MemoryStorageAdapter } from '@hold-baby/bucket-db';
@@ -176,7 +367,7 @@ import { BucketDB, MemoryStorageAdapter } from '@hold-baby/bucket-db';
176
367
  const db = new BucketDB(new MemoryStorageAdapter(), 'test-db');
177
368
  ```
178
369
 
179
- ### FileSystemAdapter (Development)
370
+ ### FileSystemAdapter
180
371
 
181
372
  ```typescript
182
373
  import { FileSystemAdapter } from '@hold-baby/bucket-db';
@@ -184,29 +375,50 @@ import { FileSystemAdapter } from '@hold-baby/bucket-db';
184
375
  const adapter = new FileSystemAdapter({ basePath: './data' });
185
376
  ```
186
377
 
187
- ### S3Adapter (Production)
378
+ Each object is a file under `basePath`. Writes go to a temporary file that is then linked (create) or renamed (replace) into place, so a reader never sees a partial file. Create-if-absent works across processes; `etag` checks work within one process.
379
+
380
+ ### S3Adapter
188
381
 
189
382
  ```typescript
190
383
  import { S3Adapter } from '@hold-baby/bucket-db';
191
384
 
192
- const adapter = new S3Adapter({
385
+ // Region, endpoint and credentials from AWS_* environment variables
386
+ const adapter = new S3Adapter({ bucket: 'my-bucket' });
387
+
388
+ // Or explicitly
389
+ const explicit = new S3Adapter({
193
390
  bucket: 'my-bucket',
194
391
  region: 'us-east-1',
195
392
  credentials: {
196
- accessKeyId: process.env.AWS_ACCESS_KEY_ID!,
197
- secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY!,
393
+ accessKeyId: process.env.MY_KEY_ID!,
394
+ secretAccessKey: process.env.MY_SECRET!,
198
395
  },
199
396
  });
200
397
  ```
201
398
 
202
- ### OSSAdapter (Production)
399
+ | Option | Default |
400
+ |---|---|
401
+ | `bucket` | required |
402
+ | `region` | `AWS_REGION`, then `AWS_DEFAULT_REGION` |
403
+ | `endpoint` | `AWS_ENDPOINT_URL_S3`, then `AWS_ENDPOINT_URL`, then `https://s3.<region>.amazonaws.com` |
404
+ | `credentials` | `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_SESSION_TOKEN` |
405
+ | `forcePathStyle` | `true` when the endpoint host is an IP address or `localhost` |
406
+ | `timeoutMs` | `10000` (per attempt) |
407
+
408
+ Any S3-compatible service can be used through `endpoint` (MinIO, Cloudflare R2, …), but only AWS S3 and Alibaba Cloud OSS are tested.
409
+
410
+ ### OSSAdapter
203
411
 
204
412
  ```typescript
205
413
  import { OSSAdapter } from '@hold-baby/bucket-db';
206
414
 
207
- const adapter = new OSSAdapter({
415
+ // Region and credentials from OSS_* environment variables
416
+ const adapter = new OSSAdapter({ bucket: 'my-bucket' });
417
+
418
+ // Or explicitly
419
+ const explicit = new OSSAdapter({
208
420
  bucket: 'my-bucket',
209
- region: 'oss-cn-hangzhou',
421
+ region: 'cn-hangzhou',
210
422
  credentials: {
211
423
  accessKeyId: process.env.OSS_ACCESS_KEY_ID!,
212
424
  secretAccessKey: process.env.OSS_ACCESS_KEY_SECRET!,
@@ -214,41 +426,98 @@ const adapter = new OSSAdapter({
214
426
  });
215
427
  ```
216
428
 
217
- ## BucketDB Options
429
+ | Option | Default |
430
+ |---|---|
431
+ | `bucket` | required |
432
+ | `region` | `OSS_REGION`. Write `cn-hangzhou`; the 0.3 form `oss-cn-hangzhou` is also accepted |
433
+ | `endpoint` | `https://s3.oss-<region>.aliyuncs.com` (OSS's S3-compatible endpoint). `OSS_ENDPOINT` is not read |
434
+ | `credentials` | `OSS_ACCESS_KEY_ID`, `OSS_ACCESS_KEY_SECRET`, `OSS_SESSION_TOKEN`; if unset, `ALIBABA_CLOUD_ACCESS_KEY_ID`, `ALIBABA_CLOUD_ACCESS_KEY_SECRET`, `ALIBABA_CLOUD_SECURITY_TOKEN` (what `aliyun/configure-aliyun-credentials-action` exports after OIDC) |
435
+ | `timeoutMs` | `10000` (per attempt) |
436
+
437
+ New OSS users in mainland China may have to access buckets through a custom domain, passed as `endpoint`. S3-compatible access through a custom domain has not been verified.
438
+
439
+ OSS does not support conditional updates: `update` with `etag` throws `UnsupportedError`. `insert` relies on `x-oss-forbid-overwrite`, which OSS ignores on versioned buckets.
440
+
441
+ ### Credentials
442
+
443
+ `credentials` is either an object or an async function:
218
444
 
219
445
  ```typescript
220
- new BucketDB(adapter, dbPath, options?)
446
+ interface Credentials {
447
+ accessKeyId: string;
448
+ secretAccessKey: string;
449
+ sessionToken?: string;
450
+ expiresAt?: Date;
451
+ }
452
+ type CredentialProvider = () => Promise<Credentials>;
453
+
454
+ const adapter = new S3Adapter({
455
+ bucket: 'my-bucket',
456
+ region: 'us-east-1',
457
+ credentials: async () => {
458
+ const sts = await fetchTemporaryCredentials(); // your code
459
+ return {
460
+ accessKeyId: sts.accessKeyId,
461
+ secretAccessKey: sts.secretAccessKey,
462
+ sessionToken: sts.sessionToken,
463
+ expiresAt: new Date(sts.expiration),
464
+ };
465
+ },
466
+ });
221
467
  ```
222
468
 
223
- | Parameter | Type | Description |
224
- |---|---|---|
225
- | `adapter` | `StorageAdapter` | Storage backend |
226
- | `dbPath` | `string` | Namespace for this database (e.g. `'my-app'`, `'tenant-123'`) |
227
- | `options.defaultShardCount` | `number` | Default index shard count (default: `16`) |
469
+ The function's result is cached; it is called again 5 minutes before `expiresAt` (never, if `expiresAt` is absent). A failed call is not cached. Without the option, missing environment variables throw `ValidationError` on the first request.
470
+
471
+ ### Retries
472
+
473
+ `S3Adapter` and `OSSAdapter` make up to 3 attempts per request, with jittered exponential backoff, on network errors, timeouts, 5xx, 429, `SlowDown` and `RequestTimeout`. Other 4xx responses are not retried. A failure that persists becomes `StorageError` with `kind: 'transient'`. If a conditional write hits a conflict after a transient failure, the adapter reads the object back and compares it with what it wrote, so an earlier attempt that did succeed is not reported as a conflict.
228
474
 
229
- ## Collection Options
475
+ ## BucketDB
230
476
 
231
477
  ```typescript
232
- db.collection<T>(name, options?)
478
+ new BucketDB(adapter: StorageAdapter, dbPath: string, options?: BucketDBOptions)
479
+ BucketDB.connect(url?: string, options?: BucketDBOptions): BucketDB // see Connection URLs; default BUCKET_DB_URL
480
+ db.collection<T extends Document>(name: string): Collection<T>
481
+
482
+ interface BucketDBOptions {
483
+ logger?: Logger; // default: console.warn
484
+ }
485
+ type Logger = (event: LogEvent) => void;
486
+ interface LogEvent {
487
+ level: 'warn' | 'error';
488
+ message: string;
489
+ error?: unknown;
490
+ }
233
491
  ```
234
492
 
235
- | Parameter | Type | Description |
236
- |---|---|---|
237
- | `name` | `string` | Collection name |
238
- | `options.shardCount` | `number` | Override shard count for this collection |
493
+ | Parameter | Description |
494
+ |---|---|
495
+ | `adapter` | Storage backend |
496
+ | `dbPath` | Key prefix for this database (e.g. `'my-app'`, `'tenants/123'`); `''` for the bucket root |
497
+ | `options.logger` | Receives snapshot write failures and envelope warnings; see [Logger](#logger) |
498
+ | `name` | Collection name; the same name returns the same `Collection` instance |
499
+
500
+ A custom `StorageAdapter` can set `readonly remote = true` to get the query snapshot; the built-in `S3Adapter` and `OSSAdapter` do.
239
501
 
240
- ## Features
502
+ ## Upgrading
241
503
 
242
- - No database servers — uses S3 / OSS / local filesystem as backend
243
- - Type-safe TypeScript API with full inference
244
- - Optimistic locking via ETag
245
- - Sharded index design for scale
246
- - Unified API across all storage backends
504
+ **From 0.4.** The storage format is unchanged. Queries changed: `$ne` / `$nin` now match documents without the field, range comparisons between mismatched types no longer throw, `find` without `sort` orders by `_createdAt` then `id`, and `limit: 0` returns `[]`. The MCP server is configured by `BUCKET_DB_URL`. Details: [MIGRATION-0.5.md](https://github.com/holdbaby5483/bucket-db/blob/main/docs/MIGRATION-0.5.md).
505
+
506
+ **From 0.3.x.**
507
+
508
+ 0.4 reads 0.3 collections as they are: `docs/<id>.json` is unchanged. Besides `docs/`, 0.4 reads only its own `_snapshot.v1.json` (S3 / OSS), so the index and log directories 0.3 kept next to `docs/`, and 0.3's `*.meta` files, are ignored and can be deleted. On S3 / OSS, queries create a `_snapshot.v1.json` next to `docs/` in the collections they read (once more than 16 documents had to be fetched); with read-only credentials it is simply not written. Stop every 0.3 writer before 0.4 starts writing. The API changes are listed in the [migration guide](https://github.com/holdbaby5483/bucket-db/blob/main/docs/MIGRATION-0.4.md) and the [changelog](https://github.com/holdbaby5483/bucket-db/blob/main/CHANGELOG.md).
247
509
 
248
510
  ## Documentation
249
511
 
250
512
  📚 https://bucket-db.vercel.app
251
513
 
514
+ - [API reference](https://github.com/holdbaby5483/bucket-db/blob/main/docs/API.md)
515
+ - [CLI reference](https://github.com/holdbaby5483/bucket-db/blob/main/docs/CLI.md) and [CI recipes](https://github.com/holdbaby5483/bucket-db/tree/main/examples/ci)
516
+ - [Guarantees and limits](https://github.com/holdbaby5483/bucket-db/blob/main/docs/GUARANTEES.md): what each backend guarantees and the test that checks it
517
+ - [Storage format](https://github.com/holdbaby5483/bucket-db/blob/main/docs/STORAGE-FORMAT.md): the keys and JSON BucketDB writes
518
+ - [Benchmarks](https://github.com/holdbaby5483/bucket-db/blob/main/docs/BENCHMARKS.md): requests and latency at 100 to 10,000 documents on OSS, S3 and FileSystem
519
+ - [Migrating from 0.4](https://github.com/holdbaby5483/bucket-db/blob/main/docs/MIGRATION-0.5.md) and [from 0.3](https://github.com/holdbaby5483/bucket-db/blob/main/docs/MIGRATION-0.4.md)
520
+
252
521
  ## License
253
522
 
254
523
  MIT
@@ -0,0 +1,25 @@
1
+ import type { Document, Logger } from './types.js';
2
+ import type { StorageAdapter } from './storage/adapter.js';
3
+ import { Collection } from './collection.js';
4
+ export interface BucketDBOptions {
5
+ /** Receives snapshot write failures and envelope warnings. Default: console.warn. */
6
+ logger?: Logger;
7
+ }
8
+ /**
9
+ * Entry point: a storage backend plus a key prefix (`dbPath`, `''` for the bucket root).
10
+ */
11
+ export declare class BucketDB {
12
+ private readonly adapter;
13
+ private readonly dbPath;
14
+ private readonly options;
15
+ private readonly collections;
16
+ constructor(adapter: StorageAdapter, dbPath: string, options?: BucketDBOptions);
17
+ /**
18
+ * Opens a database from a connection URL, by default `BUCKET_DB_URL`:
19
+ * `s3://bucket/prefix?region=…`, `oss://bucket/prefix?region=…`,
20
+ * `file:///dir` or `memory://`. Credentials come from the environment, never the URL.
21
+ */
22
+ static connect(url?: string, options?: BucketDBOptions): BucketDB;
23
+ /** Returns the same Collection instance for the same name. */
24
+ collection<T extends Document>(name: string): Collection<T>;
25
+ }
@@ -0,0 +1,36 @@
1
+ import { Collection } from './collection.js';
2
+ import { validateDbPath } from './keys.js';
3
+ import { openUrl } from './url.js';
4
+ /**
5
+ * Entry point: a storage backend plus a key prefix (`dbPath`, `''` for the bucket root).
6
+ */
7
+ export class BucketDB {
8
+ adapter;
9
+ dbPath;
10
+ options;
11
+ collections = new Map();
12
+ constructor(adapter, dbPath, options = {}) {
13
+ this.adapter = adapter;
14
+ this.dbPath = dbPath;
15
+ this.options = options;
16
+ validateDbPath(dbPath);
17
+ }
18
+ /**
19
+ * Opens a database from a connection URL, by default `BUCKET_DB_URL`:
20
+ * `s3://bucket/prefix?region=…`, `oss://bucket/prefix?region=…`,
21
+ * `file:///dir` or `memory://`. Credentials come from the environment, never the URL.
22
+ */
23
+ static connect(url, options = {}) {
24
+ const { adapter, dbPath } = openUrl(url);
25
+ return new BucketDB(adapter, dbPath, options);
26
+ }
27
+ /** Returns the same Collection instance for the same name. */
28
+ collection(name) {
29
+ let collection = this.collections.get(name);
30
+ if (!collection) {
31
+ collection = new Collection(this.adapter, this.dbPath, name, { logger: this.options.logger });
32
+ this.collections.set(name, collection);
33
+ }
34
+ return collection;
35
+ }
36
+ }
package/dist/cli.d.ts ADDED
@@ -0,0 +1,2 @@
1
+ #!/usr/bin/env node
2
+ export {};