@hold-baby/bucket-db 0.3.1 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (114) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +331 -62
  3. package/dist/bucket-db.d.ts +25 -0
  4. package/dist/bucket-db.js +36 -0
  5. package/dist/cli.d.ts +2 -0
  6. package/dist/cli.js +133 -0
  7. package/dist/collection.d.ts +78 -0
  8. package/dist/collection.js +321 -0
  9. package/dist/doctor.d.ts +7 -0
  10. package/dist/doctor.js +100 -0
  11. package/dist/errors.d.ts +59 -0
  12. package/dist/errors.js +57 -0
  13. package/dist/index.d.ts +13 -9
  14. package/dist/index.js +6 -10
  15. package/dist/keys.d.ts +12 -0
  16. package/dist/keys.js +31 -0
  17. package/dist/query.d.ts +16 -0
  18. package/dist/query.js +133 -0
  19. package/dist/snapshot.d.ts +16 -0
  20. package/dist/snapshot.js +40 -0
  21. package/dist/storage/adapter.d.ts +75 -0
  22. package/dist/storage/adapter.js +27 -0
  23. package/dist/storage/fs.d.ts +28 -0
  24. package/dist/storage/fs.js +175 -0
  25. package/dist/storage/memory.d.ts +18 -0
  26. package/dist/storage/memory.js +49 -0
  27. package/dist/storage/s3/adapters.d.ts +38 -0
  28. package/dist/storage/s3/adapters.js +65 -0
  29. package/dist/storage/s3/client.d.ts +52 -0
  30. package/dist/storage/s3/client.js +231 -0
  31. package/dist/storage/s3/credentials.d.ts +17 -0
  32. package/dist/storage/s3/credentials.js +42 -0
  33. package/dist/storage/s3/sigv4.d.ts +32 -0
  34. package/dist/storage/s3/sigv4.js +75 -0
  35. package/dist/types.d.ts +55 -0
  36. package/dist/types.js +1 -0
  37. package/dist/url.d.ts +16 -0
  38. package/dist/url.js +66 -0
  39. package/llms.txt +116 -20
  40. package/package.json +7 -5
  41. package/dist/cache/lru-cache.d.ts +0 -47
  42. package/dist/cache/lru-cache.d.ts.map +0 -1
  43. package/dist/cache/lru-cache.js +0 -110
  44. package/dist/cache/lru-cache.js.map +0 -1
  45. package/dist/core/bucket-db.d.ts +0 -20
  46. package/dist/core/bucket-db.d.ts.map +0 -1
  47. package/dist/core/bucket-db.js +0 -32
  48. package/dist/core/bucket-db.js.map +0 -1
  49. package/dist/core/collection.d.ts +0 -26
  50. package/dist/core/collection.d.ts.map +0 -1
  51. package/dist/core/collection.js +0 -213
  52. package/dist/core/collection.js.map +0 -1
  53. package/dist/core/wal.d.ts +0 -45
  54. package/dist/core/wal.d.ts.map +0 -1
  55. package/dist/core/wal.js +0 -80
  56. package/dist/core/wal.js.map +0 -1
  57. package/dist/index/shard-manager.d.ts +0 -36
  58. package/dist/index/shard-manager.d.ts.map +0 -1
  59. package/dist/index/shard-manager.js +0 -177
  60. package/dist/index/shard-manager.js.map +0 -1
  61. package/dist/index.d.ts.map +0 -1
  62. package/dist/index.js.map +0 -1
  63. package/dist/query/evaluator.d.ts +0 -10
  64. package/dist/query/evaluator.d.ts.map +0 -1
  65. package/dist/query/evaluator.js +0 -127
  66. package/dist/query/evaluator.js.map +0 -1
  67. package/dist/storage/fs-adapter.d.ts +0 -24
  68. package/dist/storage/fs-adapter.d.ts.map +0 -1
  69. package/dist/storage/fs-adapter.js +0 -208
  70. package/dist/storage/fs-adapter.js.map +0 -1
  71. package/dist/storage/memory-adapter.d.ts +0 -22
  72. package/dist/storage/memory-adapter.d.ts.map +0 -1
  73. package/dist/storage/memory-adapter.js +0 -78
  74. package/dist/storage/memory-adapter.js.map +0 -1
  75. package/dist/storage/oss-adapter.d.ts +0 -26
  76. package/dist/storage/oss-adapter.d.ts.map +0 -1
  77. package/dist/storage/oss-adapter.js +0 -145
  78. package/dist/storage/oss-adapter.js.map +0 -1
  79. package/dist/storage/s3-adapter.d.ts +0 -26
  80. package/dist/storage/s3-adapter.d.ts.map +0 -1
  81. package/dist/storage/s3-adapter.js +0 -170
  82. package/dist/storage/s3-adapter.js.map +0 -1
  83. package/dist/types/collection.d.ts +0 -40
  84. package/dist/types/collection.d.ts.map +0 -1
  85. package/dist/types/collection.js +0 -2
  86. package/dist/types/collection.js.map +0 -1
  87. package/dist/types/document.d.ts +0 -18
  88. package/dist/types/document.d.ts.map +0 -1
  89. package/dist/types/document.js +0 -2
  90. package/dist/types/document.js.map +0 -1
  91. package/dist/types/errors.d.ts +0 -32
  92. package/dist/types/errors.d.ts.map +0 -1
  93. package/dist/types/errors.js +0 -48
  94. package/dist/types/errors.js.map +0 -1
  95. package/dist/types/index.d.ts +0 -6
  96. package/dist/types/index.d.ts.map +0 -1
  97. package/dist/types/index.js +0 -3
  98. package/dist/types/index.js.map +0 -1
  99. package/dist/types/query.d.ts +0 -37
  100. package/dist/types/query.d.ts.map +0 -1
  101. package/dist/types/query.js +0 -2
  102. package/dist/types/query.js.map +0 -1
  103. package/dist/types/storage.d.ts +0 -45
  104. package/dist/types/storage.d.ts.map +0 -1
  105. package/dist/types/storage.js +0 -2
  106. package/dist/types/storage.js.map +0 -1
  107. package/dist/utils/hash.d.ts +0 -13
  108. package/dist/utils/hash.d.ts.map +0 -1
  109. package/dist/utils/hash.js +0 -23
  110. package/dist/utils/hash.js.map +0 -1
  111. package/dist/utils/path-validator.d.ts +0 -46
  112. package/dist/utils/path-validator.d.ts.map +0 -1
  113. package/dist/utils/path-validator.js +0 -99
  114. package/dist/utils/path-validator.js.map +0 -1
@@ -0,0 +1,42 @@
1
+ import { ValidationError } from '../../errors.js';
2
+ const REFRESH_MARGIN_MS = 5 * 60 * 1000;
3
+ /**
4
+ * Turns the `credentials` option into a function returning current
5
+ * credentials. Without the option, credentials come from the first variable
6
+ * set in `envSets` that has both an id and a secret, read on first use;
7
+ * none set throws ValidationError then.
8
+ */
9
+ export function resolveCredentials(option, envSets) {
10
+ if (typeof option === 'object')
11
+ return () => Promise.resolve(option);
12
+ const provider = option ??
13
+ (async () => {
14
+ for (const [idVar, secretVar, tokenVar] of envSets) {
15
+ const accessKeyId = process.env[idVar];
16
+ const secretAccessKey = process.env[secretVar];
17
+ if (accessKeyId && secretAccessKey) {
18
+ return { accessKeyId, secretAccessKey, sessionToken: process.env[tokenVar] || undefined };
19
+ }
20
+ }
21
+ const names = envSets.map(([idVar, secretVar]) => `${idVar} and ${secretVar}`).join(', or ');
22
+ throw new ValidationError(`No credentials: pass the credentials option or set ${names}`);
23
+ });
24
+ let cached;
25
+ let refreshAt = Infinity;
26
+ return () => {
27
+ if (!cached || Date.now() >= refreshAt) {
28
+ const pending = provider().then(credentials => {
29
+ refreshAt = credentials.expiresAt ? credentials.expiresAt.getTime() - REFRESH_MARGIN_MS : Infinity;
30
+ return credentials;
31
+ });
32
+ // A failed call is not cached; the next request calls the provider again.
33
+ pending.catch(() => {
34
+ if (cached === pending)
35
+ cached = undefined;
36
+ });
37
+ cached = pending;
38
+ refreshAt = Infinity;
39
+ }
40
+ return cached;
41
+ };
42
+ }
@@ -0,0 +1,32 @@
1
+ /**
2
+ * AWS Signature Version 4 for S3-protocol requests, on Web Crypto.
3
+ * https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html
4
+ */
5
+ export interface SigningCredentials {
6
+ accessKeyId: string;
7
+ secretAccessKey: string;
8
+ sessionToken?: string;
9
+ }
10
+ export interface SignInput {
11
+ method: string;
12
+ /** Path must already be RFC 3986 encoded; it is signed as is (S3 does not double-encode). */
13
+ url: URL;
14
+ /** Headers the caller sets itself; all of them are signed. `host` is taken from `url`. */
15
+ headers: Record<string, string>;
16
+ /** Hex SHA-256 of the body; `EMPTY_SHA256` without a body. */
17
+ payloadHash: string;
18
+ credentials: SigningCredentials;
19
+ region: string;
20
+ date: Date;
21
+ service?: string;
22
+ }
23
+ export declare const EMPTY_SHA256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
24
+ /** RFC 3986 percent-encoding: everything except unreserved `A-Z a-z 0-9 - . _ ~`. */
25
+ export declare function encodeRfc3986(value: string): string;
26
+ export declare function sha256Hex(data: Uint8Array | string): Promise<string>;
27
+ /**
28
+ * Returns the headers to add to the request: `x-amz-date`,
29
+ * `x-amz-content-sha256`, `x-amz-security-token` (with a session token) and
30
+ * `authorization`.
31
+ */
32
+ export declare function signRequest(input: SignInput): Promise<Record<string, string>>;
@@ -0,0 +1,75 @@
1
+ /**
2
+ * AWS Signature Version 4 for S3-protocol requests, on Web Crypto.
3
+ * https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html
4
+ */
5
+ export const EMPTY_SHA256 = 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855';
6
+ const encoder = new TextEncoder();
7
+ /** RFC 3986 percent-encoding: everything except unreserved `A-Z a-z 0-9 - . _ ~`. */
8
+ export function encodeRfc3986(value) {
9
+ return encodeURIComponent(value).replace(/[!'()*]/g, c => `%${c.charCodeAt(0).toString(16).toUpperCase()}`);
10
+ }
11
+ function hex(buffer) {
12
+ return Array.from(new Uint8Array(buffer), b => b.toString(16).padStart(2, '0')).join('');
13
+ }
14
+ export async function sha256Hex(data) {
15
+ const bytes = typeof data === 'string' ? encoder.encode(data) : data;
16
+ return hex(await crypto.subtle.digest('SHA-256', bytes));
17
+ }
18
+ async function hmac(key, data) {
19
+ const cryptoKey = await crypto.subtle.importKey('raw', key, { name: 'HMAC', hash: 'SHA-256' }, false, ['sign']);
20
+ return crypto.subtle.sign('HMAC', cryptoKey, encoder.encode(data));
21
+ }
22
+ /** `20130524T000000Z` */
23
+ function amzDate(date) {
24
+ return date
25
+ .toISOString()
26
+ .replace(/[-:]/g, '')
27
+ .replace(/\.\d{3}/, '');
28
+ }
29
+ /**
30
+ * Returns the headers to add to the request: `x-amz-date`,
31
+ * `x-amz-content-sha256`, `x-amz-security-token` (with a session token) and
32
+ * `authorization`.
33
+ */
34
+ export async function signRequest(input) {
35
+ const { credentials, region, payloadHash } = input;
36
+ const service = input.service ?? 's3';
37
+ const timestamp = amzDate(input.date);
38
+ const day = timestamp.slice(0, 8);
39
+ const added = {
40
+ 'x-amz-date': timestamp,
41
+ 'x-amz-content-sha256': payloadHash,
42
+ };
43
+ if (credentials.sessionToken)
44
+ added['x-amz-security-token'] = credentials.sessionToken;
45
+ const signed = new Map([['host', input.url.host]]);
46
+ for (const [name, value] of Object.entries({ ...input.headers, ...added })) {
47
+ signed.set(name.toLowerCase(), value.trim().replace(/\s+/g, ' '));
48
+ }
49
+ const names = [...signed.keys()].sort();
50
+ const signedHeaders = names.join(';');
51
+ // Sort by encoded name, then value.
52
+ const canonicalQuery = [...input.url.searchParams]
53
+ .map(([name, value]) => [encodeRfc3986(name), encodeRfc3986(value)])
54
+ .sort(([a, x], [b, y]) => (a === b ? (x < y ? -1 : x > y ? 1 : 0) : a < b ? -1 : 1))
55
+ .map(([name, value]) => `${name}=${value}`)
56
+ .join('&');
57
+ const canonicalRequest = [
58
+ input.method,
59
+ input.url.pathname || '/',
60
+ canonicalQuery,
61
+ names.map(name => `${name}:${signed.get(name)}\n`).join(''),
62
+ signedHeaders,
63
+ payloadHash,
64
+ ].join('\n');
65
+ const scope = `${day}/${region}/${service}/aws4_request`;
66
+ const stringToSign = ['AWS4-HMAC-SHA256', timestamp, scope, await sha256Hex(canonicalRequest)].join('\n');
67
+ let key = await hmac(encoder.encode(`AWS4${credentials.secretAccessKey}`), day);
68
+ for (const part of [region, service, 'aws4_request'])
69
+ key = await hmac(key, part);
70
+ const signature = hex(await hmac(key, stringToSign));
71
+ added.authorization =
72
+ `AWS4-HMAC-SHA256 Credential=${credentials.accessKeyId}/${scope}, ` +
73
+ `SignedHeaders=${signedHeaders}, Signature=${signature}`;
74
+ return added;
75
+ }
@@ -0,0 +1,55 @@
1
+ /**
2
+ * Every document carries these fields. `id` comes from the object key and
3
+ * `_etag` from the storage service; neither is stored in the document body.
4
+ */
5
+ export interface Document {
6
+ id: string;
7
+ _etag?: string;
8
+ _createdAt?: string;
9
+ _updatedAt?: string;
10
+ }
11
+ type SystemField = 'id' | '_etag' | '_createdAt' | '_updatedAt';
12
+ /** Fields accepted by `insert`: everything except system fields. */
13
+ export type InsertDocument<T extends Document> = Omit<T, SystemField>;
14
+ /** Fields accepted by `update`: a partial document without system fields. */
15
+ export type UpdateDocument<T extends Document> = Partial<Omit<T, SystemField>>;
16
+ export type QueryOperator = '$eq' | '$ne' | '$gt' | '$gte' | '$lt' | '$lte' | '$in' | '$nin';
17
+ export type QueryValue<T> = T | {
18
+ $eq?: T;
19
+ $ne?: T;
20
+ $gt?: T;
21
+ $gte?: T;
22
+ $lt?: T;
23
+ $lte?: T;
24
+ $in?: T[];
25
+ $nin?: T[];
26
+ };
27
+ export type QueryFilter<T> = {
28
+ [K in keyof T]?: QueryValue<T[K]>;
29
+ };
30
+ export interface QueryOptions {
31
+ /** 0 returns no documents. Default: all. */
32
+ limit?: number;
33
+ offset?: number;
34
+ /**
35
+ * Fields to sort by, in order: 1 ascending, -1 descending. Missing values sort
36
+ * last. Ties, and the default order, are `_createdAt` then `id`.
37
+ */
38
+ sort?: Record<string, 1 | -1>;
39
+ }
40
+ export interface InsertOptions {
41
+ /** Use this id instead of a generated one; fails with DocumentExistsError if it is taken. */
42
+ id?: string;
43
+ }
44
+ export interface UpdateOptions {
45
+ /** Only update if the document still has this etag. */
46
+ etag?: string;
47
+ }
48
+ export interface LogEvent {
49
+ level: 'warn' | 'error';
50
+ message: string;
51
+ error?: unknown;
52
+ }
53
+ /** Receives problems that do not fail the current call. Default: `console.warn`. */
54
+ export type Logger = (event: LogEvent) => void;
55
+ export {};
package/dist/types.js ADDED
@@ -0,0 +1 @@
1
+ export {};
package/dist/url.d.ts ADDED
@@ -0,0 +1,16 @@
1
+ import type { StorageAdapter } from './storage/adapter.js';
2
+ /**
3
+ * Resolves a connection URL (default: `BUCKET_DB_URL`) to a backend and a dbPath:
4
+ *
5
+ * - `s3://bucket/prefix?region=…&endpoint=…&forcePathStyle=true&timeoutMs=…`
6
+ * - `oss://bucket/prefix?region=…&endpoint=…&timeoutMs=…`
7
+ * - `file:///absolute/dir`, `file://./relative/dir`
8
+ * - `memory://prefix`
9
+ *
10
+ * Credentials are never accepted in the URL, so it can be logged safely.
11
+ * Error messages never repeat the URL.
12
+ */
13
+ export declare function openUrl(url?: string | undefined): {
14
+ adapter: StorageAdapter;
15
+ dbPath: string;
16
+ };
package/dist/url.js ADDED
@@ -0,0 +1,66 @@
1
+ import { MemoryStorageAdapter } from './storage/memory.js';
2
+ import { FileSystemAdapter } from './storage/fs.js';
3
+ import { OSSAdapter, S3Adapter } from './storage/s3/adapters.js';
4
+ import { ValidationError } from './errors.js';
5
+ const PARAMETERS = {
6
+ s3: ['region', 'endpoint', 'forcePathStyle', 'timeoutMs'],
7
+ oss: ['region', 'endpoint', 'timeoutMs'],
8
+ memory: [],
9
+ };
10
+ /**
11
+ * Resolves a connection URL (default: `BUCKET_DB_URL`) to a backend and a dbPath:
12
+ *
13
+ * - `s3://bucket/prefix?region=…&endpoint=…&forcePathStyle=true&timeoutMs=…`
14
+ * - `oss://bucket/prefix?region=…&endpoint=…&timeoutMs=…`
15
+ * - `file:///absolute/dir`, `file://./relative/dir`
16
+ * - `memory://prefix`
17
+ *
18
+ * Credentials are never accepted in the URL, so it can be logged safely.
19
+ * Error messages never repeat the URL.
20
+ */
21
+ export function openUrl(url = process.env.BUCKET_DB_URL) {
22
+ if (!url)
23
+ throw new ValidationError('No connection URL: pass one or set BUCKET_DB_URL');
24
+ if (url.startsWith('file:')) {
25
+ const path = decodeURIComponent(url.replace(/^file:(\/\/)?/, ''));
26
+ if (!path || path.includes('?'))
27
+ throw new ValidationError('file:// URLs take a directory and no parameters');
28
+ return { adapter: new FileSystemAdapter({ basePath: path }), dbPath: '' };
29
+ }
30
+ let parsed;
31
+ try {
32
+ parsed = new URL(url);
33
+ }
34
+ catch {
35
+ throw new ValidationError('Invalid connection URL');
36
+ }
37
+ if (parsed.username || parsed.password) {
38
+ throw new ValidationError('Connection URLs must not contain credentials; use environment variables or the credentials option');
39
+ }
40
+ const scheme = parsed.protocol.slice(0, -1);
41
+ const allowed = PARAMETERS[scheme];
42
+ if (!allowed)
43
+ throw new ValidationError(`Unsupported scheme ${scheme}: use s3://, oss://, file:// or memory://`);
44
+ const params = {};
45
+ for (const [name, value] of parsed.searchParams) {
46
+ if (!allowed.includes(name)) {
47
+ throw new ValidationError(`Unknown parameter "${name}" for ${scheme}://; allowed: ${allowed.join(', ') || 'none'}`);
48
+ }
49
+ params[name] = value;
50
+ }
51
+ const path = decodeURIComponent(parsed.pathname).replace(/^\/+|\/+$/g, '');
52
+ if (scheme === 'memory') {
53
+ return { adapter: new MemoryStorageAdapter(), dbPath: [parsed.hostname, path].filter(Boolean).join('/') };
54
+ }
55
+ if (!parsed.hostname)
56
+ throw new ValidationError(`${scheme}:// URLs need a bucket: ${scheme}://bucket/prefix`);
57
+ const timeoutMs = params.timeoutMs === undefined ? undefined : Number(params.timeoutMs);
58
+ const options = { bucket: parsed.hostname, region: params.region, endpoint: params.endpoint, timeoutMs };
59
+ const adapter = scheme === 's3'
60
+ ? new S3Adapter({
61
+ ...options,
62
+ forcePathStyle: params.forcePathStyle === undefined ? undefined : params.forcePathStyle === 'true',
63
+ })
64
+ : new OSSAdapter(options);
65
+ return { adapter, dbPath: path };
66
+ }
package/llms.txt CHANGED
@@ -1,46 +1,142 @@
1
1
  # BucketDB
2
2
 
3
- TypeScript document database. Uses S3 / Alibaba Cloud OSS / local filesystem as backend. No database server required.
3
+ TypeScript document database (version 0.5.0). Uses AWS S3 / Alibaba Cloud OSS / local filesystem / memory as backend. No database server, no runtime dependencies. Requires Node >= 20 or Bun. Includes the `bucket-db` CLI.
4
4
 
5
5
  ## Key concepts
6
6
 
7
- - `BucketDB` — main class, takes a storage adapter and a `dbPath` namespace
8
- - `Collection<T>` — typed collection, returned by `db.collection<T>(name)`
9
- - Every document extends `Document` which adds: `id` (UUID), `_etag`, `_createdAt`, `_updatedAt`
10
- - `InsertDocument<T>` omits system fields (pass only your own fields)
11
- - Optimistic locking: pass `{ etag: doc._etag }` to `update()` to prevent concurrent overwrites
7
+ - `BucketDB` — `BucketDB.connect(url?, { logger? })` (url default: env BUCKET_DB_URL) or `new BucketDB(adapter, dbPath, { logger? })`; `dbPath` is a key prefix (`''` = bucket root, may contain `/`)
8
+ - Connection URLs: `s3://bucket/prefix?region=&endpoint=&forcePathStyle=&timeoutMs=`, `oss://bucket/prefix?region=&endpoint=&timeoutMs=`, `file:///abs/dir` or `file://./rel/dir` (dbPath ''), `memory://prefix`. Path = dbPath. Unknown query parameters → ValidationError. Credentials never in the URL (`user:secret@` is refused); they come from env (AWS_* / OSS_* / ALIBABA_CLOUD_*). Error messages never repeat the URL, so it is safe to log. URL-encode `endpoint` values.
9
+ - `Collection<T>` — typed collection, returned by `db.collection<T>(name)` (same instance per name)
10
+ - Storage: one JSON object per document at `<dbPath>/<collection>/docs/<id>.json`. No index, no log, no result cache. Format unchanged since 0.4.
11
+ - Query snapshot (S3 / OSS / custom adapters with `remote: true` only): `<dbPath>/<collection>/_snapshot.v1.json` = `{ format: 'bucket-db-snapshot', version: 1, documents: { <id>: { etag, body } } }`. find LISTs docs/, uses a snapshot entry only if its etag equals the listed etag, GETs the rest (16 in parallel). The LIST is the authority: a missing/corrupt/stale/overwritten snapshot only costs speed, never changes results. Rewritten (unconditional PUT) when > 16 documents had to be read individually (+ entries of deleted docs). Memory / FileSystem keep no snapshot.
12
+ - Every document extends `Document`: `id` (generated UUID unless chosen, taken from the object key), `_etag` (from storage, not stored in the body), `_createdAt`, `_updatedAt`
13
+ - `InsertDocument<T>` and `UpdateDocument<T>` exclude `id`, `_etag`, `_createdAt`, `_updatedAt` (UpdateDocument is also Partial); system fields passed at runtime are dropped
14
+ - put vs insert: `put(id, data)` creates or replaces the whole document (unconditional, idempotent, resets `_createdAt`); `insert(data, { id })` creates only if the id is free, else DocumentExistsError; `insert(data)` generates a UUID.
15
+ - Optimistic locking: `update(id, data, { etag: doc._etag })`; stale etag or lost race throws ConcurrentUpdateError (no retry). Works on Memory, S3, and FileSystem within one process. On OSS, update with etag throws UnsupportedError.
16
+ - update without etag: on Memory / FileSystem (one process) / S3, conflicting writes are retried (up to 5 retries) so concurrent partial updates all apply; on OSS last writer wins. update keeps `_createdAt`.
17
+
18
+ ## Guarantees per backend
19
+
20
+ | | Memory | FileSystem | S3 | OSS |
21
+ |--------------------------------------|--------|------------------------|-----|----------------------------------|
22
+ | single write atomic | yes | yes | yes | yes |
23
+ | create-if-absent (insert, also { id })| yes | yes (across processes) | yes | yes (non-versioned buckets only) |
24
+ | conditional update (etag) | yes | single process only | yes | no — UnsupportedError |
25
+ | concurrent partial updates all apply | yes | single process | yes | no (last writer wins) |
26
+
27
+ Multiple writer processes are supported for insert/put/delete on every backend and for update on S3 (Memory/FileSystem: within one process). put is last-writer-wins everywhere.
28
+
29
+ ## Limitations (0.5.0)
30
+
31
+ - No transactions; each write touches one document. deleteMany is not atomic (one DELETE per match, 16 in parallel; a failure leaves earlier deletes done).
32
+ - Envelope: at most 10,000 documents and 10 MB of snapshot per collection; past it the collection logs one `warn` and queries keep getting slower. Measured warm find at 10,000 docs: OSS 1.35 s, S3 3.2 s, FileSystem ~0.5 s; no snapshot at 10,000 docs: OSS 34 s, S3 134 s (docs/BENCHMARKS.md).
33
+ - find cost on S3/OSS: ⌈N/1000⌉ LIST + 1 GET (snapshot) + 1 GET per document changed since the snapshot (≤ 16), or those GETs + 1 snapshot PUT when more changed. Memory/FileSystem: ⌈N/1000⌉ LIST + N GET. `{ id: '...' }` reads one document. sort/limit/offset do not reduce reads. count() without filter: LIST only.
34
+ - Results are not a point-in-time snapshot: each returned document is some committed version.
35
+ - Snapshot read errors other than not-found (transient, permission) fail find with StorageError. Snapshot write failures do not fail find; they go to the logger (level `error`); after 401/403 (read-only credentials) the collection stops writing it.
36
+ - Deleted documents' contents remain in the snapshot until it is rewritten or `compact()` runs.
37
+ - OSS: no conditional update (etag → UnsupportedError); concurrent etag-less updates are last-writer-wins.
38
+ - FileSystem: etag checks / concurrent update merging hold only within one process.
39
+ - 0.3.x and 0.4+ clients must not write the same collection concurrently. 0.4+ reads 0.3 data as-is; it reads only `docs/` and `_snapshot.v1.json` and skips 0.3's `*.meta` files, so 0.3's index and log directories can be deleted. 0.4 and 0.5 share the format.
12
40
 
13
41
  ## Package: @hold-baby/bucket-db
14
42
 
15
43
  ### Exports
16
44
 
17
45
  Classes: BucketDB, Collection, MemoryStorageAdapter, FileSystemAdapter, S3Adapter, OSSAdapter
18
- Errors: BucketDBError, DocumentNotFoundError, ConcurrentUpdateError, ValidationError, StorageError
19
- Types: Document, InsertDocument, UpdateDocument, QueryFilter, QueryOptions, UpdateOptions, StorageAdapter
46
+ Errors: BucketDBError, ValidationError, DocumentNotFoundError, DocumentExistsError, ConcurrentUpdateError, UnsupportedError, StorageError
47
+ Types: BucketDBOptions, CollectionOptions, Logger, LogEvent, Document, InsertDocument, UpdateDocument, QueryOperator, QueryValue, QueryFilter, QueryOptions, InsertOptions, UpdateOptions, FileSystemAdapterConfig, S3AdapterOptions, OSSAdapterOptions, Credentials, CredentialProvider, StorageAdapter, Capabilities, Guarantee, WriteCondition, ObjectEntry, ListPage, ListOptions, StorageErrorKind, StorageErrorOptions
48
+
49
+ ### BucketDB API
50
+
51
+ ```
52
+ BucketDB.connect(url?: string, options?: BucketDBOptions): BucketDB // sync; url default process.env.BUCKET_DB_URL
53
+ new BucketDB(adapter: StorageAdapter, dbPath: string, options?: BucketDBOptions)
54
+ db.collection<T extends Document>(name: string): Collection<T>
55
+ ```
20
56
 
21
57
  ### Collection API
22
58
 
23
59
  ```
24
- insert(data: InsertDocument<T>): Promise<T>
25
- findById(id: string): Promise<T> // throws DocumentNotFoundError if missing
26
- find(filter: QueryFilter<T>, options?: QueryOptions): Promise<T[]>
27
- update(id: string, data: UpdateDocument<T>, options?: UpdateOptions): Promise<T>
28
- delete(id: string, options?: UpdateOptions): Promise<void>
60
+ insert(data: InsertDocument<T>, options?: InsertOptions): Promise<T> // 1 conditional PUT; generated UUID, or options.id (DocumentExistsError if taken)
61
+ put(id: string, data: InsertDocument<T>): Promise<T> // 1 unconditional PUT; create or replace whole doc; idempotent; resets _createdAt
62
+ findById(id: string): Promise<T | null> // 1 GET; null ONLY if missing, other storage errors throw
63
+ find(filter?: QueryFilter<T>, options?: QueryOptions): Promise<T[]> // LIST + snapshot (S3/OSS) or GET all (local); default order _createdAt then id
64
+ count(filter?: QueryFilter<T>): Promise<number> // no filter: LIST only; with filter: like find
65
+ update(id: string, data: UpdateDocument<T>, options?: UpdateOptions): Promise<T> // GET + conditional PUT
66
+ delete(id: string): Promise<void> // 1 DELETE; no error if missing
67
+ deleteMany(filter: QueryFilter<T>): Promise<number> // find + 1 DELETE per match (16 parallel); NOT atomic; returns count
68
+ compact(): Promise<void> // S3/OSS: rebuild the snapshot (LIST + GET all + PUT), drops deleted docs' contents; throws on failure; no-op locally
29
69
  ```
30
70
 
31
- ### QueryOptions
71
+ ### Logger
72
+
73
+ ```
74
+ BucketDB.connect(url, { logger: (event: LogEvent) => void }) // or new BucketDB(adapter, dbPath, { logger }); default: console.warn
75
+ LogEvent = { level: 'warn' | 'error', message: string, error?: unknown }
76
+ ```
77
+
78
+ Receives: snapshot write failures (error), unparsable snapshot being rebuilt (warn), envelope exceeded (warn, once per collection).
79
+
80
+ ### QueryOptions / InsertOptions / UpdateOptions
32
81
 
33
82
  ```
34
- { limit?: number, offset?: number }
83
+ { sort?: Record<string, 1 | -1>, limit?: number, offset?: number } // filter → sort → offset/limit, in memory
84
+ { id?: string }
85
+ { etag?: string }
35
86
  ```
36
87
 
88
+ Order: default `_createdAt` asc then `id` asc (stable pagination); with sort, ties fall back to `_createdAt`, `id`. Missing/null values sort last in either direction. Mixed types: booleans < numbers < strings < others (JSON text). `limit: 0` → []. Negative/fractional limit/offset, sort direction not 1/-1, reserved sort field → ValidationError before any request.
89
+
37
90
  ### Query operators
38
91
 
39
- $eq, $ne, $gt, $gte, $lt, $lte, $in, $nin
92
+ $eq, $ne, $gt, $gte, $lt, $lte, $in, $nin (fields and operators combined with AND).
93
+ Semantics (settled in 0.5): a missing field is `undefined`. Missing field: $eq / plain value / ranges / $in → no match; $ne / $nin → match. Range operators only compare number-number or string-string; any other combination (mismatched types, booleans, null, objects) is no match, never an error. Equality is strict ===. `$in: []` matches nothing; `$nin: []` excludes nothing. `{ f: null }` does not match a missing f; `{ f: { $ne: null } }` does. No $exists.
94
+ Unknown operators, non-array $in/$nin, reserved field names (constructor, __proto__, …) throw ValidationError before any request.
95
+
96
+ ### Adapters
97
+
98
+ ```
99
+ new MemoryStorageAdapter()
100
+ new FileSystemAdapter({ basePath })
101
+ new S3Adapter({ bucket, region?, endpoint?, credentials?, forcePathStyle?, timeoutMs? })
102
+ new OSSAdapter({ bucket, region?, endpoint?, credentials?, timeoutMs? })
103
+ credentials: { accessKeyId, secretAccessKey, sessionToken?, expiresAt? } | (() => Promise<that>) // provider re-called 5 min before expiresAt
104
+ ```
105
+
106
+ Custom adapters implement StorageAdapter { capabilities, remote?, get, put, delete, list }; set `remote: true` to get the query snapshot (S3Adapter and OSSAdapter do). insert with { id } needs capabilities.createIfAbsent !== 'none' (else UnsupportedError).
107
+
108
+ S3 defaults: region AWS_REGION / AWS_DEFAULT_REGION; endpoint AWS_ENDPOINT_URL_S3 / AWS_ENDPOINT_URL / https://s3.<region>.amazonaws.com; credentials AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY / AWS_SESSION_TOKEN. Any S3-compatible service via endpoint; only AWS S3 and OSS are tested.
109
+ OSS defaults: region OSS_REGION (`cn-hangzhou`; `oss-cn-hangzhou` also accepted); endpoint https://s3.oss-<region>.aliyuncs.com (S3-compatible; OSS_ENDPOINT is not read); credentials OSS_ACCESS_KEY_ID / OSS_ACCESS_KEY_SECRET / OSS_SESSION_TOKEN, else ALIBABA_CLOUD_ACCESS_KEY_ID / ALIBABA_CLOUD_ACCESS_KEY_SECRET / ALIBABA_CLOUD_SECURITY_TOKEN (aliyun/configure-aliyun-credentials-action).
110
+ Path-style addressing is automatic for IP/localhost endpoints.
111
+ Retries (S3 and OSS): up to 3 attempts, per-attempt timeout 10 s (timeoutMs), on network errors, timeouts, 5xx, 429, SlowDown, RequestTimeout.
40
112
 
41
113
  ### Error mapping
42
114
 
43
- DocumentNotFoundError — findById / update / delete on missing doc
44
- ConcurrentUpdateError — update / delete with stale etag
45
- ValidationError — invalid input
46
- StorageError — underlying I/O failure
115
+ ValidationError — invalid collection name / document ID / dbPath segment / filter / sort / limit / offset; missing or invalid connection URL (unknown scheme or parameter, no bucket, credentials in URL); invalid adapter options; missing credentials
116
+ DocumentNotFoundError — update on a missing doc
117
+ DocumentExistsError — insert with an id that is taken
118
+ ConcurrentUpdateError — update with stale etag, lost race with etag, or etag-less update retries exhausted
119
+ UnsupportedError — backend lacks the capability (OSS update with etag)
120
+ StorageError — storage failure; fields kind ('not-found' | 'conflict' | 'transient' | 'permanent'), key, status. Also thrown by find for a file under docs/ that is not `<id>.json` holding a JSON object (key names the file). Hidden files (starting with `.`) are ignored.
121
+ All extend BucketDBError.
122
+
123
+ Names: collection names, document IDs and each `/`-separated dbPath segment must match ^[A-Za-z0-9_.-]{1,255}$ and not start with `.`; otherwise ValidationError (never rewritten).
124
+
125
+ ## CLI: bucket-db
126
+
127
+ ```
128
+ npx @hold-baby/bucket-db <command> [arguments] [--url <url>] # URL from --url or BUCKET_DB_URL
129
+ get <collection> <id> # print document; exit 2 if missing
130
+ put <collection> <id> [json] # create or replace; JSON from argument or stdin
131
+ find <collection> [filter] [--sort field[:asc|:desc]]... [--limit n] [--offset n] # prints one JSON array
132
+ count <collection> [filter] # prints a number
133
+ delete <collection> <id> # no error if missing
134
+ export <collection> # JSON Lines, no _etag
135
+ import <collection> [file] # JSON Lines from file/stdin; lines with id written in storage format (keeps id + timestamps, overwrites); lines without id inserted; prints count
136
+ compact <collection> # rebuild snapshot
137
+ doctor # check backend guarantees; writes only under <prefix>/_doctor-<time>/, cleans up
138
+ ```
139
+
140
+ stdout: one JSON value per line (jq-friendly); errors on stderr prefixed `bucket-db: `. Exit codes: 0 ok, 1 error (incl. a failed doctor check), 2 not found (get). Filters are JSON objects: `'{"ms":{"$lt":100}}'`.
141
+ doctor checks: object round trip; LIST reflects writes/overwrites/deletes immediately with matching etags; create-if-absent with 10 concurrent writers (exactly one wins); compare-and-swap with 10 concurrent writers (skipped where declared none, e.g. OSS). Lines start with `ok`, `skip` or `FAIL`; last line `all checks passed` or `<n> check(s) failed`.
142
+ CI: key records by run (`put runs "$GITHUB_RUN_ID-<shard>"`) so re-runs overwrite; per-job OIDC credentials (AWS_* / ALIBABA_CLOUD_*); recipes in examples/ci/ of the repository.
package/package.json CHANGED
@@ -1,10 +1,13 @@
1
1
  {
2
2
  "name": "@hold-baby/bucket-db",
3
- "version": "0.3.1",
3
+ "version": "0.5.0",
4
4
  "description": "A TypeScript document database built on cloud object storage (AWS S3 / Alibaba Cloud OSS) or local file system. No database servers needed.",
5
5
  "type": "module",
6
6
  "main": "./dist/index.js",
7
7
  "types": "./dist/index.d.ts",
8
+ "bin": {
9
+ "bucket-db": "dist/cli.js"
10
+ },
8
11
  "exports": {
9
12
  ".": {
10
13
  "types": "./dist/index.d.ts",
@@ -17,10 +20,6 @@
17
20
  "test": "bun test",
18
21
  "test:watch": "bun test --watch"
19
22
  },
20
- "dependencies": {
21
- "@aws-sdk/client-s3": "^3.490.0",
22
- "ali-oss": "^6.18.1"
23
- },
24
23
  "devDependencies": {
25
24
  "@types/bun": "latest",
26
25
  "typescript": "^5.3.3"
@@ -52,5 +51,8 @@
52
51
  "homepage": "https://github.com/holdbaby5483/bucket-db#readme",
53
52
  "bugs": {
54
53
  "url": "https://github.com/holdbaby5483/bucket-db/issues"
54
+ },
55
+ "engines": {
56
+ "node": ">=20"
55
57
  }
56
58
  }
@@ -1,47 +0,0 @@
1
- /**
2
- * LRU (Least Recently Used) Cache implementation
3
- */
4
- export declare class LRUCache<T> {
5
- private cache;
6
- private maxSize;
7
- private ttl;
8
- private hits;
9
- private misses;
10
- private evictions;
11
- constructor(maxSize?: number, ttl?: number);
12
- /**
13
- * Generate cache key from query filter and options
14
- */
15
- static generateKey(filter: Record<string, any>, options?: Record<string, any>): string;
16
- /**
17
- * Get value from cache
18
- */
19
- get(key: string): T | null;
20
- /**
21
- * Put value into cache
22
- */
23
- set(key: string, value: T): void;
24
- /**
25
- * Invalidate cache entries (clear all by default)
26
- */
27
- invalidate(pattern?: RegExp): void;
28
- /**
29
- * Get cache statistics
30
- */
31
- getStats(): {
32
- size: number;
33
- hits: number;
34
- misses: number;
35
- evictions: number;
36
- hitRate: string;
37
- };
38
- /**
39
- * Clear all cache statistics
40
- */
41
- resetStats(): void;
42
- /**
43
- * Clear the entire cache
44
- */
45
- clear(): void;
46
- }
47
- //# sourceMappingURL=lru-cache.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"lru-cache.d.ts","sourceRoot":"","sources":["../../src/cache/lru-cache.ts"],"names":[],"mappings":"AAOA;;GAEG;AACH,qBAAa,QAAQ,CAAC,CAAC;IACrB,OAAO,CAAC,KAAK,CAA6B;IAC1C,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,GAAG,CAAS;IAGpB,OAAO,CAAC,IAAI,CAAa;IACzB,OAAO,CAAC,MAAM,CAAa;IAC3B,OAAO,CAAC,SAAS,CAAa;gBAElB,OAAO,GAAE,MAAY,EAAE,GAAG,GAAE,MAAsB;IAM9D;;OAEG;IACH,MAAM,CAAC,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,MAAM;IAKtF;;OAEG;IACH,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,CAAC,GAAG,IAAI;IAuB1B;;OAEG;IACH,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,GAAG,IAAI;IAqBhC;;OAEG;IACH,UAAU,CAAC,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI;IAclC;;OAEG;IACH,QAAQ;;;;;;;IAWR;;OAEG;IACH,UAAU,IAAI,IAAI;IAMlB;;OAEG;IACH,KAAK,IAAI,IAAI;CAGd"}