@hold-baby/bucket-db 0.3.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +234 -19
- package/dist/cache/lru-cache.d.ts +47 -0
- package/dist/cache/lru-cache.d.ts.map +1 -0
- package/dist/cache/lru-cache.js +110 -0
- package/dist/cache/lru-cache.js.map +1 -0
- package/dist/core/bucket-db.d.ts +20 -0
- package/dist/core/bucket-db.d.ts.map +1 -0
- package/dist/core/bucket-db.js +32 -0
- package/dist/core/bucket-db.js.map +1 -0
- package/dist/core/collection.d.ts +26 -0
- package/dist/core/collection.d.ts.map +1 -0
- package/dist/core/collection.js +213 -0
- package/dist/core/collection.js.map +1 -0
- package/dist/core/wal.d.ts +45 -0
- package/dist/core/wal.d.ts.map +1 -0
- package/dist/core/wal.js +80 -0
- package/dist/core/wal.js.map +1 -0
- package/dist/index/shard-manager.d.ts +36 -0
- package/dist/index/shard-manager.d.ts.map +1 -0
- package/dist/index/shard-manager.js +177 -0
- package/dist/index/shard-manager.js.map +1 -0
- package/dist/index.d.ts +9 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +10 -0
- package/dist/index.js.map +1 -0
- package/dist/query/evaluator.d.ts +10 -0
- package/dist/query/evaluator.d.ts.map +1 -0
- package/dist/query/evaluator.js +127 -0
- package/dist/query/evaluator.js.map +1 -0
- package/dist/storage/fs-adapter.d.ts +24 -0
- package/dist/storage/fs-adapter.d.ts.map +1 -0
- package/dist/storage/fs-adapter.js +208 -0
- package/dist/storage/fs-adapter.js.map +1 -0
- package/dist/storage/memory-adapter.d.ts +22 -0
- package/dist/storage/memory-adapter.d.ts.map +1 -0
- package/dist/storage/memory-adapter.js +78 -0
- package/dist/storage/memory-adapter.js.map +1 -0
- package/dist/storage/oss-adapter.d.ts +26 -0
- package/dist/storage/oss-adapter.d.ts.map +1 -0
- package/dist/storage/oss-adapter.js +145 -0
- package/dist/storage/oss-adapter.js.map +1 -0
- package/dist/storage/s3-adapter.d.ts +26 -0
- package/dist/storage/s3-adapter.d.ts.map +1 -0
- package/dist/storage/s3-adapter.js +170 -0
- package/dist/storage/s3-adapter.js.map +1 -0
- package/dist/types/collection.d.ts +40 -0
- package/dist/types/collection.d.ts.map +1 -0
- package/dist/types/collection.js +2 -0
- package/dist/types/collection.js.map +1 -0
- package/{src/types/document.ts → dist/types/document.d.ts} +6 -10
- package/dist/types/document.d.ts.map +1 -0
- package/dist/types/document.js +2 -0
- package/dist/types/document.js.map +1 -0
- package/dist/types/errors.d.ts +32 -0
- package/dist/types/errors.d.ts.map +1 -0
- package/dist/types/errors.js +48 -0
- package/dist/types/errors.js.map +1 -0
- package/dist/types/index.d.ts +6 -0
- package/dist/types/index.d.ts.map +1 -0
- package/dist/types/index.js +3 -0
- package/dist/types/index.js.map +1 -0
- package/dist/types/query.d.ts +37 -0
- package/dist/types/query.d.ts.map +1 -0
- package/dist/types/query.js +2 -0
- package/dist/types/query.js.map +1 -0
- package/dist/types/storage.d.ts +45 -0
- package/dist/types/storage.d.ts.map +1 -0
- package/dist/types/storage.js +2 -0
- package/dist/types/storage.js.map +1 -0
- package/dist/utils/hash.d.ts +13 -0
- package/dist/utils/hash.d.ts.map +1 -0
- package/dist/utils/hash.js +23 -0
- package/dist/utils/hash.js.map +1 -0
- package/dist/utils/path-validator.d.ts +46 -0
- package/dist/utils/path-validator.d.ts.map +1 -0
- package/dist/utils/path-validator.js +99 -0
- package/dist/utils/path-validator.js.map +1 -0
- package/llms.txt +46 -0
- package/package.json +14 -3
- package/src/cache/lru-cache.ts +0 -131
- package/src/core/bucket-db.ts +0 -47
- package/src/core/collection.ts +0 -278
- package/src/core/wal.ts +0 -106
- package/src/index/shard-manager.ts +0 -197
- package/src/index.ts +0 -37
- package/src/query/evaluator.ts +0 -143
- package/src/storage/ali-oss.d.ts +0 -43
- package/src/storage/fs-adapter.ts +0 -248
- package/src/storage/memory-adapter.ts +0 -105
- package/src/storage/oss-adapter.ts +0 -164
- package/src/storage/s3-adapter.ts +0 -200
- package/src/types/collection.ts +0 -43
- package/src/types/errors.ts +0 -49
- package/src/types/index.ts +0 -37
- package/src/types/query.ts +0 -50
- package/src/types/storage.ts +0 -46
- package/src/utils/hash.ts +0 -24
- package/src/utils/path-validator.ts +0 -119
- package/tests/core/bucket-db.test.d.ts +0 -2
- package/tests/core/bucket-db.test.d.ts.map +0 -1
- package/tests/core/bucket-db.test.js +0 -34
- package/tests/core/bucket-db.test.js.map +0 -1
- package/tests/core/bucket-db.test.ts +0 -50
- package/tests/core/collection.test.d.ts +0 -2
- package/tests/core/collection.test.d.ts.map +0 -1
- package/tests/core/collection.test.js +0 -148
- package/tests/core/collection.test.js.map +0 -1
- package/tests/core/collection.test.ts +0 -195
- package/tests/core/dbpath.test.ts +0 -52
- package/tests/index/shard-manager.test.d.ts +0 -2
- package/tests/index/shard-manager.test.d.ts.map +0 -1
- package/tests/index/shard-manager.test.js +0 -106
- package/tests/index/shard-manager.test.js.map +0 -1
- package/tests/index/shard-manager.test.ts +0 -129
- package/tests/query/evaluator.test.d.ts +0 -2
- package/tests/query/evaluator.test.d.ts.map +0 -1
- package/tests/query/evaluator.test.js +0 -87
- package/tests/query/evaluator.test.js.map +0 -1
- package/tests/query/evaluator.test.ts +0 -104
- package/tests/storage/fs-adapter.test.ts +0 -111
- package/tests/storage/memory-adapter.test.d.ts +0 -2
- package/tests/storage/memory-adapter.test.d.ts.map +0 -1
- package/tests/storage/memory-adapter.test.js +0 -65
- package/tests/storage/memory-adapter.test.js.map +0 -1
- package/tests/storage/memory-adapter.test.ts +0 -84
- package/tests/storage/oss-adapter.test.d.ts +0 -2
- package/tests/storage/oss-adapter.test.d.ts.map +0 -1
- package/tests/storage/oss-adapter.test.js +0 -34
- package/tests/storage/oss-adapter.test.js.map +0 -1
- package/tests/storage/oss-adapter.test.ts +0 -40
- package/tests/storage/s3-adapter.test.d.ts +0 -2
- package/tests/storage/s3-adapter.test.d.ts.map +0 -1
- package/tests/storage/s3-adapter.test.js +0 -51
- package/tests/storage/s3-adapter.test.js.map +0 -1
- package/tests/storage/s3-adapter.test.ts +0 -62
- package/tests/utils/hash.test.d.ts +0 -2
- package/tests/utils/hash.test.d.ts.map +0 -1
- package/tests/utils/hash.test.js +0 -36
- package/tests/utils/hash.test.js.map +0 -1
- package/tests/utils/hash.test.ts +0 -42
- package/tsconfig.json +0 -12
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
import { join, resolve, relative } from 'path';
|
|
2
|
+
/**
|
|
3
|
+
* ValidationError for path validation failures
|
|
4
|
+
*/
|
|
5
|
+
export class PathValidationError extends Error {
|
|
6
|
+
constructor(message) {
|
|
7
|
+
super(message);
|
|
8
|
+
this.name = 'PathValidationError';
|
|
9
|
+
}
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* Sanitize and validate path components to prevent directory traversal attacks
|
|
13
|
+
* This is for SINGLE path components (like collection names, document IDs)
|
|
14
|
+
*
|
|
15
|
+
* @param input - Path component to sanitize
|
|
16
|
+
* @returns Sanitized path component
|
|
17
|
+
* @throws PathValidationError if validation fails
|
|
18
|
+
*/
|
|
19
|
+
export function sanitizePathComponent(input) {
|
|
20
|
+
if (!input || typeof input !== 'string') {
|
|
21
|
+
throw new PathValidationError('Path component must be a non-empty string');
|
|
22
|
+
}
|
|
23
|
+
// Remove any path traversal sequences
|
|
24
|
+
let sanitized = input.replace(/\.\./g, '');
|
|
25
|
+
// Remove directory separators (single components cannot have slashes)
|
|
26
|
+
sanitized = sanitized.replace(/[\/\\]/g, '_');
|
|
27
|
+
// Only allow alphanumeric, underscore, hyphen, and dot
|
|
28
|
+
if (!/^[a-zA-Z0-9_.-]+$/.test(sanitized)) {
|
|
29
|
+
throw new PathValidationError(`Invalid characters in path component. Only alphanumeric, underscore, hyphen, and dot are allowed.`);
|
|
30
|
+
}
|
|
31
|
+
// Prevent excessively long names
|
|
32
|
+
if (sanitized.length > 255) {
|
|
33
|
+
throw new PathValidationError('Path component exceeds maximum length of 255 characters');
|
|
34
|
+
}
|
|
35
|
+
// Prevent hidden files (starting with .)
|
|
36
|
+
if (sanitized.startsWith('.')) {
|
|
37
|
+
throw new PathValidationError('Path component cannot start with a dot');
|
|
38
|
+
}
|
|
39
|
+
return sanitized;
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* Validate that a target path is within the allowed base path
|
|
43
|
+
* Prevents directory traversal attacks by ensuring the resolved path
|
|
44
|
+
* doesn't escape the base directory
|
|
45
|
+
*
|
|
46
|
+
* @param basePath - Base directory path
|
|
47
|
+
* @param targetPath - Target path to validate
|
|
48
|
+
* @throws PathValidationError if path escapes base directory
|
|
49
|
+
*/
|
|
50
|
+
export function validatePathWithinBase(basePath, targetPath) {
|
|
51
|
+
const normalizedBase = resolve(basePath);
|
|
52
|
+
const normalizedTarget = resolve(targetPath);
|
|
53
|
+
const relativePath = relative(normalizedBase, normalizedTarget);
|
|
54
|
+
// If relative path starts with '..', it's outside the base directory
|
|
55
|
+
if (relativePath.startsWith('..') || resolve(normalizedBase, relativePath) !== normalizedTarget) {
|
|
56
|
+
throw new PathValidationError('Path traversal detected: target path is outside base directory');
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* Safely join path components with validation
|
|
61
|
+
* This is for joining SINGLE components (not full paths with slashes)
|
|
62
|
+
*
|
|
63
|
+
* @param basePath - Base directory path
|
|
64
|
+
* @param components - Path components to join
|
|
65
|
+
* @returns Validated full path
|
|
66
|
+
* @throws PathValidationError if validation fails
|
|
67
|
+
*/
|
|
68
|
+
export function safePathJoin(basePath, ...components) {
|
|
69
|
+
// Sanitize each component
|
|
70
|
+
const sanitizedComponents = components.map(sanitizePathComponent);
|
|
71
|
+
// Join paths
|
|
72
|
+
const fullPath = join(basePath, ...sanitizedComponents);
|
|
73
|
+
// Validate the result doesn't escape base path
|
|
74
|
+
validatePathWithinBase(basePath, fullPath);
|
|
75
|
+
return fullPath;
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* Safely join a key (which may contain slashes) with base path
|
|
79
|
+
* Validates the result doesn't escape base directory
|
|
80
|
+
*
|
|
81
|
+
* @param basePath - Base directory path
|
|
82
|
+
* @param key - Storage key (may contain path separators)
|
|
83
|
+
* @returns Validated full path
|
|
84
|
+
* @throws PathValidationError if validation fails
|
|
85
|
+
*/
|
|
86
|
+
export function safeKeyPath(basePath, key) {
|
|
87
|
+
if (!key || typeof key !== 'string') {
|
|
88
|
+
throw new PathValidationError('Key must be a non-empty string');
|
|
89
|
+
}
|
|
90
|
+
// Check for path traversal patterns
|
|
91
|
+
if (key.includes('..')) {
|
|
92
|
+
throw new PathValidationError('Key contains path traversal sequence');
|
|
93
|
+
}
|
|
94
|
+
// Join and validate
|
|
95
|
+
const fullPath = join(basePath, key);
|
|
96
|
+
validatePathWithinBase(basePath, fullPath);
|
|
97
|
+
return fullPath;
|
|
98
|
+
}
|
|
99
|
+
//# sourceMappingURL=path-validator.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"path-validator.js","sourceRoot":"","sources":["../../src/utils/path-validator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,MAAM,CAAC;AAE/C;;GAEG;AACH,MAAM,OAAO,mBAAoB,SAAQ,KAAK;IAC5C,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,qBAAqB,CAAC;IACpC,CAAC;CACF;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,qBAAqB,CAAC,KAAa;IACjD,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QACxC,MAAM,IAAI,mBAAmB,CAAC,2CAA2C,CAAC,CAAC;IAC7E,CAAC;IAED,sCAAsC;IACtC,IAAI,SAAS,GAAG,KAAK,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;IAE3C,sEAAsE;IACtE,SAAS,GAAG,SAAS,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;IAE9C,uDAAuD;IACvD,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;QACzC,MAAM,IAAI,mBAAmB,CAC3B,mGAAmG,CACpG,CAAC;IACJ,CAAC;IAED,iCAAiC;IACjC,IAAI,SAAS,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;QAC3B,MAAM,IAAI,mBAAmB,CAAC,yDAAyD,CAAC,CAAC;IAC3F,CAAC;IAED,yCAAyC;IACzC,IAAI,SAAS,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QAC9B,MAAM,IAAI,mBAAmB,CAAC,wCAAwC,CAAC,CAAC;IAC1E,CAAC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,sBAAsB,CAAC,QAAgB,EAAE,UAAkB;IACzE,MAAM,cAAc,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;IACzC,MAAM,gBAAgB,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;IAE7C,MAAM,YAAY,GAAG,QAAQ,CAAC,cAAc,EAAE,gBAAgB,CAAC,CAAC;IAEhE,qEAAqE;IACrE,IAAI,YAAY,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,OAAO,CAAC,cAAc,EAAE,YAAY,CAAC,KAAK,gBAAgB,EAAE,CAAC;QAChG,MAAM,IAAI,mBAAmB,CAAC,gEAAgE,CAAC,CAAC;IAClG,CAAC;AACH,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,YAAY,CAAC,QAAgB,EAAE,GAAG,UAAoB;IACpE,0BAA0B;IAC1B,MAAM,mBAAmB,GAAG,UAAU,CAAC,GAAG,CAAC,qBAAqB,CAAC,CAAC;IAElE,aAAa;IACb,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,EAAE,GAAG,mBAAmB,CAAC,CAAC;IAExD,+CAA+C;IAC/C,sBAAsB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;IAE3C,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,WAAW,CAAC,QAAgB,EAAE,GAAW;IACvD,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;QACpC,MAAM,IAAI,mBAAmB,CAAC,gCAAgC,CAAC,CAAC;IAClE,CAAC;IAED,oCAAoC;IACpC,IAAI,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QACvB,MAAM,IAAI,mBAAmB,CAAC,sCAAsC,CAAC,CAAC;IACxE,CAAC;IAED,oBAAoB;IACpB,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;IACrC,sBAAsB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;IAE3C,OAAO,QAAQ,CAAC;AAClB,CAAC"}
|
package/llms.txt
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
# BucketDB
|
|
2
|
+
|
|
3
|
+
TypeScript document database. Uses S3 / Alibaba Cloud OSS / local filesystem as backend. No database server required.
|
|
4
|
+
|
|
5
|
+
## Key concepts
|
|
6
|
+
|
|
7
|
+
- `BucketDB` — main class, takes a storage adapter and a `dbPath` namespace
|
|
8
|
+
- `Collection<T>` — typed collection, returned by `db.collection<T>(name)`
|
|
9
|
+
- Every document extends `Document` which adds: `id` (UUID), `_etag`, `_createdAt`, `_updatedAt`
|
|
10
|
+
- `InsertDocument<T>` omits system fields (pass only your own fields)
|
|
11
|
+
- Optimistic locking: pass `{ etag: doc._etag }` to `update()` to prevent concurrent overwrites
|
|
12
|
+
|
|
13
|
+
## Package: @hold-baby/bucket-db
|
|
14
|
+
|
|
15
|
+
### Exports
|
|
16
|
+
|
|
17
|
+
Classes: BucketDB, Collection, MemoryStorageAdapter, FileSystemAdapter, S3Adapter, OSSAdapter
|
|
18
|
+
Errors: BucketDBError, DocumentNotFoundError, ConcurrentUpdateError, ValidationError, StorageError
|
|
19
|
+
Types: Document, InsertDocument, UpdateDocument, QueryFilter, QueryOptions, UpdateOptions, StorageAdapter
|
|
20
|
+
|
|
21
|
+
### Collection API
|
|
22
|
+
|
|
23
|
+
```
|
|
24
|
+
insert(data: InsertDocument<T>): Promise<T>
|
|
25
|
+
findById(id: string): Promise<T> // throws DocumentNotFoundError if missing
|
|
26
|
+
find(filter: QueryFilter<T>, options?: QueryOptions): Promise<T[]>
|
|
27
|
+
update(id: string, data: UpdateDocument<T>, options?: UpdateOptions): Promise<T>
|
|
28
|
+
delete(id: string, options?: UpdateOptions): Promise<void>
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
### QueryOptions
|
|
32
|
+
|
|
33
|
+
```
|
|
34
|
+
{ limit?: number, offset?: number }
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
### Query operators
|
|
38
|
+
|
|
39
|
+
$eq, $ne, $gt, $gte, $lt, $lte, $in, $nin
|
|
40
|
+
|
|
41
|
+
### Error mapping
|
|
42
|
+
|
|
43
|
+
DocumentNotFoundError — findById / update / delete on missing doc
|
|
44
|
+
ConcurrentUpdateError — update / delete with stale etag
|
|
45
|
+
ValidationError — invalid input
|
|
46
|
+
StorageError — underlying I/O failure
|
package/package.json
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@hold-baby/bucket-db",
|
|
3
|
-
"version": "0.3.
|
|
3
|
+
"version": "0.3.1",
|
|
4
|
+
"description": "A TypeScript document database built on cloud object storage (AWS S3 / Alibaba Cloud OSS) or local file system. No database servers needed.",
|
|
4
5
|
"type": "module",
|
|
5
6
|
"main": "./dist/index.js",
|
|
6
7
|
"types": "./dist/index.d.ts",
|
|
@@ -25,8 +26,14 @@
|
|
|
25
26
|
"typescript": "^5.3.3"
|
|
26
27
|
},
|
|
27
28
|
"publishConfig": {
|
|
28
|
-
"access": "public"
|
|
29
|
+
"access": "public",
|
|
30
|
+
"registry": "https://registry.npmjs.org/"
|
|
29
31
|
},
|
|
32
|
+
"files": [
|
|
33
|
+
"dist",
|
|
34
|
+
"README.md",
|
|
35
|
+
"llms.txt"
|
|
36
|
+
],
|
|
30
37
|
"keywords": [
|
|
31
38
|
"database",
|
|
32
39
|
"document-db",
|
|
@@ -39,7 +46,11 @@
|
|
|
39
46
|
"license": "MIT",
|
|
40
47
|
"repository": {
|
|
41
48
|
"type": "git",
|
|
42
|
-
"url": "https://github.com/holdbaby5483/bucket-db.git",
|
|
49
|
+
"url": "git+https://github.com/holdbaby5483/bucket-db.git",
|
|
43
50
|
"directory": "packages/core"
|
|
51
|
+
},
|
|
52
|
+
"homepage": "https://github.com/holdbaby5483/bucket-db#readme",
|
|
53
|
+
"bugs": {
|
|
54
|
+
"url": "https://github.com/holdbaby5483/bucket-db/issues"
|
|
44
55
|
}
|
|
45
56
|
}
|
package/src/cache/lru-cache.ts
DELETED
|
@@ -1,131 +0,0 @@
|
|
|
1
|
-
import { createHash } from 'crypto';
|
|
2
|
-
|
|
3
|
-
interface CacheEntry<T> {
|
|
4
|
-
value: T;
|
|
5
|
-
timestamp: number;
|
|
6
|
-
}
|
|
7
|
-
|
|
8
|
-
/**
|
|
9
|
-
* LRU (Least Recently Used) Cache implementation
|
|
10
|
-
*/
|
|
11
|
-
export class LRUCache<T> {
|
|
12
|
-
private cache: Map<string, CacheEntry<T>>;
|
|
13
|
-
private maxSize: number;
|
|
14
|
-
private ttl: number; // Time To Live in milliseconds
|
|
15
|
-
|
|
16
|
-
// Statistics
|
|
17
|
-
private hits: number = 0;
|
|
18
|
-
private misses: number = 0;
|
|
19
|
-
private evictions: number = 0;
|
|
20
|
-
|
|
21
|
-
constructor(maxSize: number = 100, ttl: number = 5 * 60 * 1000) {
|
|
22
|
-
this.cache = new Map();
|
|
23
|
-
this.maxSize = maxSize;
|
|
24
|
-
this.ttl = ttl;
|
|
25
|
-
}
|
|
26
|
-
|
|
27
|
-
/**
|
|
28
|
-
* Generate cache key from query filter and options
|
|
29
|
-
*/
|
|
30
|
-
static generateKey(filter: Record<string, any>, options?: Record<string, any>): string {
|
|
31
|
-
const data = JSON.stringify({ filter, options });
|
|
32
|
-
return createHash('md5').update(data).digest('hex');
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
/**
|
|
36
|
-
* Get value from cache
|
|
37
|
-
*/
|
|
38
|
-
get(key: string): T | null {
|
|
39
|
-
const entry = this.cache.get(key);
|
|
40
|
-
|
|
41
|
-
if (!entry) {
|
|
42
|
-
this.misses++;
|
|
43
|
-
return null;
|
|
44
|
-
}
|
|
45
|
-
|
|
46
|
-
// Check if entry has expired
|
|
47
|
-
if (Date.now() - entry.timestamp > this.ttl) {
|
|
48
|
-
this.cache.delete(key);
|
|
49
|
-
this.misses++;
|
|
50
|
-
return null;
|
|
51
|
-
}
|
|
52
|
-
|
|
53
|
-
// Move to end (mark as recently used)
|
|
54
|
-
this.cache.delete(key);
|
|
55
|
-
this.cache.set(key, entry);
|
|
56
|
-
this.hits++;
|
|
57
|
-
|
|
58
|
-
return entry.value;
|
|
59
|
-
}
|
|
60
|
-
|
|
61
|
-
/**
|
|
62
|
-
* Put value into cache
|
|
63
|
-
*/
|
|
64
|
-
set(key: string, value: T): void {
|
|
65
|
-
// Remove if already exists (to update position)
|
|
66
|
-
if (this.cache.has(key)) {
|
|
67
|
-
this.cache.delete(key);
|
|
68
|
-
}
|
|
69
|
-
|
|
70
|
-
// Evict oldest entry if cache is full
|
|
71
|
-
if (this.cache.size >= this.maxSize) {
|
|
72
|
-
const firstKey = this.cache.keys().next().value;
|
|
73
|
-
if (firstKey) {
|
|
74
|
-
this.cache.delete(firstKey);
|
|
75
|
-
this.evictions++;
|
|
76
|
-
}
|
|
77
|
-
}
|
|
78
|
-
|
|
79
|
-
this.cache.set(key, {
|
|
80
|
-
value,
|
|
81
|
-
timestamp: Date.now(),
|
|
82
|
-
});
|
|
83
|
-
}
|
|
84
|
-
|
|
85
|
-
/**
|
|
86
|
-
* Invalidate cache entries (clear all by default)
|
|
87
|
-
*/
|
|
88
|
-
invalidate(pattern?: RegExp): void {
|
|
89
|
-
if (!pattern) {
|
|
90
|
-
this.cache.clear();
|
|
91
|
-
return;
|
|
92
|
-
}
|
|
93
|
-
|
|
94
|
-
// Remove entries matching pattern
|
|
95
|
-
for (const key of this.cache.keys()) {
|
|
96
|
-
if (pattern.test(key)) {
|
|
97
|
-
this.cache.delete(key);
|
|
98
|
-
}
|
|
99
|
-
}
|
|
100
|
-
}
|
|
101
|
-
|
|
102
|
-
/**
|
|
103
|
-
* Get cache statistics
|
|
104
|
-
*/
|
|
105
|
-
getStats() {
|
|
106
|
-
const total = this.hits + this.misses;
|
|
107
|
-
return {
|
|
108
|
-
size: this.cache.size,
|
|
109
|
-
hits: this.hits,
|
|
110
|
-
misses: this.misses,
|
|
111
|
-
evictions: this.evictions,
|
|
112
|
-
hitRate: total > 0 ? (this.hits / total * 100).toFixed(2) + '%' : '0%',
|
|
113
|
-
};
|
|
114
|
-
}
|
|
115
|
-
|
|
116
|
-
/**
|
|
117
|
-
* Clear all cache statistics
|
|
118
|
-
*/
|
|
119
|
-
resetStats(): void {
|
|
120
|
-
this.hits = 0;
|
|
121
|
-
this.misses = 0;
|
|
122
|
-
this.evictions = 0;
|
|
123
|
-
}
|
|
124
|
-
|
|
125
|
-
/**
|
|
126
|
-
* Clear the entire cache
|
|
127
|
-
*/
|
|
128
|
-
clear(): void {
|
|
129
|
-
this.cache.clear();
|
|
130
|
-
}
|
|
131
|
-
}
|
package/src/core/bucket-db.ts
DELETED
|
@@ -1,47 +0,0 @@
|
|
|
1
|
-
import type { StorageAdapter, Document } from '../types/index.js';
|
|
2
|
-
import { Collection, CollectionOptions } from './collection.js';
|
|
3
|
-
import { sanitizePathComponent } from '../utils/path-validator.js';
|
|
4
|
-
|
|
5
|
-
export interface BucketDBOptions {
|
|
6
|
-
defaultShardCount?: number;
|
|
7
|
-
}
|
|
8
|
-
|
|
9
|
-
/**
|
|
10
|
-
* Main BucketDB class
|
|
11
|
-
*/
|
|
12
|
-
export class BucketDB {
|
|
13
|
-
private collections = new Map<string, Collection<any>>();
|
|
14
|
-
|
|
15
|
-
constructor(
|
|
16
|
-
private adapter: StorageAdapter,
|
|
17
|
-
private dbPath: string,
|
|
18
|
-
private options: BucketDBOptions = {}
|
|
19
|
-
) {}
|
|
20
|
-
|
|
21
|
-
/**
|
|
22
|
-
* Get or create a collection
|
|
23
|
-
*/
|
|
24
|
-
collection<T extends Document>(
|
|
25
|
-
name: string,
|
|
26
|
-
options?: CollectionOptions
|
|
27
|
-
): Collection<T> {
|
|
28
|
-
// Validate collection name to prevent path traversal
|
|
29
|
-
sanitizePathComponent(name);
|
|
30
|
-
|
|
31
|
-
if (this.collections.has(name)) {
|
|
32
|
-
return this.collections.get(name)!;
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
const collection = new Collection<T>(
|
|
36
|
-
this.adapter,
|
|
37
|
-
this.dbPath,
|
|
38
|
-
name,
|
|
39
|
-
{
|
|
40
|
-
shardCount: options?.shardCount || this.options.defaultShardCount || 16,
|
|
41
|
-
}
|
|
42
|
-
);
|
|
43
|
-
|
|
44
|
-
this.collections.set(name, collection);
|
|
45
|
-
return collection;
|
|
46
|
-
}
|
|
47
|
-
}
|
package/src/core/collection.ts
DELETED
|
@@ -1,278 +0,0 @@
|
|
|
1
|
-
import type {
|
|
2
|
-
ICollection,
|
|
3
|
-
Document,
|
|
4
|
-
InsertDocument,
|
|
5
|
-
UpdateDocument,
|
|
6
|
-
QueryFilter,
|
|
7
|
-
QueryOptions,
|
|
8
|
-
UpdateOptions,
|
|
9
|
-
StorageAdapter,
|
|
10
|
-
} from '../types/index.js';
|
|
11
|
-
import { DocumentNotFoundError, ConcurrentUpdateError } from '../types/index.js';
|
|
12
|
-
import { ShardManager } from '../index/shard-manager.js';
|
|
13
|
-
import { sanitizePathComponent, safePathJoin } from '../utils/path-validator.js';
|
|
14
|
-
import { WAL, type WALEntry } from './wal.js';
|
|
15
|
-
import { LRUCache } from '../cache/lru-cache.js';
|
|
16
|
-
|
|
17
|
-
/**
|
|
18
|
-
* Generate UUID v4
|
|
19
|
-
*/
|
|
20
|
-
function generateId(): string {
|
|
21
|
-
return crypto.randomUUID();
|
|
22
|
-
}
|
|
23
|
-
|
|
24
|
-
/**
|
|
25
|
-
* Get current ISO timestamp
|
|
26
|
-
*/
|
|
27
|
-
function now(): string {
|
|
28
|
-
return new Date().toISOString();
|
|
29
|
-
}
|
|
30
|
-
|
|
31
|
-
export interface CollectionOptions {
|
|
32
|
-
shardCount?: number;
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
/**
|
|
36
|
-
* Collection class for type-safe CRUD operations
|
|
37
|
-
*/
|
|
38
|
-
export class Collection<T extends Document> implements ICollection<T> {
|
|
39
|
-
private shardManager: ShardManager;
|
|
40
|
-
private wal: WAL;
|
|
41
|
-
private queryCache: LRUCache<T[]>;
|
|
42
|
-
private basePath: string;
|
|
43
|
-
|
|
44
|
-
constructor(
|
|
45
|
-
private adapter: StorageAdapter,
|
|
46
|
-
dbPath: string,
|
|
47
|
-
collectionName: string,
|
|
48
|
-
options: CollectionOptions = {}
|
|
49
|
-
) {
|
|
50
|
-
// Validate collection name to prevent path traversal
|
|
51
|
-
const sanitizedCollectionName = sanitizePathComponent(collectionName);
|
|
52
|
-
this.basePath = safePathJoin(dbPath, sanitizedCollectionName);
|
|
53
|
-
this.shardManager = new ShardManager(
|
|
54
|
-
adapter,
|
|
55
|
-
this.basePath,
|
|
56
|
-
options.shardCount || 16
|
|
57
|
-
);
|
|
58
|
-
this.wal = new WAL(adapter, this.basePath);
|
|
59
|
-
this.queryCache = new LRUCache<T[]>(100, 5 * 60 * 1000); // 100 entries, 5 min TTL
|
|
60
|
-
|
|
61
|
-
// Recover from any pending WAL entries
|
|
62
|
-
this.recoverFromWAL().catch(err => {
|
|
63
|
-
console.error('WAL recovery failed:', err);
|
|
64
|
-
});
|
|
65
|
-
}
|
|
66
|
-
|
|
67
|
-
/**
|
|
68
|
-
* Recover from pending WAL entries
|
|
69
|
-
*/
|
|
70
|
-
private async recoverFromWAL(): Promise<void> {
|
|
71
|
-
await this.wal.recover(async (entry: WALEntry) => {
|
|
72
|
-
// Replay the operation based on WAL entry
|
|
73
|
-
switch (entry.operation) {
|
|
74
|
-
case 'insert':
|
|
75
|
-
case 'update':
|
|
76
|
-
// Ensure document is indexed
|
|
77
|
-
if (entry.documentData) {
|
|
78
|
-
await this.shardManager.addDocument(entry.documentData);
|
|
79
|
-
}
|
|
80
|
-
break;
|
|
81
|
-
case 'delete':
|
|
82
|
-
// Ensure document is removed from index
|
|
83
|
-
await this.shardManager.removeDocument(entry.documentId);
|
|
84
|
-
break;
|
|
85
|
-
}
|
|
86
|
-
});
|
|
87
|
-
}
|
|
88
|
-
|
|
89
|
-
private getDocPath(id: string): string {
|
|
90
|
-
// Validate document ID to prevent path traversal
|
|
91
|
-
const sanitizedId = sanitizePathComponent(id);
|
|
92
|
-
return safePathJoin(this.basePath, 'docs', `${sanitizedId}.json`);
|
|
93
|
-
}
|
|
94
|
-
|
|
95
|
-
async insert(data: InsertDocument<T>): Promise<T> {
|
|
96
|
-
const id = generateId();
|
|
97
|
-
const timestamp = now();
|
|
98
|
-
|
|
99
|
-
const document: T = {
|
|
100
|
-
...data,
|
|
101
|
-
id,
|
|
102
|
-
_createdAt: timestamp,
|
|
103
|
-
_updatedAt: timestamp,
|
|
104
|
-
} as T;
|
|
105
|
-
|
|
106
|
-
// 1. Log operation to WAL first
|
|
107
|
-
await this.wal.log({
|
|
108
|
-
operation: 'insert',
|
|
109
|
-
collectionPath: this.basePath,
|
|
110
|
-
documentId: id,
|
|
111
|
-
documentData: document,
|
|
112
|
-
timestamp,
|
|
113
|
-
});
|
|
114
|
-
|
|
115
|
-
try {
|
|
116
|
-
// 2. Write document to storage
|
|
117
|
-
const { etag } = await this.adapter.put(this.getDocPath(id), document);
|
|
118
|
-
document._etag = etag;
|
|
119
|
-
|
|
120
|
-
// 3. Add to index
|
|
121
|
-
await this.shardManager.addDocument(document);
|
|
122
|
-
|
|
123
|
-
// 4. Clear WAL entry after successful completion
|
|
124
|
-
await this.wal.clear(id);
|
|
125
|
-
|
|
126
|
-
// 5. Invalidate query cache (new document may match existing queries)
|
|
127
|
-
this.queryCache.invalidate();
|
|
128
|
-
|
|
129
|
-
return document;
|
|
130
|
-
} catch (error) {
|
|
131
|
-
// On failure, WAL entry remains for recovery
|
|
132
|
-
throw error;
|
|
133
|
-
}
|
|
134
|
-
}
|
|
135
|
-
|
|
136
|
-
async findById(id: string): Promise<T | null> {
|
|
137
|
-
try {
|
|
138
|
-
const { data, etag } = await this.adapter.get(this.getDocPath(id));
|
|
139
|
-
const doc = data as T;
|
|
140
|
-
doc._etag = etag;
|
|
141
|
-
return doc;
|
|
142
|
-
} catch (error) {
|
|
143
|
-
return null;
|
|
144
|
-
}
|
|
145
|
-
}
|
|
146
|
-
|
|
147
|
-
async find(filter: QueryFilter<T>, options?: QueryOptions): Promise<T[]> {
|
|
148
|
-
// Check cache first
|
|
149
|
-
const cacheKey = LRUCache.generateKey(filter as Record<string, any>, options as Record<string, any>);
|
|
150
|
-
const cached = this.queryCache.get(cacheKey);
|
|
151
|
-
if (cached) {
|
|
152
|
-
return cached;
|
|
153
|
-
}
|
|
154
|
-
|
|
155
|
-
// Query index to get matching document IDs
|
|
156
|
-
const indexResults = await this.shardManager.query(filter);
|
|
157
|
-
|
|
158
|
-
// Get full documents
|
|
159
|
-
let ids = indexResults.map(doc => doc.id);
|
|
160
|
-
|
|
161
|
-
if (ids.length === 0) {
|
|
162
|
-
return [];
|
|
163
|
-
}
|
|
164
|
-
|
|
165
|
-
// Apply pagination on IDs before loading documents (PERF optimization)
|
|
166
|
-
if (options?.offset) {
|
|
167
|
-
ids = ids.slice(options.offset);
|
|
168
|
-
}
|
|
169
|
-
if (options?.limit) {
|
|
170
|
-
ids = ids.slice(0, options.limit);
|
|
171
|
-
}
|
|
172
|
-
|
|
173
|
-
// Batch get only the required documents
|
|
174
|
-
const docMap = await this.adapter.batchGet(ids.map(id => this.getDocPath(id)));
|
|
175
|
-
|
|
176
|
-
// Convert to array and add etags
|
|
177
|
-
const results = Array.from(docMap.values()).map(obj => {
|
|
178
|
-
const doc = obj.data as T;
|
|
179
|
-
doc._etag = obj.etag;
|
|
180
|
-
return doc;
|
|
181
|
-
});
|
|
182
|
-
|
|
183
|
-
// Cache the results
|
|
184
|
-
this.queryCache.set(cacheKey, results);
|
|
185
|
-
|
|
186
|
-
return results;
|
|
187
|
-
}
|
|
188
|
-
|
|
189
|
-
async update(
|
|
190
|
-
id: string,
|
|
191
|
-
data: UpdateDocument<T>,
|
|
192
|
-
options?: UpdateOptions
|
|
193
|
-
): Promise<T> {
|
|
194
|
-
// Get current document
|
|
195
|
-
const current = await this.findById(id);
|
|
196
|
-
if (!current) {
|
|
197
|
-
throw new DocumentNotFoundError(id);
|
|
198
|
-
}
|
|
199
|
-
|
|
200
|
-
// Check optimistic lock if etag provided
|
|
201
|
-
if (options?.etag && current._etag !== options.etag) {
|
|
202
|
-
throw new ConcurrentUpdateError(`ETag mismatch for document ${id}`);
|
|
203
|
-
}
|
|
204
|
-
|
|
205
|
-
// Merge updates
|
|
206
|
-
const updated: T = {
|
|
207
|
-
...current,
|
|
208
|
-
...data,
|
|
209
|
-
id,
|
|
210
|
-
_updatedAt: now(),
|
|
211
|
-
} as T;
|
|
212
|
-
|
|
213
|
-
// 1. Log operation to WAL first
|
|
214
|
-
await this.wal.log({
|
|
215
|
-
operation: 'update',
|
|
216
|
-
collectionPath: this.basePath,
|
|
217
|
-
documentId: id,
|
|
218
|
-
documentData: updated,
|
|
219
|
-
timestamp: now(),
|
|
220
|
-
});
|
|
221
|
-
|
|
222
|
-
try {
|
|
223
|
-
// 2. Write document
|
|
224
|
-
const { etag } = await this.adapter.put(
|
|
225
|
-
this.getDocPath(id),
|
|
226
|
-
updated,
|
|
227
|
-
current._etag ? { ifMatch: current._etag } : undefined
|
|
228
|
-
);
|
|
229
|
-
|
|
230
|
-
updated._etag = etag;
|
|
231
|
-
|
|
232
|
-
// 3. Update index
|
|
233
|
-
await this.shardManager.updateDocument(updated);
|
|
234
|
-
|
|
235
|
-
// 4. Clear WAL entry
|
|
236
|
-
await this.wal.clear(id);
|
|
237
|
-
|
|
238
|
-
// 5. Invalidate query cache (document changed)
|
|
239
|
-
this.queryCache.invalidate();
|
|
240
|
-
|
|
241
|
-
return updated;
|
|
242
|
-
} catch (error) {
|
|
243
|
-
throw error;
|
|
244
|
-
}
|
|
245
|
-
}
|
|
246
|
-
|
|
247
|
-
async delete(id: string): Promise<void> {
|
|
248
|
-
// Check if document exists
|
|
249
|
-
const exists = await this.findById(id);
|
|
250
|
-
if (!exists) {
|
|
251
|
-
throw new DocumentNotFoundError(id);
|
|
252
|
-
}
|
|
253
|
-
|
|
254
|
-
// 1. Log operation to WAL first
|
|
255
|
-
await this.wal.log({
|
|
256
|
-
operation: 'delete',
|
|
257
|
-
collectionPath: this.basePath,
|
|
258
|
-
documentId: id,
|
|
259
|
-
timestamp: now(),
|
|
260
|
-
});
|
|
261
|
-
|
|
262
|
-
try {
|
|
263
|
-
// 2. Delete from storage
|
|
264
|
-
await this.adapter.delete(this.getDocPath(id));
|
|
265
|
-
|
|
266
|
-
// 3. Remove from index
|
|
267
|
-
await this.shardManager.removeDocument(id);
|
|
268
|
-
|
|
269
|
-
// 4. Clear WAL entry
|
|
270
|
-
await this.wal.clear(id);
|
|
271
|
-
|
|
272
|
-
// 5. Invalidate query cache (document deleted)
|
|
273
|
-
this.queryCache.invalidate();
|
|
274
|
-
} catch (error) {
|
|
275
|
-
throw error;
|
|
276
|
-
}
|
|
277
|
-
}
|
|
278
|
-
}
|