@hold-baby/bucket-db 0.3.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +503 -19
- package/dist/bucket-db.d.ts +25 -0
- package/dist/bucket-db.js +36 -0
- package/dist/cli.d.ts +2 -0
- package/dist/cli.js +133 -0
- package/dist/collection.d.ts +78 -0
- package/dist/collection.js +321 -0
- package/dist/doctor.d.ts +7 -0
- package/dist/doctor.js +100 -0
- package/dist/errors.d.ts +59 -0
- package/dist/errors.js +57 -0
- package/dist/index.d.ts +13 -0
- package/dist/index.js +6 -0
- package/dist/keys.d.ts +12 -0
- package/dist/keys.js +31 -0
- package/dist/query.d.ts +16 -0
- package/dist/query.js +133 -0
- package/dist/snapshot.d.ts +16 -0
- package/dist/snapshot.js +40 -0
- package/dist/storage/adapter.d.ts +75 -0
- package/dist/storage/adapter.js +27 -0
- package/dist/storage/fs.d.ts +28 -0
- package/dist/storage/fs.js +175 -0
- package/dist/storage/memory.d.ts +18 -0
- package/dist/storage/memory.js +49 -0
- package/dist/storage/s3/adapters.d.ts +38 -0
- package/dist/storage/s3/adapters.js +65 -0
- package/dist/storage/s3/client.d.ts +52 -0
- package/dist/storage/s3/client.js +231 -0
- package/dist/storage/s3/credentials.d.ts +17 -0
- package/dist/storage/s3/credentials.js +42 -0
- package/dist/storage/s3/sigv4.d.ts +32 -0
- package/dist/storage/s3/sigv4.js +75 -0
- package/dist/types.d.ts +55 -0
- package/dist/types.js +1 -0
- package/dist/url.d.ts +16 -0
- package/dist/url.js +66 -0
- package/llms.txt +142 -0
- package/package.json +20 -7
- package/src/cache/lru-cache.ts +0 -131
- package/src/core/bucket-db.ts +0 -47
- package/src/core/collection.ts +0 -278
- package/src/core/wal.ts +0 -106
- package/src/index/shard-manager.ts +0 -197
- package/src/index.ts +0 -37
- package/src/query/evaluator.ts +0 -143
- package/src/storage/ali-oss.d.ts +0 -43
- package/src/storage/fs-adapter.ts +0 -248
- package/src/storage/memory-adapter.ts +0 -105
- package/src/storage/oss-adapter.ts +0 -164
- package/src/storage/s3-adapter.ts +0 -200
- package/src/types/collection.ts +0 -43
- package/src/types/document.ts +0 -22
- package/src/types/errors.ts +0 -49
- package/src/types/index.ts +0 -37
- package/src/types/query.ts +0 -50
- package/src/types/storage.ts +0 -46
- package/src/utils/hash.ts +0 -24
- package/src/utils/path-validator.ts +0 -119
- package/tests/core/bucket-db.test.d.ts +0 -2
- package/tests/core/bucket-db.test.d.ts.map +0 -1
- package/tests/core/bucket-db.test.js +0 -34
- package/tests/core/bucket-db.test.js.map +0 -1
- package/tests/core/bucket-db.test.ts +0 -50
- package/tests/core/collection.test.d.ts +0 -2
- package/tests/core/collection.test.d.ts.map +0 -1
- package/tests/core/collection.test.js +0 -148
- package/tests/core/collection.test.js.map +0 -1
- package/tests/core/collection.test.ts +0 -195
- package/tests/core/dbpath.test.ts +0 -52
- package/tests/index/shard-manager.test.d.ts +0 -2
- package/tests/index/shard-manager.test.d.ts.map +0 -1
- package/tests/index/shard-manager.test.js +0 -106
- package/tests/index/shard-manager.test.js.map +0 -1
- package/tests/index/shard-manager.test.ts +0 -129
- package/tests/query/evaluator.test.d.ts +0 -2
- package/tests/query/evaluator.test.d.ts.map +0 -1
- package/tests/query/evaluator.test.js +0 -87
- package/tests/query/evaluator.test.js.map +0 -1
- package/tests/query/evaluator.test.ts +0 -104
- package/tests/storage/fs-adapter.test.ts +0 -111
- package/tests/storage/memory-adapter.test.d.ts +0 -2
- package/tests/storage/memory-adapter.test.d.ts.map +0 -1
- package/tests/storage/memory-adapter.test.js +0 -65
- package/tests/storage/memory-adapter.test.js.map +0 -1
- package/tests/storage/memory-adapter.test.ts +0 -84
- package/tests/storage/oss-adapter.test.d.ts +0 -2
- package/tests/storage/oss-adapter.test.d.ts.map +0 -1
- package/tests/storage/oss-adapter.test.js +0 -34
- package/tests/storage/oss-adapter.test.js.map +0 -1
- package/tests/storage/oss-adapter.test.ts +0 -40
- package/tests/storage/s3-adapter.test.d.ts +0 -2
- package/tests/storage/s3-adapter.test.d.ts.map +0 -1
- package/tests/storage/s3-adapter.test.js +0 -51
- package/tests/storage/s3-adapter.test.js.map +0 -1
- package/tests/storage/s3-adapter.test.ts +0 -62
- package/tests/utils/hash.test.d.ts +0 -2
- package/tests/utils/hash.test.d.ts.map +0 -1
- package/tests/utils/hash.test.js +0 -36
- package/tests/utils/hash.test.js.map +0 -1
- package/tests/utils/hash.test.ts +0 -42
- package/tsconfig.json +0 -12
package/src/core/bucket-db.ts
DELETED
|
@@ -1,47 +0,0 @@
|
|
|
1
|
-
import type { StorageAdapter, Document } from '../types/index.js';
|
|
2
|
-
import { Collection, CollectionOptions } from './collection.js';
|
|
3
|
-
import { sanitizePathComponent } from '../utils/path-validator.js';
|
|
4
|
-
|
|
5
|
-
export interface BucketDBOptions {
|
|
6
|
-
defaultShardCount?: number;
|
|
7
|
-
}
|
|
8
|
-
|
|
9
|
-
/**
|
|
10
|
-
* Main BucketDB class
|
|
11
|
-
*/
|
|
12
|
-
export class BucketDB {
|
|
13
|
-
private collections = new Map<string, Collection<any>>();
|
|
14
|
-
|
|
15
|
-
constructor(
|
|
16
|
-
private adapter: StorageAdapter,
|
|
17
|
-
private dbPath: string,
|
|
18
|
-
private options: BucketDBOptions = {}
|
|
19
|
-
) {}
|
|
20
|
-
|
|
21
|
-
/**
|
|
22
|
-
* Get or create a collection
|
|
23
|
-
*/
|
|
24
|
-
collection<T extends Document>(
|
|
25
|
-
name: string,
|
|
26
|
-
options?: CollectionOptions
|
|
27
|
-
): Collection<T> {
|
|
28
|
-
// Validate collection name to prevent path traversal
|
|
29
|
-
sanitizePathComponent(name);
|
|
30
|
-
|
|
31
|
-
if (this.collections.has(name)) {
|
|
32
|
-
return this.collections.get(name)!;
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
const collection = new Collection<T>(
|
|
36
|
-
this.adapter,
|
|
37
|
-
this.dbPath,
|
|
38
|
-
name,
|
|
39
|
-
{
|
|
40
|
-
shardCount: options?.shardCount || this.options.defaultShardCount || 16,
|
|
41
|
-
}
|
|
42
|
-
);
|
|
43
|
-
|
|
44
|
-
this.collections.set(name, collection);
|
|
45
|
-
return collection;
|
|
46
|
-
}
|
|
47
|
-
}
|
package/src/core/collection.ts
DELETED
|
@@ -1,278 +0,0 @@
|
|
|
1
|
-
import type {
|
|
2
|
-
ICollection,
|
|
3
|
-
Document,
|
|
4
|
-
InsertDocument,
|
|
5
|
-
UpdateDocument,
|
|
6
|
-
QueryFilter,
|
|
7
|
-
QueryOptions,
|
|
8
|
-
UpdateOptions,
|
|
9
|
-
StorageAdapter,
|
|
10
|
-
} from '../types/index.js';
|
|
11
|
-
import { DocumentNotFoundError, ConcurrentUpdateError } from '../types/index.js';
|
|
12
|
-
import { ShardManager } from '../index/shard-manager.js';
|
|
13
|
-
import { sanitizePathComponent, safePathJoin } from '../utils/path-validator.js';
|
|
14
|
-
import { WAL, type WALEntry } from './wal.js';
|
|
15
|
-
import { LRUCache } from '../cache/lru-cache.js';
|
|
16
|
-
|
|
17
|
-
/**
|
|
18
|
-
* Generate UUID v4
|
|
19
|
-
*/
|
|
20
|
-
function generateId(): string {
|
|
21
|
-
return crypto.randomUUID();
|
|
22
|
-
}
|
|
23
|
-
|
|
24
|
-
/**
|
|
25
|
-
* Get current ISO timestamp
|
|
26
|
-
*/
|
|
27
|
-
function now(): string {
|
|
28
|
-
return new Date().toISOString();
|
|
29
|
-
}
|
|
30
|
-
|
|
31
|
-
export interface CollectionOptions {
|
|
32
|
-
shardCount?: number;
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
/**
|
|
36
|
-
* Collection class for type-safe CRUD operations
|
|
37
|
-
*/
|
|
38
|
-
export class Collection<T extends Document> implements ICollection<T> {
|
|
39
|
-
private shardManager: ShardManager;
|
|
40
|
-
private wal: WAL;
|
|
41
|
-
private queryCache: LRUCache<T[]>;
|
|
42
|
-
private basePath: string;
|
|
43
|
-
|
|
44
|
-
constructor(
|
|
45
|
-
private adapter: StorageAdapter,
|
|
46
|
-
dbPath: string,
|
|
47
|
-
collectionName: string,
|
|
48
|
-
options: CollectionOptions = {}
|
|
49
|
-
) {
|
|
50
|
-
// Validate collection name to prevent path traversal
|
|
51
|
-
const sanitizedCollectionName = sanitizePathComponent(collectionName);
|
|
52
|
-
this.basePath = safePathJoin(dbPath, sanitizedCollectionName);
|
|
53
|
-
this.shardManager = new ShardManager(
|
|
54
|
-
adapter,
|
|
55
|
-
this.basePath,
|
|
56
|
-
options.shardCount || 16
|
|
57
|
-
);
|
|
58
|
-
this.wal = new WAL(adapter, this.basePath);
|
|
59
|
-
this.queryCache = new LRUCache<T[]>(100, 5 * 60 * 1000); // 100 entries, 5 min TTL
|
|
60
|
-
|
|
61
|
-
// Recover from any pending WAL entries
|
|
62
|
-
this.recoverFromWAL().catch(err => {
|
|
63
|
-
console.error('WAL recovery failed:', err);
|
|
64
|
-
});
|
|
65
|
-
}
|
|
66
|
-
|
|
67
|
-
/**
|
|
68
|
-
* Recover from pending WAL entries
|
|
69
|
-
*/
|
|
70
|
-
private async recoverFromWAL(): Promise<void> {
|
|
71
|
-
await this.wal.recover(async (entry: WALEntry) => {
|
|
72
|
-
// Replay the operation based on WAL entry
|
|
73
|
-
switch (entry.operation) {
|
|
74
|
-
case 'insert':
|
|
75
|
-
case 'update':
|
|
76
|
-
// Ensure document is indexed
|
|
77
|
-
if (entry.documentData) {
|
|
78
|
-
await this.shardManager.addDocument(entry.documentData);
|
|
79
|
-
}
|
|
80
|
-
break;
|
|
81
|
-
case 'delete':
|
|
82
|
-
// Ensure document is removed from index
|
|
83
|
-
await this.shardManager.removeDocument(entry.documentId);
|
|
84
|
-
break;
|
|
85
|
-
}
|
|
86
|
-
});
|
|
87
|
-
}
|
|
88
|
-
|
|
89
|
-
private getDocPath(id: string): string {
|
|
90
|
-
// Validate document ID to prevent path traversal
|
|
91
|
-
const sanitizedId = sanitizePathComponent(id);
|
|
92
|
-
return safePathJoin(this.basePath, 'docs', `${sanitizedId}.json`);
|
|
93
|
-
}
|
|
94
|
-
|
|
95
|
-
async insert(data: InsertDocument<T>): Promise<T> {
|
|
96
|
-
const id = generateId();
|
|
97
|
-
const timestamp = now();
|
|
98
|
-
|
|
99
|
-
const document: T = {
|
|
100
|
-
...data,
|
|
101
|
-
id,
|
|
102
|
-
_createdAt: timestamp,
|
|
103
|
-
_updatedAt: timestamp,
|
|
104
|
-
} as T;
|
|
105
|
-
|
|
106
|
-
// 1. Log operation to WAL first
|
|
107
|
-
await this.wal.log({
|
|
108
|
-
operation: 'insert',
|
|
109
|
-
collectionPath: this.basePath,
|
|
110
|
-
documentId: id,
|
|
111
|
-
documentData: document,
|
|
112
|
-
timestamp,
|
|
113
|
-
});
|
|
114
|
-
|
|
115
|
-
try {
|
|
116
|
-
// 2. Write document to storage
|
|
117
|
-
const { etag } = await this.adapter.put(this.getDocPath(id), document);
|
|
118
|
-
document._etag = etag;
|
|
119
|
-
|
|
120
|
-
// 3. Add to index
|
|
121
|
-
await this.shardManager.addDocument(document);
|
|
122
|
-
|
|
123
|
-
// 4. Clear WAL entry after successful completion
|
|
124
|
-
await this.wal.clear(id);
|
|
125
|
-
|
|
126
|
-
// 5. Invalidate query cache (new document may match existing queries)
|
|
127
|
-
this.queryCache.invalidate();
|
|
128
|
-
|
|
129
|
-
return document;
|
|
130
|
-
} catch (error) {
|
|
131
|
-
// On failure, WAL entry remains for recovery
|
|
132
|
-
throw error;
|
|
133
|
-
}
|
|
134
|
-
}
|
|
135
|
-
|
|
136
|
-
async findById(id: string): Promise<T | null> {
|
|
137
|
-
try {
|
|
138
|
-
const { data, etag } = await this.adapter.get(this.getDocPath(id));
|
|
139
|
-
const doc = data as T;
|
|
140
|
-
doc._etag = etag;
|
|
141
|
-
return doc;
|
|
142
|
-
} catch (error) {
|
|
143
|
-
return null;
|
|
144
|
-
}
|
|
145
|
-
}
|
|
146
|
-
|
|
147
|
-
async find(filter: QueryFilter<T>, options?: QueryOptions): Promise<T[]> {
|
|
148
|
-
// Check cache first
|
|
149
|
-
const cacheKey = LRUCache.generateKey(filter as Record<string, any>, options as Record<string, any>);
|
|
150
|
-
const cached = this.queryCache.get(cacheKey);
|
|
151
|
-
if (cached) {
|
|
152
|
-
return cached;
|
|
153
|
-
}
|
|
154
|
-
|
|
155
|
-
// Query index to get matching document IDs
|
|
156
|
-
const indexResults = await this.shardManager.query(filter);
|
|
157
|
-
|
|
158
|
-
// Get full documents
|
|
159
|
-
let ids = indexResults.map(doc => doc.id);
|
|
160
|
-
|
|
161
|
-
if (ids.length === 0) {
|
|
162
|
-
return [];
|
|
163
|
-
}
|
|
164
|
-
|
|
165
|
-
// Apply pagination on IDs before loading documents (PERF optimization)
|
|
166
|
-
if (options?.offset) {
|
|
167
|
-
ids = ids.slice(options.offset);
|
|
168
|
-
}
|
|
169
|
-
if (options?.limit) {
|
|
170
|
-
ids = ids.slice(0, options.limit);
|
|
171
|
-
}
|
|
172
|
-
|
|
173
|
-
// Batch get only the required documents
|
|
174
|
-
const docMap = await this.adapter.batchGet(ids.map(id => this.getDocPath(id)));
|
|
175
|
-
|
|
176
|
-
// Convert to array and add etags
|
|
177
|
-
const results = Array.from(docMap.values()).map(obj => {
|
|
178
|
-
const doc = obj.data as T;
|
|
179
|
-
doc._etag = obj.etag;
|
|
180
|
-
return doc;
|
|
181
|
-
});
|
|
182
|
-
|
|
183
|
-
// Cache the results
|
|
184
|
-
this.queryCache.set(cacheKey, results);
|
|
185
|
-
|
|
186
|
-
return results;
|
|
187
|
-
}
|
|
188
|
-
|
|
189
|
-
async update(
|
|
190
|
-
id: string,
|
|
191
|
-
data: UpdateDocument<T>,
|
|
192
|
-
options?: UpdateOptions
|
|
193
|
-
): Promise<T> {
|
|
194
|
-
// Get current document
|
|
195
|
-
const current = await this.findById(id);
|
|
196
|
-
if (!current) {
|
|
197
|
-
throw new DocumentNotFoundError(id);
|
|
198
|
-
}
|
|
199
|
-
|
|
200
|
-
// Check optimistic lock if etag provided
|
|
201
|
-
if (options?.etag && current._etag !== options.etag) {
|
|
202
|
-
throw new ConcurrentUpdateError(`ETag mismatch for document ${id}`);
|
|
203
|
-
}
|
|
204
|
-
|
|
205
|
-
// Merge updates
|
|
206
|
-
const updated: T = {
|
|
207
|
-
...current,
|
|
208
|
-
...data,
|
|
209
|
-
id,
|
|
210
|
-
_updatedAt: now(),
|
|
211
|
-
} as T;
|
|
212
|
-
|
|
213
|
-
// 1. Log operation to WAL first
|
|
214
|
-
await this.wal.log({
|
|
215
|
-
operation: 'update',
|
|
216
|
-
collectionPath: this.basePath,
|
|
217
|
-
documentId: id,
|
|
218
|
-
documentData: updated,
|
|
219
|
-
timestamp: now(),
|
|
220
|
-
});
|
|
221
|
-
|
|
222
|
-
try {
|
|
223
|
-
// 2. Write document
|
|
224
|
-
const { etag } = await this.adapter.put(
|
|
225
|
-
this.getDocPath(id),
|
|
226
|
-
updated,
|
|
227
|
-
current._etag ? { ifMatch: current._etag } : undefined
|
|
228
|
-
);
|
|
229
|
-
|
|
230
|
-
updated._etag = etag;
|
|
231
|
-
|
|
232
|
-
// 3. Update index
|
|
233
|
-
await this.shardManager.updateDocument(updated);
|
|
234
|
-
|
|
235
|
-
// 4. Clear WAL entry
|
|
236
|
-
await this.wal.clear(id);
|
|
237
|
-
|
|
238
|
-
// 5. Invalidate query cache (document changed)
|
|
239
|
-
this.queryCache.invalidate();
|
|
240
|
-
|
|
241
|
-
return updated;
|
|
242
|
-
} catch (error) {
|
|
243
|
-
throw error;
|
|
244
|
-
}
|
|
245
|
-
}
|
|
246
|
-
|
|
247
|
-
async delete(id: string): Promise<void> {
|
|
248
|
-
// Check if document exists
|
|
249
|
-
const exists = await this.findById(id);
|
|
250
|
-
if (!exists) {
|
|
251
|
-
throw new DocumentNotFoundError(id);
|
|
252
|
-
}
|
|
253
|
-
|
|
254
|
-
// 1. Log operation to WAL first
|
|
255
|
-
await this.wal.log({
|
|
256
|
-
operation: 'delete',
|
|
257
|
-
collectionPath: this.basePath,
|
|
258
|
-
documentId: id,
|
|
259
|
-
timestamp: now(),
|
|
260
|
-
});
|
|
261
|
-
|
|
262
|
-
try {
|
|
263
|
-
// 2. Delete from storage
|
|
264
|
-
await this.adapter.delete(this.getDocPath(id));
|
|
265
|
-
|
|
266
|
-
// 3. Remove from index
|
|
267
|
-
await this.shardManager.removeDocument(id);
|
|
268
|
-
|
|
269
|
-
// 4. Clear WAL entry
|
|
270
|
-
await this.wal.clear(id);
|
|
271
|
-
|
|
272
|
-
// 5. Invalidate query cache (document deleted)
|
|
273
|
-
this.queryCache.invalidate();
|
|
274
|
-
} catch (error) {
|
|
275
|
-
throw error;
|
|
276
|
-
}
|
|
277
|
-
}
|
|
278
|
-
}
|
package/src/core/wal.ts
DELETED
|
@@ -1,106 +0,0 @@
|
|
|
1
|
-
import type { StorageAdapter } from '../types/index.js';
|
|
2
|
-
import { safePathJoin } from '../utils/path-validator.js';
|
|
3
|
-
|
|
4
|
-
/**
|
|
5
|
-
* WAL operation types
|
|
6
|
-
*/
|
|
7
|
-
export type WALOperation = 'insert' | 'update' | 'delete';
|
|
8
|
-
|
|
9
|
-
/**
|
|
10
|
-
* WAL entry structure
|
|
11
|
-
*/
|
|
12
|
-
export interface WALEntry {
|
|
13
|
-
operation: WALOperation;
|
|
14
|
-
collectionPath: string;
|
|
15
|
-
documentId: string;
|
|
16
|
-
documentData?: any;
|
|
17
|
-
timestamp: string;
|
|
18
|
-
}
|
|
19
|
-
|
|
20
|
-
/**
|
|
21
|
-
* Write-Ahead Log for ensuring index-document consistency
|
|
22
|
-
*/
|
|
23
|
-
export class WAL {
|
|
24
|
-
private walPath: string;
|
|
25
|
-
|
|
26
|
-
constructor(
|
|
27
|
-
private adapter: StorageAdapter,
|
|
28
|
-
basePath: string
|
|
29
|
-
) {
|
|
30
|
-
this.walPath = safePathJoin(basePath, 'wal');
|
|
31
|
-
}
|
|
32
|
-
|
|
33
|
-
/**
|
|
34
|
-
* Get WAL entry path for a document
|
|
35
|
-
*/
|
|
36
|
-
private getWALEntryPath(documentId: string): string {
|
|
37
|
-
return `${this.walPath}/${documentId}.wal.json`;
|
|
38
|
-
}
|
|
39
|
-
|
|
40
|
-
/**
|
|
41
|
-
* Log an operation before execution
|
|
42
|
-
*/
|
|
43
|
-
async log(entry: WALEntry): Promise<void> {
|
|
44
|
-
const path = this.getWALEntryPath(entry.documentId);
|
|
45
|
-
await this.adapter.put(path, entry);
|
|
46
|
-
}
|
|
47
|
-
|
|
48
|
-
/**
|
|
49
|
-
* Clear WAL entry after successful operation
|
|
50
|
-
*/
|
|
51
|
-
async clear(documentId: string): Promise<void> {
|
|
52
|
-
const path = this.getWALEntryPath(documentId);
|
|
53
|
-
try {
|
|
54
|
-
await this.adapter.delete(path);
|
|
55
|
-
} catch {
|
|
56
|
-
// Ignore if entry doesn't exist
|
|
57
|
-
}
|
|
58
|
-
}
|
|
59
|
-
|
|
60
|
-
/**
|
|
61
|
-
* List all pending WAL entries
|
|
62
|
-
*/
|
|
63
|
-
async listPending(): Promise<WALEntry[]> {
|
|
64
|
-
try {
|
|
65
|
-
const keys = await this.adapter.listKeys(this.walPath);
|
|
66
|
-
const entries: WALEntry[] = [];
|
|
67
|
-
|
|
68
|
-
for (const key of keys) {
|
|
69
|
-
try {
|
|
70
|
-
const { data } = await this.adapter.get(key);
|
|
71
|
-
entries.push(data as WALEntry);
|
|
72
|
-
} catch {
|
|
73
|
-
// Skip corrupted entries
|
|
74
|
-
}
|
|
75
|
-
}
|
|
76
|
-
|
|
77
|
-
return entries;
|
|
78
|
-
} catch {
|
|
79
|
-
return [];
|
|
80
|
-
}
|
|
81
|
-
}
|
|
82
|
-
|
|
83
|
-
/**
|
|
84
|
-
* Recover from pending WAL entries
|
|
85
|
-
* This should be called on collection initialization
|
|
86
|
-
*/
|
|
87
|
-
async recover(
|
|
88
|
-
onRecover: (entry: WALEntry) => Promise<void>
|
|
89
|
-
): Promise<number> {
|
|
90
|
-
const pending = await this.listPending();
|
|
91
|
-
let recovered = 0;
|
|
92
|
-
|
|
93
|
-
for (const entry of pending) {
|
|
94
|
-
try {
|
|
95
|
-
await onRecover(entry);
|
|
96
|
-
await this.clear(entry.documentId);
|
|
97
|
-
recovered++;
|
|
98
|
-
} catch (error) {
|
|
99
|
-
// Log error but continue recovery
|
|
100
|
-
console.error(`Failed to recover WAL entry for ${entry.documentId}:`, error);
|
|
101
|
-
}
|
|
102
|
-
}
|
|
103
|
-
|
|
104
|
-
return recovered;
|
|
105
|
-
}
|
|
106
|
-
}
|
|
@@ -1,197 +0,0 @@
|
|
|
1
|
-
import type { StorageAdapter, IndexShard, QueryFilter, ShardMetadata } from '../types/index.js';
|
|
2
|
-
import { StorageError } from '../types/index.js';
|
|
3
|
-
import { getShardId, formatShardId } from '../utils/hash.js';
|
|
4
|
-
import { evaluateFilter } from '../query/evaluator.js';
|
|
5
|
-
|
|
6
|
-
/**
|
|
7
|
-
* Manages index shards for a collection
|
|
8
|
-
*/
|
|
9
|
-
export class ShardManager {
|
|
10
|
-
constructor(
|
|
11
|
-
private adapter: StorageAdapter,
|
|
12
|
-
private collectionPath: string,
|
|
13
|
-
private shardCount: number
|
|
14
|
-
) {}
|
|
15
|
-
|
|
16
|
-
private getShardPath(shardId: number): string {
|
|
17
|
-
return `${this.collectionPath}/index/shard-${formatShardId(shardId)}.json`;
|
|
18
|
-
}
|
|
19
|
-
|
|
20
|
-
private getShardMetadataPath(shardId: number): string {
|
|
21
|
-
return `${this.collectionPath}/index/shard-${formatShardId(shardId)}.meta.json`;
|
|
22
|
-
}
|
|
23
|
-
|
|
24
|
-
private async readShardMetadata(shardId: number): Promise<ShardMetadata | null> {
|
|
25
|
-
const path = this.getShardMetadataPath(shardId);
|
|
26
|
-
try {
|
|
27
|
-
const { data } = await this.adapter.get(path);
|
|
28
|
-
return data as ShardMetadata;
|
|
29
|
-
} catch (error) {
|
|
30
|
-
return null;
|
|
31
|
-
}
|
|
32
|
-
}
|
|
33
|
-
|
|
34
|
-
private async readShard(shardId: number): Promise<{ shard: IndexShard; etag?: string }> {
|
|
35
|
-
const path = this.getShardPath(shardId);
|
|
36
|
-
try {
|
|
37
|
-
const { data, etag } = await this.adapter.get(path);
|
|
38
|
-
return { shard: data as IndexShard, etag };
|
|
39
|
-
} catch (error) {
|
|
40
|
-
// Return empty shard if not found
|
|
41
|
-
return {
|
|
42
|
-
shard: {
|
|
43
|
-
shardId: formatShardId(shardId),
|
|
44
|
-
documents: {},
|
|
45
|
-
},
|
|
46
|
-
etag: undefined,
|
|
47
|
-
};
|
|
48
|
-
}
|
|
49
|
-
}
|
|
50
|
-
|
|
51
|
-
private async writeShard(shardId: number, shard: IndexShard, etag?: string): Promise<void> {
|
|
52
|
-
const path = this.getShardPath(shardId);
|
|
53
|
-
const metaPath = this.getShardMetadataPath(shardId);
|
|
54
|
-
|
|
55
|
-
// Write shard data
|
|
56
|
-
await this.adapter.put(path, shard, etag ? { ifMatch: etag } : undefined);
|
|
57
|
-
|
|
58
|
-
// Write metadata
|
|
59
|
-
const metadata: ShardMetadata = {
|
|
60
|
-
shardId: formatShardId(shardId),
|
|
61
|
-
docCount: Object.keys(shard.documents).length,
|
|
62
|
-
lastUpdated: new Date().toISOString(),
|
|
63
|
-
};
|
|
64
|
-
await this.adapter.put(metaPath, metadata);
|
|
65
|
-
}
|
|
66
|
-
|
|
67
|
-
/**
|
|
68
|
-
* Add document to index
|
|
69
|
-
*/
|
|
70
|
-
async addDocument(doc: Record<string, any>): Promise<void> {
|
|
71
|
-
const docId = doc.id;
|
|
72
|
-
const shardId = getShardId(docId, this.shardCount);
|
|
73
|
-
|
|
74
|
-
// Retry logic for optimistic locking with exponential backoff
|
|
75
|
-
const maxRetries = 3;
|
|
76
|
-
for (let attempt = 0; attempt < maxRetries; attempt++) {
|
|
77
|
-
try {
|
|
78
|
-
// Read shard and etag in a single operation
|
|
79
|
-
const { shard, etag } = await this.readShard(shardId);
|
|
80
|
-
|
|
81
|
-
shard.documents[docId] = { ...doc };
|
|
82
|
-
|
|
83
|
-
await this.writeShard(shardId, shard, etag);
|
|
84
|
-
return;
|
|
85
|
-
} catch (error) {
|
|
86
|
-
if (error instanceof StorageError && error.message.includes('PreconditionFailed')) {
|
|
87
|
-
if (attempt < maxRetries - 1) {
|
|
88
|
-
// Exponential backoff with jitter: 100ms * 2^attempt + random(0-50ms)
|
|
89
|
-
const baseDelay = 100 * Math.pow(2, attempt);
|
|
90
|
-
const jitter = Math.random() * 50;
|
|
91
|
-
await new Promise(resolve => setTimeout(resolve, baseDelay + jitter));
|
|
92
|
-
continue;
|
|
93
|
-
}
|
|
94
|
-
}
|
|
95
|
-
throw error;
|
|
96
|
-
}
|
|
97
|
-
}
|
|
98
|
-
throw new StorageError('Failed to add document after retries');
|
|
99
|
-
}
|
|
100
|
-
|
|
101
|
-
/**
|
|
102
|
-
* Update document in index
|
|
103
|
-
*/
|
|
104
|
-
async updateDocument(doc: Record<string, any>): Promise<void> {
|
|
105
|
-
await this.addDocument(doc); // Same as add - upsert behavior
|
|
106
|
-
}
|
|
107
|
-
|
|
108
|
-
/**
|
|
109
|
-
* Remove document from index
|
|
110
|
-
*/
|
|
111
|
-
async removeDocument(docId: string): Promise<void> {
|
|
112
|
-
const shardId = getShardId(docId, this.shardCount);
|
|
113
|
-
|
|
114
|
-
const maxRetries = 3;
|
|
115
|
-
for (let attempt = 0; attempt < maxRetries; attempt++) {
|
|
116
|
-
try {
|
|
117
|
-
// Read shard and etag in a single operation
|
|
118
|
-
const { shard, etag } = await this.readShard(shardId);
|
|
119
|
-
|
|
120
|
-
delete shard.documents[docId];
|
|
121
|
-
|
|
122
|
-
await this.writeShard(shardId, shard, etag);
|
|
123
|
-
return;
|
|
124
|
-
} catch (error) {
|
|
125
|
-
if (error instanceof StorageError && error.message.includes('PreconditionFailed')) {
|
|
126
|
-
if (attempt < maxRetries - 1) {
|
|
127
|
-
// Exponential backoff with jitter
|
|
128
|
-
const baseDelay = 100 * Math.pow(2, attempt);
|
|
129
|
-
const jitter = Math.random() * 50;
|
|
130
|
-
await new Promise(resolve => setTimeout(resolve, baseDelay + jitter));
|
|
131
|
-
continue;
|
|
132
|
-
}
|
|
133
|
-
}
|
|
134
|
-
// Ignore if shard doesn't exist
|
|
135
|
-
if (error instanceof StorageError && error.message.includes('not found')) {
|
|
136
|
-
return;
|
|
137
|
-
}
|
|
138
|
-
throw error;
|
|
139
|
-
}
|
|
140
|
-
}
|
|
141
|
-
throw new StorageError('Failed to remove document after retries');
|
|
142
|
-
}
|
|
143
|
-
|
|
144
|
-
/**
|
|
145
|
-
* Find document by ID
|
|
146
|
-
*/
|
|
147
|
-
async findById(docId: string): Promise<Record<string, any> | null> {
|
|
148
|
-
const shardId = getShardId(docId, this.shardCount);
|
|
149
|
-
const { shard } = await this.readShard(shardId);
|
|
150
|
-
return shard.documents[docId] || null;
|
|
151
|
-
}
|
|
152
|
-
|
|
153
|
-
/**
|
|
154
|
-
* Query documents matching filter
|
|
155
|
-
*/
|
|
156
|
-
async query(filter: QueryFilter<any>): Promise<Array<Record<string, any>>> {
|
|
157
|
-
// Optimization: If filter contains id equality, target specific shard
|
|
158
|
-
if (filter.id && typeof filter.id === 'string') {
|
|
159
|
-
const targetShardId = getShardId(filter.id, this.shardCount);
|
|
160
|
-
const { shard } = await this.readShard(targetShardId);
|
|
161
|
-
const doc = shard.documents[filter.id];
|
|
162
|
-
if (doc && evaluateFilter(doc, filter)) {
|
|
163
|
-
return [doc];
|
|
164
|
-
}
|
|
165
|
-
return [];
|
|
166
|
-
}
|
|
167
|
-
|
|
168
|
-
// Read all shard metadata first (lightweight)
|
|
169
|
-
const metadataPromises = Array.from(
|
|
170
|
-
{ length: this.shardCount },
|
|
171
|
-
(_, i) => this.readShardMetadata(i)
|
|
172
|
-
);
|
|
173
|
-
const allMetadata = await Promise.all(metadataPromises);
|
|
174
|
-
|
|
175
|
-
// Filter out empty shards
|
|
176
|
-
const nonEmptyShardIds = allMetadata
|
|
177
|
-
.map((meta, index) => ({ meta, index }))
|
|
178
|
-
.filter(({ meta }) => meta && meta.docCount > 0)
|
|
179
|
-
.map(({ index }) => index);
|
|
180
|
-
|
|
181
|
-
// Read only non-empty shards in parallel
|
|
182
|
-
const shardPromises = nonEmptyShardIds.map(i => this.readShard(i));
|
|
183
|
-
const shardResults = await Promise.all(shardPromises);
|
|
184
|
-
|
|
185
|
-
// Filter documents across all shards
|
|
186
|
-
const results: Array<Record<string, any>> = [];
|
|
187
|
-
for (const { shard } of shardResults) {
|
|
188
|
-
for (const doc of Object.values(shard.documents)) {
|
|
189
|
-
if (evaluateFilter(doc, filter)) {
|
|
190
|
-
results.push(doc);
|
|
191
|
-
}
|
|
192
|
-
}
|
|
193
|
-
}
|
|
194
|
-
|
|
195
|
-
return results;
|
|
196
|
-
}
|
|
197
|
-
}
|
package/src/index.ts
DELETED
|
@@ -1,37 +0,0 @@
|
|
|
1
|
-
// Core classes
|
|
2
|
-
export { BucketDB } from './core/bucket-db.js';
|
|
3
|
-
export { Collection } from './core/collection.js';
|
|
4
|
-
|
|
5
|
-
// Storage adapters
|
|
6
|
-
export { MemoryStorageAdapter } from './storage/memory-adapter.js';
|
|
7
|
-
export { S3Adapter } from './storage/s3-adapter.js';
|
|
8
|
-
export { OSSAdapter } from './storage/oss-adapter.js';
|
|
9
|
-
export { FileSystemAdapter } from './storage/fs-adapter.js';
|
|
10
|
-
|
|
11
|
-
// Re-export types from @bucket-db/types
|
|
12
|
-
export type {
|
|
13
|
-
Document,
|
|
14
|
-
InsertDocument,
|
|
15
|
-
UpdateDocument,
|
|
16
|
-
QueryOperator,
|
|
17
|
-
QueryValue,
|
|
18
|
-
QueryFilter,
|
|
19
|
-
QueryOptions,
|
|
20
|
-
UpdateOptions,
|
|
21
|
-
StorageAdapter,
|
|
22
|
-
StorageObject,
|
|
23
|
-
PutOptions,
|
|
24
|
-
StorageCredentials,
|
|
25
|
-
StorageAdapterConfig,
|
|
26
|
-
ICollection,
|
|
27
|
-
CollectionMeta,
|
|
28
|
-
IndexShard,
|
|
29
|
-
} from './types/index.js';
|
|
30
|
-
|
|
31
|
-
export {
|
|
32
|
-
BucketDBError,
|
|
33
|
-
DocumentNotFoundError,
|
|
34
|
-
ConcurrentUpdateError,
|
|
35
|
-
ValidationError,
|
|
36
|
-
StorageError,
|
|
37
|
-
} from './types/index.js';
|