@hold-baby/bucket-db 0.2.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@hold-baby/bucket-db",
3
- "version": "0.2.0",
3
+ "version": "0.3.0",
4
4
  "type": "module",
5
5
  "main": "./dist/index.js",
6
6
  "types": "./dist/index.d.ts",
@@ -0,0 +1,131 @@
1
+ import { createHash } from 'crypto';
2
+
3
+ interface CacheEntry<T> {
4
+ value: T;
5
+ timestamp: number;
6
+ }
7
+
8
+ /**
9
+ * LRU (Least Recently Used) Cache implementation
10
+ */
11
+ export class LRUCache<T> {
12
+ private cache: Map<string, CacheEntry<T>>;
13
+ private maxSize: number;
14
+ private ttl: number; // Time To Live in milliseconds
15
+
16
+ // Statistics
17
+ private hits: number = 0;
18
+ private misses: number = 0;
19
+ private evictions: number = 0;
20
+
21
+ constructor(maxSize: number = 100, ttl: number = 5 * 60 * 1000) {
22
+ this.cache = new Map();
23
+ this.maxSize = maxSize;
24
+ this.ttl = ttl;
25
+ }
26
+
27
+ /**
28
+ * Generate cache key from query filter and options
29
+ */
30
+ static generateKey(filter: Record<string, any>, options?: Record<string, any>): string {
31
+ const data = JSON.stringify({ filter, options });
32
+ return createHash('md5').update(data).digest('hex');
33
+ }
34
+
35
+ /**
36
+ * Get value from cache
37
+ */
38
+ get(key: string): T | null {
39
+ const entry = this.cache.get(key);
40
+
41
+ if (!entry) {
42
+ this.misses++;
43
+ return null;
44
+ }
45
+
46
+ // Check if entry has expired
47
+ if (Date.now() - entry.timestamp > this.ttl) {
48
+ this.cache.delete(key);
49
+ this.misses++;
50
+ return null;
51
+ }
52
+
53
+ // Move to end (mark as recently used)
54
+ this.cache.delete(key);
55
+ this.cache.set(key, entry);
56
+ this.hits++;
57
+
58
+ return entry.value;
59
+ }
60
+
61
+ /**
62
+ * Put value into cache
63
+ */
64
+ set(key: string, value: T): void {
65
+ // Remove if already exists (to update position)
66
+ if (this.cache.has(key)) {
67
+ this.cache.delete(key);
68
+ }
69
+
70
+ // Evict oldest entry if cache is full
71
+ if (this.cache.size >= this.maxSize) {
72
+ const firstKey = this.cache.keys().next().value;
73
+ if (firstKey) {
74
+ this.cache.delete(firstKey);
75
+ this.evictions++;
76
+ }
77
+ }
78
+
79
+ this.cache.set(key, {
80
+ value,
81
+ timestamp: Date.now(),
82
+ });
83
+ }
84
+
85
+ /**
86
+ * Invalidate cache entries (clear all by default)
87
+ */
88
+ invalidate(pattern?: RegExp): void {
89
+ if (!pattern) {
90
+ this.cache.clear();
91
+ return;
92
+ }
93
+
94
+ // Remove entries matching pattern
95
+ for (const key of this.cache.keys()) {
96
+ if (pattern.test(key)) {
97
+ this.cache.delete(key);
98
+ }
99
+ }
100
+ }
101
+
102
+ /**
103
+ * Get cache statistics
104
+ */
105
+ getStats() {
106
+ const total = this.hits + this.misses;
107
+ return {
108
+ size: this.cache.size,
109
+ hits: this.hits,
110
+ misses: this.misses,
111
+ evictions: this.evictions,
112
+ hitRate: total > 0 ? (this.hits / total * 100).toFixed(2) + '%' : '0%',
113
+ };
114
+ }
115
+
116
+ /**
117
+ * Clear all cache statistics
118
+ */
119
+ resetStats(): void {
120
+ this.hits = 0;
121
+ this.misses = 0;
122
+ this.evictions = 0;
123
+ }
124
+
125
+ /**
126
+ * Clear the entire cache
127
+ */
128
+ clear(): void {
129
+ this.cache.clear();
130
+ }
131
+ }
@@ -1,5 +1,6 @@
1
1
  import type { StorageAdapter, Document } from '../types/index.js';
2
2
  import { Collection, CollectionOptions } from './collection.js';
3
+ import { sanitizePathComponent } from '../utils/path-validator.js';
3
4
 
4
5
  export interface BucketDBOptions {
5
6
  defaultShardCount?: number;
@@ -24,6 +25,9 @@ export class BucketDB {
24
25
  name: string,
25
26
  options?: CollectionOptions
26
27
  ): Collection<T> {
28
+ // Validate collection name to prevent path traversal
29
+ sanitizePathComponent(name);
30
+
27
31
  if (this.collections.has(name)) {
28
32
  return this.collections.get(name)!;
29
33
  }
@@ -10,6 +10,9 @@ import type {
10
10
  } from '../types/index.js';
11
11
  import { DocumentNotFoundError, ConcurrentUpdateError } from '../types/index.js';
12
12
  import { ShardManager } from '../index/shard-manager.js';
13
+ import { sanitizePathComponent, safePathJoin } from '../utils/path-validator.js';
14
+ import { WAL, type WALEntry } from './wal.js';
15
+ import { LRUCache } from '../cache/lru-cache.js';
13
16
 
14
17
  /**
15
18
  * Generate UUID v4
@@ -34,24 +37,59 @@ export interface CollectionOptions {
34
37
  */
35
38
  export class Collection<T extends Document> implements ICollection<T> {
36
39
  private shardManager: ShardManager;
40
+ private wal: WAL;
41
+ private queryCache: LRUCache<T[]>;
37
42
  private basePath: string;
38
43
 
39
44
  constructor(
40
45
  private adapter: StorageAdapter,
41
- private dbPath: string,
42
- private collectionName: string,
46
+ dbPath: string,
47
+ collectionName: string,
43
48
  options: CollectionOptions = {}
44
49
  ) {
45
- this.basePath = `${dbPath}/${collectionName}`;
50
+ // Validate collection name to prevent path traversal
51
+ const sanitizedCollectionName = sanitizePathComponent(collectionName);
52
+ this.basePath = safePathJoin(dbPath, sanitizedCollectionName);
46
53
  this.shardManager = new ShardManager(
47
54
  adapter,
48
55
  this.basePath,
49
56
  options.shardCount || 16
50
57
  );
58
+ this.wal = new WAL(adapter, this.basePath);
59
+ this.queryCache = new LRUCache<T[]>(100, 5 * 60 * 1000); // 100 entries, 5 min TTL
60
+
61
+ // Recover from any pending WAL entries
62
+ this.recoverFromWAL().catch(err => {
63
+ console.error('WAL recovery failed:', err);
64
+ });
65
+ }
66
+
67
+ /**
68
+ * Recover from pending WAL entries
69
+ */
70
+ private async recoverFromWAL(): Promise<void> {
71
+ await this.wal.recover(async (entry: WALEntry) => {
72
+ // Replay the operation based on WAL entry
73
+ switch (entry.operation) {
74
+ case 'insert':
75
+ case 'update':
76
+ // Ensure document is indexed
77
+ if (entry.documentData) {
78
+ await this.shardManager.addDocument(entry.documentData);
79
+ }
80
+ break;
81
+ case 'delete':
82
+ // Ensure document is removed from index
83
+ await this.shardManager.removeDocument(entry.documentId);
84
+ break;
85
+ }
86
+ });
51
87
  }
52
88
 
53
89
  private getDocPath(id: string): string {
54
- return `${this.basePath}/docs/${id}.json`;
90
+ // Validate document ID to prevent path traversal
91
+ const sanitizedId = sanitizePathComponent(id);
92
+ return safePathJoin(this.basePath, 'docs', `${sanitizedId}.json`);
55
93
  }
56
94
 
57
95
  async insert(data: InsertDocument<T>): Promise<T> {
@@ -65,15 +103,34 @@ export class Collection<T extends Document> implements ICollection<T> {
65
103
  _updatedAt: timestamp,
66
104
  } as T;
67
105
 
68
- // Write document to storage
69
- const { etag } = await this.adapter.put(this.getDocPath(id), document);
106
+ // 1. Log operation to WAL first
107
+ await this.wal.log({
108
+ operation: 'insert',
109
+ collectionPath: this.basePath,
110
+ documentId: id,
111
+ documentData: document,
112
+ timestamp,
113
+ });
114
+
115
+ try {
116
+ // 2. Write document to storage
117
+ const { etag } = await this.adapter.put(this.getDocPath(id), document);
118
+ document._etag = etag;
70
119
 
71
- document._etag = etag;
120
+ // 3. Add to index
121
+ await this.shardManager.addDocument(document);
72
122
 
73
- // Add to index
74
- await this.shardManager.addDocument(document);
123
+ // 4. Clear WAL entry after successful completion
124
+ await this.wal.clear(id);
75
125
 
76
- return document;
126
+ // 5. Invalidate query cache (new document may match existing queries)
127
+ this.queryCache.invalidate();
128
+
129
+ return document;
130
+ } catch (error) {
131
+ // On failure, WAL entry remains for recovery
132
+ throw error;
133
+ }
77
134
  }
78
135
 
79
136
  async findById(id: string): Promise<T | null> {
@@ -88,32 +145,43 @@ export class Collection<T extends Document> implements ICollection<T> {
88
145
  }
89
146
 
90
147
  async find(filter: QueryFilter<T>, options?: QueryOptions): Promise<T[]> {
148
+ // Check cache first
149
+ const cacheKey = LRUCache.generateKey(filter as Record<string, any>, options as Record<string, any>);
150
+ const cached = this.queryCache.get(cacheKey);
151
+ if (cached) {
152
+ return cached;
153
+ }
154
+
91
155
  // Query index to get matching document IDs
92
156
  const indexResults = await this.shardManager.query(filter);
93
157
 
94
158
  // Get full documents
95
- const ids = indexResults.map(doc => doc.id);
159
+ let ids = indexResults.map(doc => doc.id);
96
160
 
97
161
  if (ids.length === 0) {
98
162
  return [];
99
163
  }
100
164
 
101
- // Batch get documents
165
+ // Apply pagination on IDs before loading documents (PERF optimization)
166
+ if (options?.offset) {
167
+ ids = ids.slice(options.offset);
168
+ }
169
+ if (options?.limit) {
170
+ ids = ids.slice(0, options.limit);
171
+ }
172
+
173
+ // Batch get only the required documents
102
174
  const docMap = await this.adapter.batchGet(ids.map(id => this.getDocPath(id)));
103
175
 
104
- let results = Array.from(docMap.values()).map(obj => {
176
+ // Convert to array and add etags
177
+ const results = Array.from(docMap.values()).map(obj => {
105
178
  const doc = obj.data as T;
106
179
  doc._etag = obj.etag;
107
180
  return doc;
108
181
  });
109
182
 
110
- // Apply pagination
111
- if (options?.offset) {
112
- results = results.slice(options.offset);
113
- }
114
- if (options?.limit) {
115
- results = results.slice(0, options.limit);
116
- }
183
+ // Cache the results
184
+ this.queryCache.set(cacheKey, results);
117
185
 
118
186
  return results;
119
187
  }
@@ -142,19 +210,38 @@ export class Collection<T extends Document> implements ICollection<T> {
142
210
  _updatedAt: now(),
143
211
  } as T;
144
212
 
145
- // Write document
146
- const { etag } = await this.adapter.put(
147
- this.getDocPath(id),
148
- updated,
149
- current._etag ? { ifMatch: current._etag } : undefined
150
- );
213
+ // 1. Log operation to WAL first
214
+ await this.wal.log({
215
+ operation: 'update',
216
+ collectionPath: this.basePath,
217
+ documentId: id,
218
+ documentData: updated,
219
+ timestamp: now(),
220
+ });
221
+
222
+ try {
223
+ // 2. Write document
224
+ const { etag } = await this.adapter.put(
225
+ this.getDocPath(id),
226
+ updated,
227
+ current._etag ? { ifMatch: current._etag } : undefined
228
+ );
229
+
230
+ updated._etag = etag;
231
+
232
+ // 3. Update index
233
+ await this.shardManager.updateDocument(updated);
151
234
 
152
- updated._etag = etag;
235
+ // 4. Clear WAL entry
236
+ await this.wal.clear(id);
153
237
 
154
- // Update index
155
- await this.shardManager.updateDocument(updated);
238
+ // 5. Invalidate query cache (document changed)
239
+ this.queryCache.invalidate();
156
240
 
157
- return updated;
241
+ return updated;
242
+ } catch (error) {
243
+ throw error;
244
+ }
158
245
  }
159
246
 
160
247
  async delete(id: string): Promise<void> {
@@ -164,10 +251,28 @@ export class Collection<T extends Document> implements ICollection<T> {
164
251
  throw new DocumentNotFoundError(id);
165
252
  }
166
253
 
167
- // Delete from storage
168
- await this.adapter.delete(this.getDocPath(id));
254
+ // 1. Log operation to WAL first
255
+ await this.wal.log({
256
+ operation: 'delete',
257
+ collectionPath: this.basePath,
258
+ documentId: id,
259
+ timestamp: now(),
260
+ });
261
+
262
+ try {
263
+ // 2. Delete from storage
264
+ await this.adapter.delete(this.getDocPath(id));
169
265
 
170
- // Remove from index
171
- await this.shardManager.removeDocument(id);
266
+ // 3. Remove from index
267
+ await this.shardManager.removeDocument(id);
268
+
269
+ // 4. Clear WAL entry
270
+ await this.wal.clear(id);
271
+
272
+ // 5. Invalidate query cache (document deleted)
273
+ this.queryCache.invalidate();
274
+ } catch (error) {
275
+ throw error;
276
+ }
172
277
  }
173
278
  }
@@ -0,0 +1,106 @@
1
+ import type { StorageAdapter } from '../types/index.js';
2
+ import { safePathJoin } from '../utils/path-validator.js';
3
+
4
+ /**
5
+ * WAL operation types
6
+ */
7
+ export type WALOperation = 'insert' | 'update' | 'delete';
8
+
9
+ /**
10
+ * WAL entry structure
11
+ */
12
+ export interface WALEntry {
13
+ operation: WALOperation;
14
+ collectionPath: string;
15
+ documentId: string;
16
+ documentData?: any;
17
+ timestamp: string;
18
+ }
19
+
20
+ /**
21
+ * Write-Ahead Log for ensuring index-document consistency
22
+ */
23
+ export class WAL {
24
+ private walPath: string;
25
+
26
+ constructor(
27
+ private adapter: StorageAdapter,
28
+ basePath: string
29
+ ) {
30
+ this.walPath = safePathJoin(basePath, 'wal');
31
+ }
32
+
33
+ /**
34
+ * Get WAL entry path for a document
35
+ */
36
+ private getWALEntryPath(documentId: string): string {
37
+ return `${this.walPath}/${documentId}.wal.json`;
38
+ }
39
+
40
+ /**
41
+ * Log an operation before execution
42
+ */
43
+ async log(entry: WALEntry): Promise<void> {
44
+ const path = this.getWALEntryPath(entry.documentId);
45
+ await this.adapter.put(path, entry);
46
+ }
47
+
48
+ /**
49
+ * Clear WAL entry after successful operation
50
+ */
51
+ async clear(documentId: string): Promise<void> {
52
+ const path = this.getWALEntryPath(documentId);
53
+ try {
54
+ await this.adapter.delete(path);
55
+ } catch {
56
+ // Ignore if entry doesn't exist
57
+ }
58
+ }
59
+
60
+ /**
61
+ * List all pending WAL entries
62
+ */
63
+ async listPending(): Promise<WALEntry[]> {
64
+ try {
65
+ const keys = await this.adapter.listKeys(this.walPath);
66
+ const entries: WALEntry[] = [];
67
+
68
+ for (const key of keys) {
69
+ try {
70
+ const { data } = await this.adapter.get(key);
71
+ entries.push(data as WALEntry);
72
+ } catch {
73
+ // Skip corrupted entries
74
+ }
75
+ }
76
+
77
+ return entries;
78
+ } catch {
79
+ return [];
80
+ }
81
+ }
82
+
83
+ /**
84
+ * Recover from pending WAL entries
85
+ * This should be called on collection initialization
86
+ */
87
+ async recover(
88
+ onRecover: (entry: WALEntry) => Promise<void>
89
+ ): Promise<number> {
90
+ const pending = await this.listPending();
91
+ let recovered = 0;
92
+
93
+ for (const entry of pending) {
94
+ try {
95
+ await onRecover(entry);
96
+ await this.clear(entry.documentId);
97
+ recovered++;
98
+ } catch (error) {
99
+ // Log error but continue recovery
100
+ console.error(`Failed to recover WAL entry for ${entry.documentId}:`, error);
101
+ }
102
+ }
103
+
104
+ return recovered;
105
+ }
106
+ }
@@ -1,4 +1,4 @@
1
- import type { StorageAdapter, IndexShard, QueryFilter } from '../types/index.js';
1
+ import type { StorageAdapter, IndexShard, QueryFilter, ShardMetadata } from '../types/index.js';
2
2
  import { StorageError } from '../types/index.js';
3
3
  import { getShardId, formatShardId } from '../utils/hash.js';
4
4
  import { evaluateFilter } from '../query/evaluator.js';
@@ -17,23 +17,51 @@ export class ShardManager {
17
17
  return `${this.collectionPath}/index/shard-${formatShardId(shardId)}.json`;
18
18
  }
19
19
 
20
- private async readShard(shardId: number): Promise<IndexShard> {
21
- const path = this.getShardPath(shardId);
20
+ private getShardMetadataPath(shardId: number): string {
21
+ return `${this.collectionPath}/index/shard-${formatShardId(shardId)}.meta.json`;
22
+ }
23
+
24
+ private async readShardMetadata(shardId: number): Promise<ShardMetadata | null> {
25
+ const path = this.getShardMetadataPath(shardId);
22
26
  try {
23
27
  const { data } = await this.adapter.get(path);
24
- return data as IndexShard;
28
+ return data as ShardMetadata;
29
+ } catch (error) {
30
+ return null;
31
+ }
32
+ }
33
+
34
+ private async readShard(shardId: number): Promise<{ shard: IndexShard; etag?: string }> {
35
+ const path = this.getShardPath(shardId);
36
+ try {
37
+ const { data, etag } = await this.adapter.get(path);
38
+ return { shard: data as IndexShard, etag };
25
39
  } catch (error) {
26
40
  // Return empty shard if not found
27
41
  return {
28
- shardId: formatShardId(shardId),
29
- documents: {},
42
+ shard: {
43
+ shardId: formatShardId(shardId),
44
+ documents: {},
45
+ },
46
+ etag: undefined,
30
47
  };
31
48
  }
32
49
  }
33
50
 
34
51
  private async writeShard(shardId: number, shard: IndexShard, etag?: string): Promise<void> {
35
52
  const path = this.getShardPath(shardId);
53
+ const metaPath = this.getShardMetadataPath(shardId);
54
+
55
+ // Write shard data
36
56
  await this.adapter.put(path, shard, etag ? { ifMatch: etag } : undefined);
57
+
58
+ // Write metadata
59
+ const metadata: ShardMetadata = {
60
+ shardId: formatShardId(shardId),
61
+ docCount: Object.keys(shard.documents).length,
62
+ lastUpdated: new Date().toISOString(),
63
+ };
64
+ await this.adapter.put(metaPath, metadata);
37
65
  }
38
66
 
39
67
  /**
@@ -43,24 +71,26 @@ export class ShardManager {
43
71
  const docId = doc.id;
44
72
  const shardId = getShardId(docId, this.shardCount);
45
73
 
46
- // Retry logic for optimistic locking
47
- let retries = 3;
48
- while (retries > 0) {
74
+ // Retry logic for optimistic locking with exponential backoff
75
+ const maxRetries = 3;
76
+ for (let attempt = 0; attempt < maxRetries; attempt++) {
49
77
  try {
50
- const shard = await this.readShard(shardId);
51
- const currentEtag = shard.documents[docId] ?
52
- (await this.adapter.get(this.getShardPath(shardId))).etag :
53
- undefined;
78
+ // Read shard and etag in a single operation
79
+ const { shard, etag } = await this.readShard(shardId);
54
80
 
55
81
  shard.documents[docId] = { ...doc };
56
82
 
57
- await this.writeShard(shardId, shard, currentEtag);
83
+ await this.writeShard(shardId, shard, etag);
58
84
  return;
59
85
  } catch (error) {
60
86
  if (error instanceof StorageError && error.message.includes('PreconditionFailed')) {
61
- retries--;
62
- await new Promise(resolve => setTimeout(resolve, 100 * (4 - retries)));
63
- continue;
87
+ if (attempt < maxRetries - 1) {
88
+ // Exponential backoff with jitter: 100ms * 2^attempt + random(0-50ms)
89
+ const baseDelay = 100 * Math.pow(2, attempt);
90
+ const jitter = Math.random() * 50;
91
+ await new Promise(resolve => setTimeout(resolve, baseDelay + jitter));
92
+ continue;
93
+ }
64
94
  }
65
95
  throw error;
66
96
  }
@@ -81,11 +111,11 @@ export class ShardManager {
81
111
  async removeDocument(docId: string): Promise<void> {
82
112
  const shardId = getShardId(docId, this.shardCount);
83
113
 
84
- let retries = 3;
85
- while (retries > 0) {
114
+ const maxRetries = 3;
115
+ for (let attempt = 0; attempt < maxRetries; attempt++) {
86
116
  try {
87
- const { etag } = await this.adapter.get(this.getShardPath(shardId));
88
- const shard = await this.readShard(shardId);
117
+ // Read shard and etag in a single operation
118
+ const { shard, etag } = await this.readShard(shardId);
89
119
 
90
120
  delete shard.documents[docId];
91
121
 
@@ -93,9 +123,13 @@ export class ShardManager {
93
123
  return;
94
124
  } catch (error) {
95
125
  if (error instanceof StorageError && error.message.includes('PreconditionFailed')) {
96
- retries--;
97
- await new Promise(resolve => setTimeout(resolve, 100 * (4 - retries)));
98
- continue;
126
+ if (attempt < maxRetries - 1) {
127
+ // Exponential backoff with jitter
128
+ const baseDelay = 100 * Math.pow(2, attempt);
129
+ const jitter = Math.random() * 50;
130
+ await new Promise(resolve => setTimeout(resolve, baseDelay + jitter));
131
+ continue;
132
+ }
99
133
  }
100
134
  // Ignore if shard doesn't exist
101
135
  if (error instanceof StorageError && error.message.includes('not found')) {
@@ -104,6 +138,7 @@ export class ShardManager {
104
138
  throw error;
105
139
  }
106
140
  }
141
+ throw new StorageError('Failed to remove document after retries');
107
142
  }
108
143
 
109
144
  /**
@@ -111,7 +146,7 @@ export class ShardManager {
111
146
  */
112
147
  async findById(docId: string): Promise<Record<string, any> | null> {
113
148
  const shardId = getShardId(docId, this.shardCount);
114
- const shard = await this.readShard(shardId);
149
+ const { shard } = await this.readShard(shardId);
115
150
  return shard.documents[docId] || null;
116
151
  }
117
152
 
@@ -119,17 +154,37 @@ export class ShardManager {
119
154
  * Query documents matching filter
120
155
  */
121
156
  async query(filter: QueryFilter<any>): Promise<Array<Record<string, any>>> {
122
- // Read all shards in parallel
123
- const shardPromises = Array.from(
157
+ // Optimization: If filter contains id equality, target specific shard
158
+ if (filter.id && typeof filter.id === 'string') {
159
+ const targetShardId = getShardId(filter.id, this.shardCount);
160
+ const { shard } = await this.readShard(targetShardId);
161
+ const doc = shard.documents[filter.id];
162
+ if (doc && evaluateFilter(doc, filter)) {
163
+ return [doc];
164
+ }
165
+ return [];
166
+ }
167
+
168
+ // Read all shard metadata first (lightweight)
169
+ const metadataPromises = Array.from(
124
170
  { length: this.shardCount },
125
- (_, i) => this.readShard(i)
171
+ (_, i) => this.readShardMetadata(i)
126
172
  );
173
+ const allMetadata = await Promise.all(metadataPromises);
174
+
175
+ // Filter out empty shards
176
+ const nonEmptyShardIds = allMetadata
177
+ .map((meta, index) => ({ meta, index }))
178
+ .filter(({ meta }) => meta && meta.docCount > 0)
179
+ .map(({ index }) => index);
127
180
 
128
- const shards = await Promise.all(shardPromises);
181
+ // Read only non-empty shards in parallel
182
+ const shardPromises = nonEmptyShardIds.map(i => this.readShard(i));
183
+ const shardResults = await Promise.all(shardPromises);
129
184
 
130
185
  // Filter documents across all shards
131
186
  const results: Array<Record<string, any>> = [];
132
- for (const shard of shards) {
187
+ for (const { shard } of shardResults) {
133
188
  for (const doc of Object.values(shard.documents)) {
134
189
  if (evaluateFilter(doc, filter)) {
135
190
  results.push(doc);
@@ -1,5 +1,47 @@
1
1
  import type { QueryValue, QueryFilter } from '../types/index.js';
2
2
 
3
+ /**
4
+ * Allowed query operators (whitelist)
5
+ */
6
+ const ALLOWED_OPERATORS = new Set([
7
+ '$eq', '$ne', '$gt', '$gte', '$lt', '$lte', '$in', '$nin'
8
+ ]);
9
+
10
+ /**
11
+ * Reserved field names that cannot be queried (security)
12
+ */
13
+ const RESERVED_FIELDS = new Set([
14
+ '__proto__',
15
+ 'constructor',
16
+ 'prototype',
17
+ 'toString',
18
+ 'valueOf',
19
+ 'hasOwnProperty',
20
+ 'isPrototypeOf',
21
+ 'propertyIsEnumerable'
22
+ ]);
23
+
24
+ /**
25
+ * Validate query filter for security
26
+ */
27
+ function validateFilter<T extends Record<string, any>>(filter: QueryFilter<T>): void {
28
+ for (const [field, condition] of Object.entries(filter)) {
29
+ // Check for reserved field names
30
+ if (RESERVED_FIELDS.has(field)) {
31
+ throw new Error('Query contains forbidden field name');
32
+ }
33
+
34
+ // Validate operators if condition is an object
35
+ if (typeof condition === 'object' && condition !== null && !Array.isArray(condition)) {
36
+ for (const op of Object.keys(condition)) {
37
+ if (!ALLOWED_OPERATORS.has(op)) {
38
+ throw new Error('Query contains invalid operator');
39
+ }
40
+ }
41
+ }
42
+ }
43
+ }
44
+
3
45
  /**
4
46
  * Evaluates a single operator condition
5
47
  */
@@ -26,31 +68,42 @@ export function evaluateOperator<T>(
26
68
  break;
27
69
 
28
70
  case '$gt':
29
- if (!(value > operand)) return false;
30
- break;
31
-
32
71
  case '$gte':
33
- if (!(value >= operand)) return false;
34
- break;
35
-
36
72
  case '$lt':
37
- if (!(value < operand)) return false;
38
- break;
39
-
40
73
  case '$lte':
41
- if (!(value <= operand)) return false;
74
+ // Type check: only allow comparison for numbers and strings
75
+ if (typeof value !== typeof operand) {
76
+ throw new Error('Type mismatch in comparison operator');
77
+ }
78
+ if (typeof value !== 'number' && typeof value !== 'string') {
79
+ throw new Error('Comparison operators require number or string values');
80
+ }
81
+
82
+ if (op === '$gt' && !(value > operand)) return false;
83
+ if (op === '$gte' && !(value >= operand)) return false;
84
+ if (op === '$lt' && !(value < operand)) return false;
85
+ if (op === '$lte' && !(value <= operand)) return false;
42
86
  break;
43
87
 
44
88
  case '$in':
45
- if (!Array.isArray(operand) || !operand.includes(value)) return false;
89
+ if (!Array.isArray(operand)) {
90
+ throw new Error('$in operator requires array operand');
91
+ }
92
+ if (operand.length === 0) return false; // Empty array never matches
93
+ if (!operand.includes(value)) return false;
46
94
  break;
47
95
 
48
96
  case '$nin':
49
- if (!Array.isArray(operand) || operand.includes(value)) return false;
97
+ if (!Array.isArray(operand)) {
98
+ throw new Error('$nin operator requires array operand');
99
+ }
100
+ if (operand.length === 0) return true; // Empty array means nothing excluded
101
+ if (operand.includes(value)) return false;
50
102
  break;
51
103
 
52
104
  default:
53
- throw new Error(`Unknown operator: ${op}`);
105
+ // Should never reach here due to validateFilter check
106
+ throw new Error('Invalid operator');
54
107
  }
55
108
  }
56
109
 
@@ -69,15 +122,18 @@ export function evaluateFilter<T extends Record<string, any>>(
69
122
  return true;
70
123
  }
71
124
 
125
+ // Validate filter for security
126
+ validateFilter(filter);
127
+
72
128
  // All conditions must match (AND logic)
73
129
  for (const [field, condition] of Object.entries(filter)) {
74
- const value = document[field];
75
-
76
- // If field doesn't exist in document, fail the match
77
- if (value === undefined) {
130
+ // Only access own properties, not inherited ones
131
+ if (!Object.prototype.hasOwnProperty.call(document, field)) {
78
132
  return false;
79
133
  }
80
134
 
135
+ const value = document[field];
136
+
81
137
  if (!evaluateOperator(value, condition as QueryValue<any>)) {
82
138
  return false;
83
139
  }
@@ -1,4 +1,4 @@
1
- import { readFileSync, writeFileSync, unlinkSync, existsSync, mkdirSync, statSync, readdirSync } from 'fs';
1
+ import { readFileSync, writeFileSync, unlinkSync, existsSync, mkdirSync, statSync, readdirSync, renameSync } from 'fs';
2
2
  import { join, dirname, relative } from 'path';
3
3
  import { createHash } from 'crypto';
4
4
  import type {
@@ -7,6 +7,7 @@ import type {
7
7
  PutOptions,
8
8
  } from '../types/index.js';
9
9
  import { StorageError } from '../types/index.js';
10
+ import { safeKeyPath, validatePathWithinBase } from '../utils/path-validator.js';
10
11
 
11
12
  export interface FileSystemAdapterConfig {
12
13
  basePath: string;
@@ -33,11 +34,11 @@ export class FileSystemAdapter implements StorageAdapter {
33
34
  }
34
35
 
35
36
  private getFilePath(key: string): string {
36
- return join(this.basePath, key);
37
+ return safeKeyPath(this.basePath, key);
37
38
  }
38
39
 
39
40
  private getMetadataPath(key: string): string {
40
- return join(this.basePath, key + '.meta');
41
+ return safeKeyPath(this.basePath, key + '.meta');
41
42
  }
42
43
 
43
44
  private generateETag(data: any): string {
@@ -116,14 +117,32 @@ export class FileSystemAdapter implements StorageAdapter {
116
117
  // Generate ETag
117
118
  const etag = this.generateETag(data);
118
119
 
119
- // Write data file
120
- writeFileSync(filePath, JSON.stringify(data, null, 2), 'utf-8');
120
+ // Use temporary files for atomic writes
121
+ const tmpFilePath = filePath + '.tmp';
122
+ const tmpMetaPath = metaPath + '.tmp';
121
123
 
122
- // Write metadata file
123
- const metadata: FileMetadata = { etag, data };
124
- writeFileSync(metaPath, JSON.stringify(metadata), 'utf-8');
125
-
126
- return { etag };
124
+ try {
125
+ // Write to temporary files first
126
+ writeFileSync(tmpFilePath, JSON.stringify(data, null, 2), 'utf-8');
127
+
128
+ const metadata: FileMetadata = { etag, data };
129
+ writeFileSync(tmpMetaPath, JSON.stringify(metadata), 'utf-8');
130
+
131
+ // Atomic rename - both files become visible together
132
+ renameSync(tmpFilePath, filePath);
133
+ renameSync(tmpMetaPath, metaPath);
134
+
135
+ return { etag };
136
+ } catch (writeError: any) {
137
+ // Clean up temporary files on failure
138
+ try {
139
+ if (existsSync(tmpFilePath)) unlinkSync(tmpFilePath);
140
+ if (existsSync(tmpMetaPath)) unlinkSync(tmpMetaPath);
141
+ } catch {
142
+ // Ignore cleanup errors
143
+ }
144
+ throw writeError;
145
+ }
127
146
  } catch (error: any) {
128
147
  if (error instanceof StorageError) {
129
148
  throw error;
@@ -157,6 +176,13 @@ export class FileSystemAdapter implements StorageAdapter {
157
176
  const keys: string[] = [];
158
177
  const prefixPath = join(this.basePath, prefix);
159
178
 
179
+ // Validate that prefix path doesn't escape base directory
180
+ try {
181
+ validatePathWithinBase(this.basePath, prefixPath);
182
+ } catch (error) {
183
+ throw new StorageError(`Invalid prefix path: ${(error as Error).message}`);
184
+ }
185
+
160
186
  try {
161
187
  const walk = (dir: string): void => {
162
188
  if (!existsSync(dir)) {
@@ -8,7 +8,7 @@ import type {
8
8
  import { StorageError } from '../types/index.js';
9
9
 
10
10
  /**
11
- * Alibaba Cloud OSS storage adapter
11
+ * Alibaba Cloud OSS storage adapter with retry logic and timeout configuration
12
12
  */
13
13
  export class OSSAdapter implements StorageAdapter {
14
14
  private client: OSS;
@@ -25,25 +25,66 @@ export class OSSAdapter implements StorageAdapter {
25
25
  });
26
26
  }
27
27
 
28
- async get(key: string): Promise<StorageObject> {
29
- try {
30
- const result = await this.client.get(key);
28
+ /**
29
+ * Check if an error is retryable (temporary network issue or rate limit)
30
+ */
31
+ private isRetryableError(error: any): boolean {
32
+ // OSS error codes for retryable errors
33
+ const retryableCodes = ['RequestTimeout', 'ServiceUnavailable', 'Throttling', 'RequestTimeTooSkewed'];
34
+ const retryableStatusCodes = [429, 500, 502, 503, 504];
35
+
36
+ return (
37
+ retryableCodes.includes(error.code) ||
38
+ retryableStatusCodes.includes(error.status) ||
39
+ error.code === 'ETIMEDOUT' ||
40
+ error.code === 'ECONNRESET'
41
+ );
42
+ }
31
43
 
32
- return {
33
- data: JSON.parse(result.content.toString()),
34
- etag: result.res.headers.etag?.replace(/"/g, '') || '',
35
- lastModified: new Date(result.res.headers['last-modified']),
36
- };
37
- } catch (error: any) {
38
- if (error.code === 'NoSuchKey') {
39
- throw new StorageError(`Object not found: ${key}`);
44
+ /**
45
+ * Retry wrapper with exponential backoff
46
+ */
47
+ private async withRetry<T>(fn: () => Promise<T>): Promise<T> {
48
+ const maxRetries = 3;
49
+ for (let attempt = 0; attempt < maxRetries; attempt++) {
50
+ try {
51
+ return await fn();
52
+ } catch (error: any) {
53
+ if (this.isRetryableError(error) && attempt < maxRetries - 1) {
54
+ // Exponential backoff with jitter: 100ms * 2^attempt + random(0-50ms)
55
+ const baseDelay = 100 * Math.pow(2, attempt);
56
+ const jitter = Math.random() * 50;
57
+ await new Promise(resolve => setTimeout(resolve, baseDelay + jitter));
58
+ continue;
59
+ }
60
+ throw error;
40
61
  }
41
- throw new StorageError(`Failed to get object: ${error.message}`, error);
42
62
  }
63
+ throw new Error('Unreachable');
64
+ }
65
+
66
+ async get(key: string): Promise<StorageObject> {
67
+ return this.withRetry(async () => {
68
+ try {
69
+ const result = await this.client.get(key);
70
+
71
+ return {
72
+ data: JSON.parse(result.content.toString()),
73
+ etag: result.res.headers.etag?.replace(/"/g, '') || '',
74
+ lastModified: new Date(result.res.headers['last-modified']),
75
+ };
76
+ } catch (error: any) {
77
+ if (error.code === 'NoSuchKey') {
78
+ throw new StorageError(`Object not found: ${key}`);
79
+ }
80
+ throw new StorageError(`Failed to get object: ${error.message}`, error);
81
+ }
82
+ });
43
83
  }
44
84
 
45
85
  async put(key: string, data: any, options?: PutOptions): Promise<{ etag: string }> {
46
- try {
86
+ return this.withRetry(async () => {
87
+ try {
47
88
  const headers: Record<string, string> = {};
48
89
 
49
90
  if (options?.ifMatch) {
@@ -60,12 +101,13 @@ export class OSSAdapter implements StorageAdapter {
60
101
  return {
61
102
  etag: result.res.headers.etag?.replace(/"/g, '') || '',
62
103
  };
63
- } catch (error: any) {
104
+ } catch (error: any) {
64
105
  if (error.code === 'PreconditionFailed') {
65
106
  throw new StorageError('PreconditionFailed: ETag mismatch');
66
107
  }
67
108
  throw new StorageError(`Failed to put object: ${error.message}`, error);
68
109
  }
110
+ });
69
111
  }
70
112
 
71
113
  async delete(key: string): Promise<void> {
@@ -15,7 +15,7 @@ import type {
15
15
  import { StorageError } from '../types/index.js';
16
16
 
17
17
  /**
18
- * S3 storage adapter
18
+ * S3 storage adapter with retry logic and timeout configuration
19
19
  */
20
20
  export class S3Adapter implements StorageAdapter {
21
21
  private client: S3Client;
@@ -27,58 +27,105 @@ export class S3Adapter implements StorageAdapter {
27
27
  region: config.region,
28
28
  credentials: config.credentials,
29
29
  endpoint: config.endpoint,
30
+ maxAttempts: 1, // Disable SDK's built-in retry, use our custom retry logic
31
+ requestHandler: {
32
+ connectionTimeout: 3000,
33
+ requestTimeout: 3000,
34
+ },
30
35
  });
31
36
  }
32
37
 
33
- async get(key: string): Promise<StorageObject> {
34
- try {
35
- const command = new GetObjectCommand({
36
- Bucket: this.bucket,
37
- Key: key,
38
- });
39
-
40
- const response = await this.client.send(command);
41
- const body = await response.Body?.transformToString();
38
+ /**
39
+ * Check if an error is retryable (temporary network issue or rate limit)
40
+ */
41
+ private isRetryableError(error: any): boolean {
42
+ // AWS SDK error codes for retryable errors
43
+ const retryableCodes = ['RequestTimeout', 'ServiceUnavailable', 'ThrottlingException', 'RequestThrottled'];
44
+ const retryableStatusCodes = [429, 500, 502, 503, 504];
45
+
46
+ return (
47
+ retryableCodes.includes(error.name) ||
48
+ retryableStatusCodes.includes(error.$metadata?.httpStatusCode) ||
49
+ error.code === 'ETIMEDOUT' ||
50
+ error.code === 'ECONNRESET'
51
+ );
52
+ }
42
53
 
43
- if (!body) {
44
- throw new StorageError(`Empty response for key: ${key}`);
54
+ /**
55
+ * Retry wrapper with exponential backoff
56
+ */
57
+ private async withRetry<T>(fn: () => Promise<T>): Promise<T> {
58
+ const maxRetries = 3;
59
+ for (let attempt = 0; attempt < maxRetries; attempt++) {
60
+ try {
61
+ return await fn();
62
+ } catch (error: any) {
63
+ if (this.isRetryableError(error) && attempt < maxRetries - 1) {
64
+ // Exponential backoff with jitter: 100ms * 2^attempt + random(0-50ms)
65
+ const baseDelay = 100 * Math.pow(2, attempt);
66
+ const jitter = Math.random() * 50;
67
+ await new Promise(resolve => setTimeout(resolve, baseDelay + jitter));
68
+ continue;
69
+ }
70
+ throw error;
45
71
  }
72
+ }
73
+ throw new Error('Unreachable');
74
+ }
46
75
 
47
- return {
48
- data: JSON.parse(body),
49
- etag: response.ETag?.replace(/"/g, '') || '',
50
- lastModified: response.LastModified || new Date(),
51
- };
52
- } catch (error: any) {
53
- if (error.name === 'NoSuchKey') {
54
- throw new StorageError(`Object not found: ${key}`);
76
+ async get(key: string): Promise<StorageObject> {
77
+ return this.withRetry(async () => {
78
+ try {
79
+ const command = new GetObjectCommand({
80
+ Bucket: this.bucket,
81
+ Key: key,
82
+ });
83
+
84
+ const response = await this.client.send(command);
85
+ const body = await response.Body?.transformToString();
86
+
87
+ if (!body) {
88
+ throw new StorageError(`Empty response for key: ${key}`);
89
+ }
90
+
91
+ return {
92
+ data: JSON.parse(body),
93
+ etag: response.ETag?.replace(/"/g, '') || '',
94
+ lastModified: response.LastModified || new Date(),
95
+ };
96
+ } catch (error: any) {
97
+ if (error.name === 'NoSuchKey') {
98
+ throw new StorageError(`Object not found: ${key}`);
99
+ }
100
+ throw new StorageError(`Failed to get object: ${error.message}`, error);
55
101
  }
56
- throw new StorageError(`Failed to get object: ${error.message}`, error);
57
- }
102
+ });
58
103
  }
59
104
 
60
105
  async put(key: string, data: any, options?: PutOptions): Promise<{ etag: string }> {
61
- try {
62
- const command = new PutObjectCommand({
63
- Bucket: this.bucket,
64
- Key: key,
65
- Body: JSON.stringify(data),
66
- ContentType: 'application/json',
67
- IfMatch: options?.ifMatch,
68
- IfNoneMatch: options?.ifNoneMatch,
69
- });
70
-
71
- const response = await this.client.send(command);
72
-
73
- return {
74
- etag: response.ETag?.replace(/"/g, '') || '',
75
- };
76
- } catch (error: any) {
77
- if (error.name === 'PreconditionFailed') {
78
- throw new StorageError('PreconditionFailed: ETag mismatch');
106
+ return this.withRetry(async () => {
107
+ try {
108
+ const command = new PutObjectCommand({
109
+ Bucket: this.bucket,
110
+ Key: key,
111
+ Body: JSON.stringify(data),
112
+ ContentType: 'application/json',
113
+ IfMatch: options?.ifMatch,
114
+ IfNoneMatch: options?.ifNoneMatch,
115
+ });
116
+
117
+ const response = await this.client.send(command);
118
+
119
+ return {
120
+ etag: response.ETag?.replace(/"/g, '') || '',
121
+ };
122
+ } catch (error: any) {
123
+ if (error.name === 'PreconditionFailed') {
124
+ throw new StorageError('PreconditionFailed: ETag mismatch');
125
+ }
126
+ throw new StorageError(`Failed to put object: ${error.message}`, error);
79
127
  }
80
- throw new StorageError(`Failed to put object: ${error.message}`, error);
81
- }
128
+ });
82
129
  }
83
130
 
84
131
  async delete(key: string): Promise<void> {
@@ -112,17 +159,19 @@ export class S3Adapter implements StorageAdapter {
112
159
  }
113
160
 
114
161
  async listKeys(prefix: string): Promise<string[]> {
115
- try {
116
- const command = new ListObjectsV2Command({
117
- Bucket: this.bucket,
118
- Prefix: prefix,
119
- });
120
-
121
- const response = await this.client.send(command);
122
- return (response.Contents || []).map(obj => obj.Key!).filter(Boolean);
123
- } catch (error: any) {
124
- throw new StorageError(`Failed to list keys: ${error.message}`, error);
125
- }
162
+ return this.withRetry(async () => {
163
+ try {
164
+ const command = new ListObjectsV2Command({
165
+ Bucket: this.bucket,
166
+ Prefix: prefix,
167
+ });
168
+
169
+ const response = await this.client.send(command);
170
+ return (response.Contents || []).map(obj => obj.Key!).filter(Boolean);
171
+ } catch (error: any) {
172
+ throw new StorageError(`Failed to list keys: ${error.message}`, error);
173
+ }
174
+ });
126
175
  }
127
176
 
128
177
  async batchGet(keys: string[]): Promise<Map<string, StorageObject>> {
@@ -32,3 +32,12 @@ export interface IndexShard {
32
32
  shardId: string;
33
33
  documents: Record<string, any>;
34
34
  }
35
+
36
+ /**
37
+ * Shard metadata for performance optimization
38
+ */
39
+ export interface ShardMetadata {
40
+ shardId: string;
41
+ docCount: number;
42
+ lastUpdated: string;
43
+ }
@@ -24,6 +24,7 @@ export type {
24
24
  ICollection,
25
25
  CollectionMeta,
26
26
  IndexShard,
27
+ ShardMetadata,
27
28
  } from './collection.js';
28
29
 
29
30
  // Error classes
@@ -0,0 +1,119 @@
1
+ import { join, resolve, relative } from 'path';
2
+
3
+ /**
4
+ * ValidationError for path validation failures
5
+ */
6
+ export class PathValidationError extends Error {
7
+ constructor(message: string) {
8
+ super(message);
9
+ this.name = 'PathValidationError';
10
+ }
11
+ }
12
+
13
+ /**
14
+ * Sanitize and validate path components to prevent directory traversal attacks
15
+ * This is for SINGLE path components (like collection names, document IDs)
16
+ *
17
+ * @param input - Path component to sanitize
18
+ * @returns Sanitized path component
19
+ * @throws PathValidationError if validation fails
20
+ */
21
+ export function sanitizePathComponent(input: string): string {
22
+ if (!input || typeof input !== 'string') {
23
+ throw new PathValidationError('Path component must be a non-empty string');
24
+ }
25
+
26
+ // Remove any path traversal sequences
27
+ let sanitized = input.replace(/\.\./g, '');
28
+
29
+ // Remove directory separators (single components cannot have slashes)
30
+ sanitized = sanitized.replace(/[\/\\]/g, '_');
31
+
32
+ // Only allow alphanumeric, underscore, hyphen, and dot
33
+ if (!/^[a-zA-Z0-9_.-]+$/.test(sanitized)) {
34
+ throw new PathValidationError(
35
+ `Invalid characters in path component. Only alphanumeric, underscore, hyphen, and dot are allowed.`
36
+ );
37
+ }
38
+
39
+ // Prevent excessively long names
40
+ if (sanitized.length > 255) {
41
+ throw new PathValidationError('Path component exceeds maximum length of 255 characters');
42
+ }
43
+
44
+ // Prevent hidden files (starting with .)
45
+ if (sanitized.startsWith('.')) {
46
+ throw new PathValidationError('Path component cannot start with a dot');
47
+ }
48
+
49
+ return sanitized;
50
+ }
51
+
52
+ /**
53
+ * Validate that a target path is within the allowed base path
54
+ * Prevents directory traversal attacks by ensuring the resolved path
55
+ * doesn't escape the base directory
56
+ *
57
+ * @param basePath - Base directory path
58
+ * @param targetPath - Target path to validate
59
+ * @throws PathValidationError if path escapes base directory
60
+ */
61
+ export function validatePathWithinBase(basePath: string, targetPath: string): void {
62
+ const normalizedBase = resolve(basePath);
63
+ const normalizedTarget = resolve(targetPath);
64
+
65
+ const relativePath = relative(normalizedBase, normalizedTarget);
66
+
67
+ // If relative path starts with '..', it's outside the base directory
68
+ if (relativePath.startsWith('..') || resolve(normalizedBase, relativePath) !== normalizedTarget) {
69
+ throw new PathValidationError('Path traversal detected: target path is outside base directory');
70
+ }
71
+ }
72
+
73
+ /**
74
+ * Safely join path components with validation
75
+ * This is for joining SINGLE components (not full paths with slashes)
76
+ *
77
+ * @param basePath - Base directory path
78
+ * @param components - Path components to join
79
+ * @returns Validated full path
80
+ * @throws PathValidationError if validation fails
81
+ */
82
+ export function safePathJoin(basePath: string, ...components: string[]): string {
83
+ // Sanitize each component
84
+ const sanitizedComponents = components.map(sanitizePathComponent);
85
+
86
+ // Join paths
87
+ const fullPath = join(basePath, ...sanitizedComponents);
88
+
89
+ // Validate the result doesn't escape base path
90
+ validatePathWithinBase(basePath, fullPath);
91
+
92
+ return fullPath;
93
+ }
94
+
95
+ /**
96
+ * Safely join a key (which may contain slashes) with base path
97
+ * Validates the result doesn't escape base directory
98
+ *
99
+ * @param basePath - Base directory path
100
+ * @param key - Storage key (may contain path separators)
101
+ * @returns Validated full path
102
+ * @throws PathValidationError if validation fails
103
+ */
104
+ export function safeKeyPath(basePath: string, key: string): string {
105
+ if (!key || typeof key !== 'string') {
106
+ throw new PathValidationError('Key must be a non-empty string');
107
+ }
108
+
109
+ // Check for path traversal patterns
110
+ if (key.includes('..')) {
111
+ throw new PathValidationError('Key contains path traversal sequence');
112
+ }
113
+
114
+ // Join and validate
115
+ const fullPath = join(basePath, key);
116
+ validatePathWithinBase(basePath, fullPath);
117
+
118
+ return fullPath;
119
+ }