@hold-baby/bucket-db 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/cache/lru-cache.ts +131 -0
- package/src/core/bucket-db.ts +4 -0
- package/src/core/collection.ts +139 -34
- package/src/core/wal.ts +106 -0
- package/src/index/shard-manager.ts +85 -30
- package/src/query/evaluator.ts +73 -17
- package/src/storage/fs-adapter.ts +36 -10
- package/src/storage/oss-adapter.ts +57 -15
- package/src/storage/s3-adapter.ts +102 -53
- package/src/types/collection.ts +9 -0
- package/src/types/index.ts +1 -0
- package/src/utils/path-validator.ts +119 -0
package/package.json
CHANGED
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
import { createHash } from 'crypto';
|
|
2
|
+
|
|
3
|
+
interface CacheEntry<T> {
|
|
4
|
+
value: T;
|
|
5
|
+
timestamp: number;
|
|
6
|
+
}
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* LRU (Least Recently Used) Cache implementation
|
|
10
|
+
*/
|
|
11
|
+
export class LRUCache<T> {
|
|
12
|
+
private cache: Map<string, CacheEntry<T>>;
|
|
13
|
+
private maxSize: number;
|
|
14
|
+
private ttl: number; // Time To Live in milliseconds
|
|
15
|
+
|
|
16
|
+
// Statistics
|
|
17
|
+
private hits: number = 0;
|
|
18
|
+
private misses: number = 0;
|
|
19
|
+
private evictions: number = 0;
|
|
20
|
+
|
|
21
|
+
constructor(maxSize: number = 100, ttl: number = 5 * 60 * 1000) {
|
|
22
|
+
this.cache = new Map();
|
|
23
|
+
this.maxSize = maxSize;
|
|
24
|
+
this.ttl = ttl;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* Generate cache key from query filter and options
|
|
29
|
+
*/
|
|
30
|
+
static generateKey(filter: Record<string, any>, options?: Record<string, any>): string {
|
|
31
|
+
const data = JSON.stringify({ filter, options });
|
|
32
|
+
return createHash('md5').update(data).digest('hex');
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* Get value from cache
|
|
37
|
+
*/
|
|
38
|
+
get(key: string): T | null {
|
|
39
|
+
const entry = this.cache.get(key);
|
|
40
|
+
|
|
41
|
+
if (!entry) {
|
|
42
|
+
this.misses++;
|
|
43
|
+
return null;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
// Check if entry has expired
|
|
47
|
+
if (Date.now() - entry.timestamp > this.ttl) {
|
|
48
|
+
this.cache.delete(key);
|
|
49
|
+
this.misses++;
|
|
50
|
+
return null;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
// Move to end (mark as recently used)
|
|
54
|
+
this.cache.delete(key);
|
|
55
|
+
this.cache.set(key, entry);
|
|
56
|
+
this.hits++;
|
|
57
|
+
|
|
58
|
+
return entry.value;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* Put value into cache
|
|
63
|
+
*/
|
|
64
|
+
set(key: string, value: T): void {
|
|
65
|
+
// Remove if already exists (to update position)
|
|
66
|
+
if (this.cache.has(key)) {
|
|
67
|
+
this.cache.delete(key);
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
// Evict oldest entry if cache is full
|
|
71
|
+
if (this.cache.size >= this.maxSize) {
|
|
72
|
+
const firstKey = this.cache.keys().next().value;
|
|
73
|
+
if (firstKey) {
|
|
74
|
+
this.cache.delete(firstKey);
|
|
75
|
+
this.evictions++;
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
this.cache.set(key, {
|
|
80
|
+
value,
|
|
81
|
+
timestamp: Date.now(),
|
|
82
|
+
});
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/**
|
|
86
|
+
* Invalidate cache entries (clear all by default)
|
|
87
|
+
*/
|
|
88
|
+
invalidate(pattern?: RegExp): void {
|
|
89
|
+
if (!pattern) {
|
|
90
|
+
this.cache.clear();
|
|
91
|
+
return;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
// Remove entries matching pattern
|
|
95
|
+
for (const key of this.cache.keys()) {
|
|
96
|
+
if (pattern.test(key)) {
|
|
97
|
+
this.cache.delete(key);
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/**
|
|
103
|
+
* Get cache statistics
|
|
104
|
+
*/
|
|
105
|
+
getStats() {
|
|
106
|
+
const total = this.hits + this.misses;
|
|
107
|
+
return {
|
|
108
|
+
size: this.cache.size,
|
|
109
|
+
hits: this.hits,
|
|
110
|
+
misses: this.misses,
|
|
111
|
+
evictions: this.evictions,
|
|
112
|
+
hitRate: total > 0 ? (this.hits / total * 100).toFixed(2) + '%' : '0%',
|
|
113
|
+
};
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
/**
|
|
117
|
+
* Clear all cache statistics
|
|
118
|
+
*/
|
|
119
|
+
resetStats(): void {
|
|
120
|
+
this.hits = 0;
|
|
121
|
+
this.misses = 0;
|
|
122
|
+
this.evictions = 0;
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/**
|
|
126
|
+
* Clear the entire cache
|
|
127
|
+
*/
|
|
128
|
+
clear(): void {
|
|
129
|
+
this.cache.clear();
|
|
130
|
+
}
|
|
131
|
+
}
|
package/src/core/bucket-db.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { StorageAdapter, Document } from '../types/index.js';
|
|
2
2
|
import { Collection, CollectionOptions } from './collection.js';
|
|
3
|
+
import { sanitizePathComponent } from '../utils/path-validator.js';
|
|
3
4
|
|
|
4
5
|
export interface BucketDBOptions {
|
|
5
6
|
defaultShardCount?: number;
|
|
@@ -24,6 +25,9 @@ export class BucketDB {
|
|
|
24
25
|
name: string,
|
|
25
26
|
options?: CollectionOptions
|
|
26
27
|
): Collection<T> {
|
|
28
|
+
// Validate collection name to prevent path traversal
|
|
29
|
+
sanitizePathComponent(name);
|
|
30
|
+
|
|
27
31
|
if (this.collections.has(name)) {
|
|
28
32
|
return this.collections.get(name)!;
|
|
29
33
|
}
|
package/src/core/collection.ts
CHANGED
|
@@ -10,6 +10,9 @@ import type {
|
|
|
10
10
|
} from '../types/index.js';
|
|
11
11
|
import { DocumentNotFoundError, ConcurrentUpdateError } from '../types/index.js';
|
|
12
12
|
import { ShardManager } from '../index/shard-manager.js';
|
|
13
|
+
import { sanitizePathComponent, safePathJoin } from '../utils/path-validator.js';
|
|
14
|
+
import { WAL, type WALEntry } from './wal.js';
|
|
15
|
+
import { LRUCache } from '../cache/lru-cache.js';
|
|
13
16
|
|
|
14
17
|
/**
|
|
15
18
|
* Generate UUID v4
|
|
@@ -34,24 +37,59 @@ export interface CollectionOptions {
|
|
|
34
37
|
*/
|
|
35
38
|
export class Collection<T extends Document> implements ICollection<T> {
|
|
36
39
|
private shardManager: ShardManager;
|
|
40
|
+
private wal: WAL;
|
|
41
|
+
private queryCache: LRUCache<T[]>;
|
|
37
42
|
private basePath: string;
|
|
38
43
|
|
|
39
44
|
constructor(
|
|
40
45
|
private adapter: StorageAdapter,
|
|
41
|
-
|
|
42
|
-
|
|
46
|
+
dbPath: string,
|
|
47
|
+
collectionName: string,
|
|
43
48
|
options: CollectionOptions = {}
|
|
44
49
|
) {
|
|
45
|
-
|
|
50
|
+
// Validate collection name to prevent path traversal
|
|
51
|
+
const sanitizedCollectionName = sanitizePathComponent(collectionName);
|
|
52
|
+
this.basePath = safePathJoin(dbPath, sanitizedCollectionName);
|
|
46
53
|
this.shardManager = new ShardManager(
|
|
47
54
|
adapter,
|
|
48
55
|
this.basePath,
|
|
49
56
|
options.shardCount || 16
|
|
50
57
|
);
|
|
58
|
+
this.wal = new WAL(adapter, this.basePath);
|
|
59
|
+
this.queryCache = new LRUCache<T[]>(100, 5 * 60 * 1000); // 100 entries, 5 min TTL
|
|
60
|
+
|
|
61
|
+
// Recover from any pending WAL entries
|
|
62
|
+
this.recoverFromWAL().catch(err => {
|
|
63
|
+
console.error('WAL recovery failed:', err);
|
|
64
|
+
});
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* Recover from pending WAL entries
|
|
69
|
+
*/
|
|
70
|
+
private async recoverFromWAL(): Promise<void> {
|
|
71
|
+
await this.wal.recover(async (entry: WALEntry) => {
|
|
72
|
+
// Replay the operation based on WAL entry
|
|
73
|
+
switch (entry.operation) {
|
|
74
|
+
case 'insert':
|
|
75
|
+
case 'update':
|
|
76
|
+
// Ensure document is indexed
|
|
77
|
+
if (entry.documentData) {
|
|
78
|
+
await this.shardManager.addDocument(entry.documentData);
|
|
79
|
+
}
|
|
80
|
+
break;
|
|
81
|
+
case 'delete':
|
|
82
|
+
// Ensure document is removed from index
|
|
83
|
+
await this.shardManager.removeDocument(entry.documentId);
|
|
84
|
+
break;
|
|
85
|
+
}
|
|
86
|
+
});
|
|
51
87
|
}
|
|
52
88
|
|
|
53
89
|
private getDocPath(id: string): string {
|
|
54
|
-
|
|
90
|
+
// Validate document ID to prevent path traversal
|
|
91
|
+
const sanitizedId = sanitizePathComponent(id);
|
|
92
|
+
return safePathJoin(this.basePath, 'docs', `${sanitizedId}.json`);
|
|
55
93
|
}
|
|
56
94
|
|
|
57
95
|
async insert(data: InsertDocument<T>): Promise<T> {
|
|
@@ -65,15 +103,34 @@ export class Collection<T extends Document> implements ICollection<T> {
|
|
|
65
103
|
_updatedAt: timestamp,
|
|
66
104
|
} as T;
|
|
67
105
|
|
|
68
|
-
//
|
|
69
|
-
|
|
106
|
+
// 1. Log operation to WAL first
|
|
107
|
+
await this.wal.log({
|
|
108
|
+
operation: 'insert',
|
|
109
|
+
collectionPath: this.basePath,
|
|
110
|
+
documentId: id,
|
|
111
|
+
documentData: document,
|
|
112
|
+
timestamp,
|
|
113
|
+
});
|
|
114
|
+
|
|
115
|
+
try {
|
|
116
|
+
// 2. Write document to storage
|
|
117
|
+
const { etag } = await this.adapter.put(this.getDocPath(id), document);
|
|
118
|
+
document._etag = etag;
|
|
70
119
|
|
|
71
|
-
|
|
120
|
+
// 3. Add to index
|
|
121
|
+
await this.shardManager.addDocument(document);
|
|
72
122
|
|
|
73
|
-
|
|
74
|
-
|
|
123
|
+
// 4. Clear WAL entry after successful completion
|
|
124
|
+
await this.wal.clear(id);
|
|
75
125
|
|
|
76
|
-
|
|
126
|
+
// 5. Invalidate query cache (new document may match existing queries)
|
|
127
|
+
this.queryCache.invalidate();
|
|
128
|
+
|
|
129
|
+
return document;
|
|
130
|
+
} catch (error) {
|
|
131
|
+
// On failure, WAL entry remains for recovery
|
|
132
|
+
throw error;
|
|
133
|
+
}
|
|
77
134
|
}
|
|
78
135
|
|
|
79
136
|
async findById(id: string): Promise<T | null> {
|
|
@@ -88,32 +145,43 @@ export class Collection<T extends Document> implements ICollection<T> {
|
|
|
88
145
|
}
|
|
89
146
|
|
|
90
147
|
async find(filter: QueryFilter<T>, options?: QueryOptions): Promise<T[]> {
|
|
148
|
+
// Check cache first
|
|
149
|
+
const cacheKey = LRUCache.generateKey(filter as Record<string, any>, options as Record<string, any>);
|
|
150
|
+
const cached = this.queryCache.get(cacheKey);
|
|
151
|
+
if (cached) {
|
|
152
|
+
return cached;
|
|
153
|
+
}
|
|
154
|
+
|
|
91
155
|
// Query index to get matching document IDs
|
|
92
156
|
const indexResults = await this.shardManager.query(filter);
|
|
93
157
|
|
|
94
158
|
// Get full documents
|
|
95
|
-
|
|
159
|
+
let ids = indexResults.map(doc => doc.id);
|
|
96
160
|
|
|
97
161
|
if (ids.length === 0) {
|
|
98
162
|
return [];
|
|
99
163
|
}
|
|
100
164
|
|
|
101
|
-
//
|
|
165
|
+
// Apply pagination on IDs before loading documents (PERF optimization)
|
|
166
|
+
if (options?.offset) {
|
|
167
|
+
ids = ids.slice(options.offset);
|
|
168
|
+
}
|
|
169
|
+
if (options?.limit) {
|
|
170
|
+
ids = ids.slice(0, options.limit);
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
// Batch get only the required documents
|
|
102
174
|
const docMap = await this.adapter.batchGet(ids.map(id => this.getDocPath(id)));
|
|
103
175
|
|
|
104
|
-
|
|
176
|
+
// Convert to array and add etags
|
|
177
|
+
const results = Array.from(docMap.values()).map(obj => {
|
|
105
178
|
const doc = obj.data as T;
|
|
106
179
|
doc._etag = obj.etag;
|
|
107
180
|
return doc;
|
|
108
181
|
});
|
|
109
182
|
|
|
110
|
-
//
|
|
111
|
-
|
|
112
|
-
results = results.slice(options.offset);
|
|
113
|
-
}
|
|
114
|
-
if (options?.limit) {
|
|
115
|
-
results = results.slice(0, options.limit);
|
|
116
|
-
}
|
|
183
|
+
// Cache the results
|
|
184
|
+
this.queryCache.set(cacheKey, results);
|
|
117
185
|
|
|
118
186
|
return results;
|
|
119
187
|
}
|
|
@@ -142,19 +210,38 @@ export class Collection<T extends Document> implements ICollection<T> {
|
|
|
142
210
|
_updatedAt: now(),
|
|
143
211
|
} as T;
|
|
144
212
|
|
|
145
|
-
//
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
213
|
+
// 1. Log operation to WAL first
|
|
214
|
+
await this.wal.log({
|
|
215
|
+
operation: 'update',
|
|
216
|
+
collectionPath: this.basePath,
|
|
217
|
+
documentId: id,
|
|
218
|
+
documentData: updated,
|
|
219
|
+
timestamp: now(),
|
|
220
|
+
});
|
|
221
|
+
|
|
222
|
+
try {
|
|
223
|
+
// 2. Write document
|
|
224
|
+
const { etag } = await this.adapter.put(
|
|
225
|
+
this.getDocPath(id),
|
|
226
|
+
updated,
|
|
227
|
+
current._etag ? { ifMatch: current._etag } : undefined
|
|
228
|
+
);
|
|
229
|
+
|
|
230
|
+
updated._etag = etag;
|
|
231
|
+
|
|
232
|
+
// 3. Update index
|
|
233
|
+
await this.shardManager.updateDocument(updated);
|
|
151
234
|
|
|
152
|
-
|
|
235
|
+
// 4. Clear WAL entry
|
|
236
|
+
await this.wal.clear(id);
|
|
153
237
|
|
|
154
|
-
|
|
155
|
-
|
|
238
|
+
// 5. Invalidate query cache (document changed)
|
|
239
|
+
this.queryCache.invalidate();
|
|
156
240
|
|
|
157
|
-
|
|
241
|
+
return updated;
|
|
242
|
+
} catch (error) {
|
|
243
|
+
throw error;
|
|
244
|
+
}
|
|
158
245
|
}
|
|
159
246
|
|
|
160
247
|
async delete(id: string): Promise<void> {
|
|
@@ -164,10 +251,28 @@ export class Collection<T extends Document> implements ICollection<T> {
|
|
|
164
251
|
throw new DocumentNotFoundError(id);
|
|
165
252
|
}
|
|
166
253
|
|
|
167
|
-
//
|
|
168
|
-
await this.
|
|
254
|
+
// 1. Log operation to WAL first
|
|
255
|
+
await this.wal.log({
|
|
256
|
+
operation: 'delete',
|
|
257
|
+
collectionPath: this.basePath,
|
|
258
|
+
documentId: id,
|
|
259
|
+
timestamp: now(),
|
|
260
|
+
});
|
|
261
|
+
|
|
262
|
+
try {
|
|
263
|
+
// 2. Delete from storage
|
|
264
|
+
await this.adapter.delete(this.getDocPath(id));
|
|
169
265
|
|
|
170
|
-
|
|
171
|
-
|
|
266
|
+
// 3. Remove from index
|
|
267
|
+
await this.shardManager.removeDocument(id);
|
|
268
|
+
|
|
269
|
+
// 4. Clear WAL entry
|
|
270
|
+
await this.wal.clear(id);
|
|
271
|
+
|
|
272
|
+
// 5. Invalidate query cache (document deleted)
|
|
273
|
+
this.queryCache.invalidate();
|
|
274
|
+
} catch (error) {
|
|
275
|
+
throw error;
|
|
276
|
+
}
|
|
172
277
|
}
|
|
173
278
|
}
|
package/src/core/wal.ts
ADDED
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
import type { StorageAdapter } from '../types/index.js';
|
|
2
|
+
import { safePathJoin } from '../utils/path-validator.js';
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* WAL operation types
|
|
6
|
+
*/
|
|
7
|
+
export type WALOperation = 'insert' | 'update' | 'delete';
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* WAL entry structure
|
|
11
|
+
*/
|
|
12
|
+
export interface WALEntry {
|
|
13
|
+
operation: WALOperation;
|
|
14
|
+
collectionPath: string;
|
|
15
|
+
documentId: string;
|
|
16
|
+
documentData?: any;
|
|
17
|
+
timestamp: string;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* Write-Ahead Log for ensuring index-document consistency
|
|
22
|
+
*/
|
|
23
|
+
export class WAL {
|
|
24
|
+
private walPath: string;
|
|
25
|
+
|
|
26
|
+
constructor(
|
|
27
|
+
private adapter: StorageAdapter,
|
|
28
|
+
basePath: string
|
|
29
|
+
) {
|
|
30
|
+
this.walPath = safePathJoin(basePath, 'wal');
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* Get WAL entry path for a document
|
|
35
|
+
*/
|
|
36
|
+
private getWALEntryPath(documentId: string): string {
|
|
37
|
+
return `${this.walPath}/${documentId}.wal.json`;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* Log an operation before execution
|
|
42
|
+
*/
|
|
43
|
+
async log(entry: WALEntry): Promise<void> {
|
|
44
|
+
const path = this.getWALEntryPath(entry.documentId);
|
|
45
|
+
await this.adapter.put(path, entry);
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Clear WAL entry after successful operation
|
|
50
|
+
*/
|
|
51
|
+
async clear(documentId: string): Promise<void> {
|
|
52
|
+
const path = this.getWALEntryPath(documentId);
|
|
53
|
+
try {
|
|
54
|
+
await this.adapter.delete(path);
|
|
55
|
+
} catch {
|
|
56
|
+
// Ignore if entry doesn't exist
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* List all pending WAL entries
|
|
62
|
+
*/
|
|
63
|
+
async listPending(): Promise<WALEntry[]> {
|
|
64
|
+
try {
|
|
65
|
+
const keys = await this.adapter.listKeys(this.walPath);
|
|
66
|
+
const entries: WALEntry[] = [];
|
|
67
|
+
|
|
68
|
+
for (const key of keys) {
|
|
69
|
+
try {
|
|
70
|
+
const { data } = await this.adapter.get(key);
|
|
71
|
+
entries.push(data as WALEntry);
|
|
72
|
+
} catch {
|
|
73
|
+
// Skip corrupted entries
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
return entries;
|
|
78
|
+
} catch {
|
|
79
|
+
return [];
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* Recover from pending WAL entries
|
|
85
|
+
* This should be called on collection initialization
|
|
86
|
+
*/
|
|
87
|
+
async recover(
|
|
88
|
+
onRecover: (entry: WALEntry) => Promise<void>
|
|
89
|
+
): Promise<number> {
|
|
90
|
+
const pending = await this.listPending();
|
|
91
|
+
let recovered = 0;
|
|
92
|
+
|
|
93
|
+
for (const entry of pending) {
|
|
94
|
+
try {
|
|
95
|
+
await onRecover(entry);
|
|
96
|
+
await this.clear(entry.documentId);
|
|
97
|
+
recovered++;
|
|
98
|
+
} catch (error) {
|
|
99
|
+
// Log error but continue recovery
|
|
100
|
+
console.error(`Failed to recover WAL entry for ${entry.documentId}:`, error);
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
return recovered;
|
|
105
|
+
}
|
|
106
|
+
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { StorageAdapter, IndexShard, QueryFilter } from '../types/index.js';
|
|
1
|
+
import type { StorageAdapter, IndexShard, QueryFilter, ShardMetadata } from '../types/index.js';
|
|
2
2
|
import { StorageError } from '../types/index.js';
|
|
3
3
|
import { getShardId, formatShardId } from '../utils/hash.js';
|
|
4
4
|
import { evaluateFilter } from '../query/evaluator.js';
|
|
@@ -17,23 +17,51 @@ export class ShardManager {
|
|
|
17
17
|
return `${this.collectionPath}/index/shard-${formatShardId(shardId)}.json`;
|
|
18
18
|
}
|
|
19
19
|
|
|
20
|
-
private
|
|
21
|
-
|
|
20
|
+
private getShardMetadataPath(shardId: number): string {
|
|
21
|
+
return `${this.collectionPath}/index/shard-${formatShardId(shardId)}.meta.json`;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
private async readShardMetadata(shardId: number): Promise<ShardMetadata | null> {
|
|
25
|
+
const path = this.getShardMetadataPath(shardId);
|
|
22
26
|
try {
|
|
23
27
|
const { data } = await this.adapter.get(path);
|
|
24
|
-
return data as
|
|
28
|
+
return data as ShardMetadata;
|
|
29
|
+
} catch (error) {
|
|
30
|
+
return null;
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
private async readShard(shardId: number): Promise<{ shard: IndexShard; etag?: string }> {
|
|
35
|
+
const path = this.getShardPath(shardId);
|
|
36
|
+
try {
|
|
37
|
+
const { data, etag } = await this.adapter.get(path);
|
|
38
|
+
return { shard: data as IndexShard, etag };
|
|
25
39
|
} catch (error) {
|
|
26
40
|
// Return empty shard if not found
|
|
27
41
|
return {
|
|
28
|
-
|
|
29
|
-
|
|
42
|
+
shard: {
|
|
43
|
+
shardId: formatShardId(shardId),
|
|
44
|
+
documents: {},
|
|
45
|
+
},
|
|
46
|
+
etag: undefined,
|
|
30
47
|
};
|
|
31
48
|
}
|
|
32
49
|
}
|
|
33
50
|
|
|
34
51
|
private async writeShard(shardId: number, shard: IndexShard, etag?: string): Promise<void> {
|
|
35
52
|
const path = this.getShardPath(shardId);
|
|
53
|
+
const metaPath = this.getShardMetadataPath(shardId);
|
|
54
|
+
|
|
55
|
+
// Write shard data
|
|
36
56
|
await this.adapter.put(path, shard, etag ? { ifMatch: etag } : undefined);
|
|
57
|
+
|
|
58
|
+
// Write metadata
|
|
59
|
+
const metadata: ShardMetadata = {
|
|
60
|
+
shardId: formatShardId(shardId),
|
|
61
|
+
docCount: Object.keys(shard.documents).length,
|
|
62
|
+
lastUpdated: new Date().toISOString(),
|
|
63
|
+
};
|
|
64
|
+
await this.adapter.put(metaPath, metadata);
|
|
37
65
|
}
|
|
38
66
|
|
|
39
67
|
/**
|
|
@@ -43,24 +71,26 @@ export class ShardManager {
|
|
|
43
71
|
const docId = doc.id;
|
|
44
72
|
const shardId = getShardId(docId, this.shardCount);
|
|
45
73
|
|
|
46
|
-
// Retry logic for optimistic locking
|
|
47
|
-
|
|
48
|
-
|
|
74
|
+
// Retry logic for optimistic locking with exponential backoff
|
|
75
|
+
const maxRetries = 3;
|
|
76
|
+
for (let attempt = 0; attempt < maxRetries; attempt++) {
|
|
49
77
|
try {
|
|
50
|
-
|
|
51
|
-
const
|
|
52
|
-
(await this.adapter.get(this.getShardPath(shardId))).etag :
|
|
53
|
-
undefined;
|
|
78
|
+
// Read shard and etag in a single operation
|
|
79
|
+
const { shard, etag } = await this.readShard(shardId);
|
|
54
80
|
|
|
55
81
|
shard.documents[docId] = { ...doc };
|
|
56
82
|
|
|
57
|
-
await this.writeShard(shardId, shard,
|
|
83
|
+
await this.writeShard(shardId, shard, etag);
|
|
58
84
|
return;
|
|
59
85
|
} catch (error) {
|
|
60
86
|
if (error instanceof StorageError && error.message.includes('PreconditionFailed')) {
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
87
|
+
if (attempt < maxRetries - 1) {
|
|
88
|
+
// Exponential backoff with jitter: 100ms * 2^attempt + random(0-50ms)
|
|
89
|
+
const baseDelay = 100 * Math.pow(2, attempt);
|
|
90
|
+
const jitter = Math.random() * 50;
|
|
91
|
+
await new Promise(resolve => setTimeout(resolve, baseDelay + jitter));
|
|
92
|
+
continue;
|
|
93
|
+
}
|
|
64
94
|
}
|
|
65
95
|
throw error;
|
|
66
96
|
}
|
|
@@ -81,11 +111,11 @@ export class ShardManager {
|
|
|
81
111
|
async removeDocument(docId: string): Promise<void> {
|
|
82
112
|
const shardId = getShardId(docId, this.shardCount);
|
|
83
113
|
|
|
84
|
-
|
|
85
|
-
|
|
114
|
+
const maxRetries = 3;
|
|
115
|
+
for (let attempt = 0; attempt < maxRetries; attempt++) {
|
|
86
116
|
try {
|
|
87
|
-
|
|
88
|
-
const shard = await this.readShard(shardId);
|
|
117
|
+
// Read shard and etag in a single operation
|
|
118
|
+
const { shard, etag } = await this.readShard(shardId);
|
|
89
119
|
|
|
90
120
|
delete shard.documents[docId];
|
|
91
121
|
|
|
@@ -93,9 +123,13 @@ export class ShardManager {
|
|
|
93
123
|
return;
|
|
94
124
|
} catch (error) {
|
|
95
125
|
if (error instanceof StorageError && error.message.includes('PreconditionFailed')) {
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
126
|
+
if (attempt < maxRetries - 1) {
|
|
127
|
+
// Exponential backoff with jitter
|
|
128
|
+
const baseDelay = 100 * Math.pow(2, attempt);
|
|
129
|
+
const jitter = Math.random() * 50;
|
|
130
|
+
await new Promise(resolve => setTimeout(resolve, baseDelay + jitter));
|
|
131
|
+
continue;
|
|
132
|
+
}
|
|
99
133
|
}
|
|
100
134
|
// Ignore if shard doesn't exist
|
|
101
135
|
if (error instanceof StorageError && error.message.includes('not found')) {
|
|
@@ -104,6 +138,7 @@ export class ShardManager {
|
|
|
104
138
|
throw error;
|
|
105
139
|
}
|
|
106
140
|
}
|
|
141
|
+
throw new StorageError('Failed to remove document after retries');
|
|
107
142
|
}
|
|
108
143
|
|
|
109
144
|
/**
|
|
@@ -111,7 +146,7 @@ export class ShardManager {
|
|
|
111
146
|
*/
|
|
112
147
|
async findById(docId: string): Promise<Record<string, any> | null> {
|
|
113
148
|
const shardId = getShardId(docId, this.shardCount);
|
|
114
|
-
const shard = await this.readShard(shardId);
|
|
149
|
+
const { shard } = await this.readShard(shardId);
|
|
115
150
|
return shard.documents[docId] || null;
|
|
116
151
|
}
|
|
117
152
|
|
|
@@ -119,17 +154,37 @@ export class ShardManager {
|
|
|
119
154
|
* Query documents matching filter
|
|
120
155
|
*/
|
|
121
156
|
async query(filter: QueryFilter<any>): Promise<Array<Record<string, any>>> {
|
|
122
|
-
//
|
|
123
|
-
|
|
157
|
+
// Optimization: If filter contains id equality, target specific shard
|
|
158
|
+
if (filter.id && typeof filter.id === 'string') {
|
|
159
|
+
const targetShardId = getShardId(filter.id, this.shardCount);
|
|
160
|
+
const { shard } = await this.readShard(targetShardId);
|
|
161
|
+
const doc = shard.documents[filter.id];
|
|
162
|
+
if (doc && evaluateFilter(doc, filter)) {
|
|
163
|
+
return [doc];
|
|
164
|
+
}
|
|
165
|
+
return [];
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
// Read all shard metadata first (lightweight)
|
|
169
|
+
const metadataPromises = Array.from(
|
|
124
170
|
{ length: this.shardCount },
|
|
125
|
-
(_, i) => this.
|
|
171
|
+
(_, i) => this.readShardMetadata(i)
|
|
126
172
|
);
|
|
173
|
+
const allMetadata = await Promise.all(metadataPromises);
|
|
174
|
+
|
|
175
|
+
// Filter out empty shards
|
|
176
|
+
const nonEmptyShardIds = allMetadata
|
|
177
|
+
.map((meta, index) => ({ meta, index }))
|
|
178
|
+
.filter(({ meta }) => meta && meta.docCount > 0)
|
|
179
|
+
.map(({ index }) => index);
|
|
127
180
|
|
|
128
|
-
|
|
181
|
+
// Read only non-empty shards in parallel
|
|
182
|
+
const shardPromises = nonEmptyShardIds.map(i => this.readShard(i));
|
|
183
|
+
const shardResults = await Promise.all(shardPromises);
|
|
129
184
|
|
|
130
185
|
// Filter documents across all shards
|
|
131
186
|
const results: Array<Record<string, any>> = [];
|
|
132
|
-
for (const shard of
|
|
187
|
+
for (const { shard } of shardResults) {
|
|
133
188
|
for (const doc of Object.values(shard.documents)) {
|
|
134
189
|
if (evaluateFilter(doc, filter)) {
|
|
135
190
|
results.push(doc);
|
package/src/query/evaluator.ts
CHANGED
|
@@ -1,5 +1,47 @@
|
|
|
1
1
|
import type { QueryValue, QueryFilter } from '../types/index.js';
|
|
2
2
|
|
|
3
|
+
/**
|
|
4
|
+
* Allowed query operators (whitelist)
|
|
5
|
+
*/
|
|
6
|
+
const ALLOWED_OPERATORS = new Set([
|
|
7
|
+
'$eq', '$ne', '$gt', '$gte', '$lt', '$lte', '$in', '$nin'
|
|
8
|
+
]);
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* Reserved field names that cannot be queried (security)
|
|
12
|
+
*/
|
|
13
|
+
const RESERVED_FIELDS = new Set([
|
|
14
|
+
'__proto__',
|
|
15
|
+
'constructor',
|
|
16
|
+
'prototype',
|
|
17
|
+
'toString',
|
|
18
|
+
'valueOf',
|
|
19
|
+
'hasOwnProperty',
|
|
20
|
+
'isPrototypeOf',
|
|
21
|
+
'propertyIsEnumerable'
|
|
22
|
+
]);
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* Validate query filter for security
|
|
26
|
+
*/
|
|
27
|
+
function validateFilter<T extends Record<string, any>>(filter: QueryFilter<T>): void {
|
|
28
|
+
for (const [field, condition] of Object.entries(filter)) {
|
|
29
|
+
// Check for reserved field names
|
|
30
|
+
if (RESERVED_FIELDS.has(field)) {
|
|
31
|
+
throw new Error('Query contains forbidden field name');
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
// Validate operators if condition is an object
|
|
35
|
+
if (typeof condition === 'object' && condition !== null && !Array.isArray(condition)) {
|
|
36
|
+
for (const op of Object.keys(condition)) {
|
|
37
|
+
if (!ALLOWED_OPERATORS.has(op)) {
|
|
38
|
+
throw new Error('Query contains invalid operator');
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
|
|
3
45
|
/**
|
|
4
46
|
* Evaluates a single operator condition
|
|
5
47
|
*/
|
|
@@ -26,31 +68,42 @@ export function evaluateOperator<T>(
|
|
|
26
68
|
break;
|
|
27
69
|
|
|
28
70
|
case '$gt':
|
|
29
|
-
if (!(value > operand)) return false;
|
|
30
|
-
break;
|
|
31
|
-
|
|
32
71
|
case '$gte':
|
|
33
|
-
if (!(value >= operand)) return false;
|
|
34
|
-
break;
|
|
35
|
-
|
|
36
72
|
case '$lt':
|
|
37
|
-
if (!(value < operand)) return false;
|
|
38
|
-
break;
|
|
39
|
-
|
|
40
73
|
case '$lte':
|
|
41
|
-
|
|
74
|
+
// Type check: only allow comparison for numbers and strings
|
|
75
|
+
if (typeof value !== typeof operand) {
|
|
76
|
+
throw new Error('Type mismatch in comparison operator');
|
|
77
|
+
}
|
|
78
|
+
if (typeof value !== 'number' && typeof value !== 'string') {
|
|
79
|
+
throw new Error('Comparison operators require number or string values');
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
if (op === '$gt' && !(value > operand)) return false;
|
|
83
|
+
if (op === '$gte' && !(value >= operand)) return false;
|
|
84
|
+
if (op === '$lt' && !(value < operand)) return false;
|
|
85
|
+
if (op === '$lte' && !(value <= operand)) return false;
|
|
42
86
|
break;
|
|
43
87
|
|
|
44
88
|
case '$in':
|
|
45
|
-
if (!Array.isArray(operand)
|
|
89
|
+
if (!Array.isArray(operand)) {
|
|
90
|
+
throw new Error('$in operator requires array operand');
|
|
91
|
+
}
|
|
92
|
+
if (operand.length === 0) return false; // Empty array never matches
|
|
93
|
+
if (!operand.includes(value)) return false;
|
|
46
94
|
break;
|
|
47
95
|
|
|
48
96
|
case '$nin':
|
|
49
|
-
if (!Array.isArray(operand)
|
|
97
|
+
if (!Array.isArray(operand)) {
|
|
98
|
+
throw new Error('$nin operator requires array operand');
|
|
99
|
+
}
|
|
100
|
+
if (operand.length === 0) return true; // Empty array means nothing excluded
|
|
101
|
+
if (operand.includes(value)) return false;
|
|
50
102
|
break;
|
|
51
103
|
|
|
52
104
|
default:
|
|
53
|
-
|
|
105
|
+
// Should never reach here due to validateFilter check
|
|
106
|
+
throw new Error('Invalid operator');
|
|
54
107
|
}
|
|
55
108
|
}
|
|
56
109
|
|
|
@@ -69,15 +122,18 @@ export function evaluateFilter<T extends Record<string, any>>(
|
|
|
69
122
|
return true;
|
|
70
123
|
}
|
|
71
124
|
|
|
125
|
+
// Validate filter for security
|
|
126
|
+
validateFilter(filter);
|
|
127
|
+
|
|
72
128
|
// All conditions must match (AND logic)
|
|
73
129
|
for (const [field, condition] of Object.entries(filter)) {
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
// If field doesn't exist in document, fail the match
|
|
77
|
-
if (value === undefined) {
|
|
130
|
+
// Only access own properties, not inherited ones
|
|
131
|
+
if (!Object.prototype.hasOwnProperty.call(document, field)) {
|
|
78
132
|
return false;
|
|
79
133
|
}
|
|
80
134
|
|
|
135
|
+
const value = document[field];
|
|
136
|
+
|
|
81
137
|
if (!evaluateOperator(value, condition as QueryValue<any>)) {
|
|
82
138
|
return false;
|
|
83
139
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { readFileSync, writeFileSync, unlinkSync, existsSync, mkdirSync, statSync, readdirSync } from 'fs';
|
|
1
|
+
import { readFileSync, writeFileSync, unlinkSync, existsSync, mkdirSync, statSync, readdirSync, renameSync } from 'fs';
|
|
2
2
|
import { join, dirname, relative } from 'path';
|
|
3
3
|
import { createHash } from 'crypto';
|
|
4
4
|
import type {
|
|
@@ -7,6 +7,7 @@ import type {
|
|
|
7
7
|
PutOptions,
|
|
8
8
|
} from '../types/index.js';
|
|
9
9
|
import { StorageError } from '../types/index.js';
|
|
10
|
+
import { safeKeyPath, validatePathWithinBase } from '../utils/path-validator.js';
|
|
10
11
|
|
|
11
12
|
export interface FileSystemAdapterConfig {
|
|
12
13
|
basePath: string;
|
|
@@ -33,11 +34,11 @@ export class FileSystemAdapter implements StorageAdapter {
|
|
|
33
34
|
}
|
|
34
35
|
|
|
35
36
|
private getFilePath(key: string): string {
|
|
36
|
-
return
|
|
37
|
+
return safeKeyPath(this.basePath, key);
|
|
37
38
|
}
|
|
38
39
|
|
|
39
40
|
private getMetadataPath(key: string): string {
|
|
40
|
-
return
|
|
41
|
+
return safeKeyPath(this.basePath, key + '.meta');
|
|
41
42
|
}
|
|
42
43
|
|
|
43
44
|
private generateETag(data: any): string {
|
|
@@ -116,14 +117,32 @@ export class FileSystemAdapter implements StorageAdapter {
|
|
|
116
117
|
// Generate ETag
|
|
117
118
|
const etag = this.generateETag(data);
|
|
118
119
|
|
|
119
|
-
//
|
|
120
|
-
|
|
120
|
+
// Use temporary files for atomic writes
|
|
121
|
+
const tmpFilePath = filePath + '.tmp';
|
|
122
|
+
const tmpMetaPath = metaPath + '.tmp';
|
|
121
123
|
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
124
|
+
try {
|
|
125
|
+
// Write to temporary files first
|
|
126
|
+
writeFileSync(tmpFilePath, JSON.stringify(data, null, 2), 'utf-8');
|
|
127
|
+
|
|
128
|
+
const metadata: FileMetadata = { etag, data };
|
|
129
|
+
writeFileSync(tmpMetaPath, JSON.stringify(metadata), 'utf-8');
|
|
130
|
+
|
|
131
|
+
// Atomic rename - both files become visible together
|
|
132
|
+
renameSync(tmpFilePath, filePath);
|
|
133
|
+
renameSync(tmpMetaPath, metaPath);
|
|
134
|
+
|
|
135
|
+
return { etag };
|
|
136
|
+
} catch (writeError: any) {
|
|
137
|
+
// Clean up temporary files on failure
|
|
138
|
+
try {
|
|
139
|
+
if (existsSync(tmpFilePath)) unlinkSync(tmpFilePath);
|
|
140
|
+
if (existsSync(tmpMetaPath)) unlinkSync(tmpMetaPath);
|
|
141
|
+
} catch {
|
|
142
|
+
// Ignore cleanup errors
|
|
143
|
+
}
|
|
144
|
+
throw writeError;
|
|
145
|
+
}
|
|
127
146
|
} catch (error: any) {
|
|
128
147
|
if (error instanceof StorageError) {
|
|
129
148
|
throw error;
|
|
@@ -157,6 +176,13 @@ export class FileSystemAdapter implements StorageAdapter {
|
|
|
157
176
|
const keys: string[] = [];
|
|
158
177
|
const prefixPath = join(this.basePath, prefix);
|
|
159
178
|
|
|
179
|
+
// Validate that prefix path doesn't escape base directory
|
|
180
|
+
try {
|
|
181
|
+
validatePathWithinBase(this.basePath, prefixPath);
|
|
182
|
+
} catch (error) {
|
|
183
|
+
throw new StorageError(`Invalid prefix path: ${(error as Error).message}`);
|
|
184
|
+
}
|
|
185
|
+
|
|
160
186
|
try {
|
|
161
187
|
const walk = (dir: string): void => {
|
|
162
188
|
if (!existsSync(dir)) {
|
|
@@ -8,7 +8,7 @@ import type {
|
|
|
8
8
|
import { StorageError } from '../types/index.js';
|
|
9
9
|
|
|
10
10
|
/**
|
|
11
|
-
* Alibaba Cloud OSS storage adapter
|
|
11
|
+
* Alibaba Cloud OSS storage adapter with retry logic and timeout configuration
|
|
12
12
|
*/
|
|
13
13
|
export class OSSAdapter implements StorageAdapter {
|
|
14
14
|
private client: OSS;
|
|
@@ -25,25 +25,66 @@ export class OSSAdapter implements StorageAdapter {
|
|
|
25
25
|
});
|
|
26
26
|
}
|
|
27
27
|
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
28
|
+
/**
|
|
29
|
+
* Check if an error is retryable (temporary network issue or rate limit)
|
|
30
|
+
*/
|
|
31
|
+
private isRetryableError(error: any): boolean {
|
|
32
|
+
// OSS error codes for retryable errors
|
|
33
|
+
const retryableCodes = ['RequestTimeout', 'ServiceUnavailable', 'Throttling', 'RequestTimeTooSkewed'];
|
|
34
|
+
const retryableStatusCodes = [429, 500, 502, 503, 504];
|
|
35
|
+
|
|
36
|
+
return (
|
|
37
|
+
retryableCodes.includes(error.code) ||
|
|
38
|
+
retryableStatusCodes.includes(error.status) ||
|
|
39
|
+
error.code === 'ETIMEDOUT' ||
|
|
40
|
+
error.code === 'ECONNRESET'
|
|
41
|
+
);
|
|
42
|
+
}
|
|
31
43
|
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
44
|
+
/**
|
|
45
|
+
* Retry wrapper with exponential backoff
|
|
46
|
+
*/
|
|
47
|
+
private async withRetry<T>(fn: () => Promise<T>): Promise<T> {
|
|
48
|
+
const maxRetries = 3;
|
|
49
|
+
for (let attempt = 0; attempt < maxRetries; attempt++) {
|
|
50
|
+
try {
|
|
51
|
+
return await fn();
|
|
52
|
+
} catch (error: any) {
|
|
53
|
+
if (this.isRetryableError(error) && attempt < maxRetries - 1) {
|
|
54
|
+
// Exponential backoff with jitter: 100ms * 2^attempt + random(0-50ms)
|
|
55
|
+
const baseDelay = 100 * Math.pow(2, attempt);
|
|
56
|
+
const jitter = Math.random() * 50;
|
|
57
|
+
await new Promise(resolve => setTimeout(resolve, baseDelay + jitter));
|
|
58
|
+
continue;
|
|
59
|
+
}
|
|
60
|
+
throw error;
|
|
40
61
|
}
|
|
41
|
-
throw new StorageError(`Failed to get object: ${error.message}`, error);
|
|
42
62
|
}
|
|
63
|
+
throw new Error('Unreachable');
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
async get(key: string): Promise<StorageObject> {
|
|
67
|
+
return this.withRetry(async () => {
|
|
68
|
+
try {
|
|
69
|
+
const result = await this.client.get(key);
|
|
70
|
+
|
|
71
|
+
return {
|
|
72
|
+
data: JSON.parse(result.content.toString()),
|
|
73
|
+
etag: result.res.headers.etag?.replace(/"/g, '') || '',
|
|
74
|
+
lastModified: new Date(result.res.headers['last-modified']),
|
|
75
|
+
};
|
|
76
|
+
} catch (error: any) {
|
|
77
|
+
if (error.code === 'NoSuchKey') {
|
|
78
|
+
throw new StorageError(`Object not found: ${key}`);
|
|
79
|
+
}
|
|
80
|
+
throw new StorageError(`Failed to get object: ${error.message}`, error);
|
|
81
|
+
}
|
|
82
|
+
});
|
|
43
83
|
}
|
|
44
84
|
|
|
45
85
|
async put(key: string, data: any, options?: PutOptions): Promise<{ etag: string }> {
|
|
46
|
-
|
|
86
|
+
return this.withRetry(async () => {
|
|
87
|
+
try {
|
|
47
88
|
const headers: Record<string, string> = {};
|
|
48
89
|
|
|
49
90
|
if (options?.ifMatch) {
|
|
@@ -60,12 +101,13 @@ export class OSSAdapter implements StorageAdapter {
|
|
|
60
101
|
return {
|
|
61
102
|
etag: result.res.headers.etag?.replace(/"/g, '') || '',
|
|
62
103
|
};
|
|
63
|
-
|
|
104
|
+
} catch (error: any) {
|
|
64
105
|
if (error.code === 'PreconditionFailed') {
|
|
65
106
|
throw new StorageError('PreconditionFailed: ETag mismatch');
|
|
66
107
|
}
|
|
67
108
|
throw new StorageError(`Failed to put object: ${error.message}`, error);
|
|
68
109
|
}
|
|
110
|
+
});
|
|
69
111
|
}
|
|
70
112
|
|
|
71
113
|
async delete(key: string): Promise<void> {
|
|
@@ -15,7 +15,7 @@ import type {
|
|
|
15
15
|
import { StorageError } from '../types/index.js';
|
|
16
16
|
|
|
17
17
|
/**
|
|
18
|
-
* S3 storage adapter
|
|
18
|
+
* S3 storage adapter with retry logic and timeout configuration
|
|
19
19
|
*/
|
|
20
20
|
export class S3Adapter implements StorageAdapter {
|
|
21
21
|
private client: S3Client;
|
|
@@ -27,58 +27,105 @@ export class S3Adapter implements StorageAdapter {
|
|
|
27
27
|
region: config.region,
|
|
28
28
|
credentials: config.credentials,
|
|
29
29
|
endpoint: config.endpoint,
|
|
30
|
+
maxAttempts: 1, // Disable SDK's built-in retry, use our custom retry logic
|
|
31
|
+
requestHandler: {
|
|
32
|
+
connectionTimeout: 3000,
|
|
33
|
+
requestTimeout: 3000,
|
|
34
|
+
},
|
|
30
35
|
});
|
|
31
36
|
}
|
|
32
37
|
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
38
|
+
/**
|
|
39
|
+
* Check if an error is retryable (temporary network issue or rate limit)
|
|
40
|
+
*/
|
|
41
|
+
private isRetryableError(error: any): boolean {
|
|
42
|
+
// AWS SDK error codes for retryable errors
|
|
43
|
+
const retryableCodes = ['RequestTimeout', 'ServiceUnavailable', 'ThrottlingException', 'RequestThrottled'];
|
|
44
|
+
const retryableStatusCodes = [429, 500, 502, 503, 504];
|
|
45
|
+
|
|
46
|
+
return (
|
|
47
|
+
retryableCodes.includes(error.name) ||
|
|
48
|
+
retryableStatusCodes.includes(error.$metadata?.httpStatusCode) ||
|
|
49
|
+
error.code === 'ETIMEDOUT' ||
|
|
50
|
+
error.code === 'ECONNRESET'
|
|
51
|
+
);
|
|
52
|
+
}
|
|
42
53
|
|
|
43
|
-
|
|
44
|
-
|
|
54
|
+
/**
|
|
55
|
+
* Retry wrapper with exponential backoff
|
|
56
|
+
*/
|
|
57
|
+
private async withRetry<T>(fn: () => Promise<T>): Promise<T> {
|
|
58
|
+
const maxRetries = 3;
|
|
59
|
+
for (let attempt = 0; attempt < maxRetries; attempt++) {
|
|
60
|
+
try {
|
|
61
|
+
return await fn();
|
|
62
|
+
} catch (error: any) {
|
|
63
|
+
if (this.isRetryableError(error) && attempt < maxRetries - 1) {
|
|
64
|
+
// Exponential backoff with jitter: 100ms * 2^attempt + random(0-50ms)
|
|
65
|
+
const baseDelay = 100 * Math.pow(2, attempt);
|
|
66
|
+
const jitter = Math.random() * 50;
|
|
67
|
+
await new Promise(resolve => setTimeout(resolve, baseDelay + jitter));
|
|
68
|
+
continue;
|
|
69
|
+
}
|
|
70
|
+
throw error;
|
|
45
71
|
}
|
|
72
|
+
}
|
|
73
|
+
throw new Error('Unreachable');
|
|
74
|
+
}
|
|
46
75
|
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
76
|
+
async get(key: string): Promise<StorageObject> {
|
|
77
|
+
return this.withRetry(async () => {
|
|
78
|
+
try {
|
|
79
|
+
const command = new GetObjectCommand({
|
|
80
|
+
Bucket: this.bucket,
|
|
81
|
+
Key: key,
|
|
82
|
+
});
|
|
83
|
+
|
|
84
|
+
const response = await this.client.send(command);
|
|
85
|
+
const body = await response.Body?.transformToString();
|
|
86
|
+
|
|
87
|
+
if (!body) {
|
|
88
|
+
throw new StorageError(`Empty response for key: ${key}`);
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
return {
|
|
92
|
+
data: JSON.parse(body),
|
|
93
|
+
etag: response.ETag?.replace(/"/g, '') || '',
|
|
94
|
+
lastModified: response.LastModified || new Date(),
|
|
95
|
+
};
|
|
96
|
+
} catch (error: any) {
|
|
97
|
+
if (error.name === 'NoSuchKey') {
|
|
98
|
+
throw new StorageError(`Object not found: ${key}`);
|
|
99
|
+
}
|
|
100
|
+
throw new StorageError(`Failed to get object: ${error.message}`, error);
|
|
55
101
|
}
|
|
56
|
-
|
|
57
|
-
}
|
|
102
|
+
});
|
|
58
103
|
}
|
|
59
104
|
|
|
60
105
|
async put(key: string, data: any, options?: PutOptions): Promise<{ etag: string }> {
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
106
|
+
return this.withRetry(async () => {
|
|
107
|
+
try {
|
|
108
|
+
const command = new PutObjectCommand({
|
|
109
|
+
Bucket: this.bucket,
|
|
110
|
+
Key: key,
|
|
111
|
+
Body: JSON.stringify(data),
|
|
112
|
+
ContentType: 'application/json',
|
|
113
|
+
IfMatch: options?.ifMatch,
|
|
114
|
+
IfNoneMatch: options?.ifNoneMatch,
|
|
115
|
+
});
|
|
116
|
+
|
|
117
|
+
const response = await this.client.send(command);
|
|
118
|
+
|
|
119
|
+
return {
|
|
120
|
+
etag: response.ETag?.replace(/"/g, '') || '',
|
|
121
|
+
};
|
|
122
|
+
} catch (error: any) {
|
|
123
|
+
if (error.name === 'PreconditionFailed') {
|
|
124
|
+
throw new StorageError('PreconditionFailed: ETag mismatch');
|
|
125
|
+
}
|
|
126
|
+
throw new StorageError(`Failed to put object: ${error.message}`, error);
|
|
79
127
|
}
|
|
80
|
-
|
|
81
|
-
}
|
|
128
|
+
});
|
|
82
129
|
}
|
|
83
130
|
|
|
84
131
|
async delete(key: string): Promise<void> {
|
|
@@ -112,17 +159,19 @@ export class S3Adapter implements StorageAdapter {
|
|
|
112
159
|
}
|
|
113
160
|
|
|
114
161
|
async listKeys(prefix: string): Promise<string[]> {
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
162
|
+
return this.withRetry(async () => {
|
|
163
|
+
try {
|
|
164
|
+
const command = new ListObjectsV2Command({
|
|
165
|
+
Bucket: this.bucket,
|
|
166
|
+
Prefix: prefix,
|
|
167
|
+
});
|
|
168
|
+
|
|
169
|
+
const response = await this.client.send(command);
|
|
170
|
+
return (response.Contents || []).map(obj => obj.Key!).filter(Boolean);
|
|
171
|
+
} catch (error: any) {
|
|
172
|
+
throw new StorageError(`Failed to list keys: ${error.message}`, error);
|
|
173
|
+
}
|
|
174
|
+
});
|
|
126
175
|
}
|
|
127
176
|
|
|
128
177
|
async batchGet(keys: string[]): Promise<Map<string, StorageObject>> {
|
package/src/types/collection.ts
CHANGED
|
@@ -32,3 +32,12 @@ export interface IndexShard {
|
|
|
32
32
|
shardId: string;
|
|
33
33
|
documents: Record<string, any>;
|
|
34
34
|
}
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Shard metadata for performance optimization
|
|
38
|
+
*/
|
|
39
|
+
export interface ShardMetadata {
|
|
40
|
+
shardId: string;
|
|
41
|
+
docCount: number;
|
|
42
|
+
lastUpdated: string;
|
|
43
|
+
}
|
package/src/types/index.ts
CHANGED
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
import { join, resolve, relative } from 'path';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* ValidationError for path validation failures
|
|
5
|
+
*/
|
|
6
|
+
export class PathValidationError extends Error {
|
|
7
|
+
constructor(message: string) {
|
|
8
|
+
super(message);
|
|
9
|
+
this.name = 'PathValidationError';
|
|
10
|
+
}
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* Sanitize and validate path components to prevent directory traversal attacks
|
|
15
|
+
* This is for SINGLE path components (like collection names, document IDs)
|
|
16
|
+
*
|
|
17
|
+
* @param input - Path component to sanitize
|
|
18
|
+
* @returns Sanitized path component
|
|
19
|
+
* @throws PathValidationError if validation fails
|
|
20
|
+
*/
|
|
21
|
+
export function sanitizePathComponent(input: string): string {
|
|
22
|
+
if (!input || typeof input !== 'string') {
|
|
23
|
+
throw new PathValidationError('Path component must be a non-empty string');
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
// Remove any path traversal sequences
|
|
27
|
+
let sanitized = input.replace(/\.\./g, '');
|
|
28
|
+
|
|
29
|
+
// Remove directory separators (single components cannot have slashes)
|
|
30
|
+
sanitized = sanitized.replace(/[\/\\]/g, '_');
|
|
31
|
+
|
|
32
|
+
// Only allow alphanumeric, underscore, hyphen, and dot
|
|
33
|
+
if (!/^[a-zA-Z0-9_.-]+$/.test(sanitized)) {
|
|
34
|
+
throw new PathValidationError(
|
|
35
|
+
`Invalid characters in path component. Only alphanumeric, underscore, hyphen, and dot are allowed.`
|
|
36
|
+
);
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
// Prevent excessively long names
|
|
40
|
+
if (sanitized.length > 255) {
|
|
41
|
+
throw new PathValidationError('Path component exceeds maximum length of 255 characters');
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
// Prevent hidden files (starting with .)
|
|
45
|
+
if (sanitized.startsWith('.')) {
|
|
46
|
+
throw new PathValidationError('Path component cannot start with a dot');
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
return sanitized;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* Validate that a target path is within the allowed base path
|
|
54
|
+
* Prevents directory traversal attacks by ensuring the resolved path
|
|
55
|
+
* doesn't escape the base directory
|
|
56
|
+
*
|
|
57
|
+
* @param basePath - Base directory path
|
|
58
|
+
* @param targetPath - Target path to validate
|
|
59
|
+
* @throws PathValidationError if path escapes base directory
|
|
60
|
+
*/
|
|
61
|
+
export function validatePathWithinBase(basePath: string, targetPath: string): void {
|
|
62
|
+
const normalizedBase = resolve(basePath);
|
|
63
|
+
const normalizedTarget = resolve(targetPath);
|
|
64
|
+
|
|
65
|
+
const relativePath = relative(normalizedBase, normalizedTarget);
|
|
66
|
+
|
|
67
|
+
// If relative path starts with '..', it's outside the base directory
|
|
68
|
+
if (relativePath.startsWith('..') || resolve(normalizedBase, relativePath) !== normalizedTarget) {
|
|
69
|
+
throw new PathValidationError('Path traversal detected: target path is outside base directory');
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* Safely join path components with validation
|
|
75
|
+
* This is for joining SINGLE components (not full paths with slashes)
|
|
76
|
+
*
|
|
77
|
+
* @param basePath - Base directory path
|
|
78
|
+
* @param components - Path components to join
|
|
79
|
+
* @returns Validated full path
|
|
80
|
+
* @throws PathValidationError if validation fails
|
|
81
|
+
*/
|
|
82
|
+
export function safePathJoin(basePath: string, ...components: string[]): string {
|
|
83
|
+
// Sanitize each component
|
|
84
|
+
const sanitizedComponents = components.map(sanitizePathComponent);
|
|
85
|
+
|
|
86
|
+
// Join paths
|
|
87
|
+
const fullPath = join(basePath, ...sanitizedComponents);
|
|
88
|
+
|
|
89
|
+
// Validate the result doesn't escape base path
|
|
90
|
+
validatePathWithinBase(basePath, fullPath);
|
|
91
|
+
|
|
92
|
+
return fullPath;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* Safely join a key (which may contain slashes) with base path
|
|
97
|
+
* Validates the result doesn't escape base directory
|
|
98
|
+
*
|
|
99
|
+
* @param basePath - Base directory path
|
|
100
|
+
* @param key - Storage key (may contain path separators)
|
|
101
|
+
* @returns Validated full path
|
|
102
|
+
* @throws PathValidationError if validation fails
|
|
103
|
+
*/
|
|
104
|
+
export function safeKeyPath(basePath: string, key: string): string {
|
|
105
|
+
if (!key || typeof key !== 'string') {
|
|
106
|
+
throw new PathValidationError('Key must be a non-empty string');
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
// Check for path traversal patterns
|
|
110
|
+
if (key.includes('..')) {
|
|
111
|
+
throw new PathValidationError('Key contains path traversal sequence');
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
// Join and validate
|
|
115
|
+
const fullPath = join(basePath, key);
|
|
116
|
+
validatePathWithinBase(basePath, fullPath);
|
|
117
|
+
|
|
118
|
+
return fullPath;
|
|
119
|
+
}
|