@hold-baby/bucket-db 0.3.0 → 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (141) hide show
  1. package/README.md +234 -19
  2. package/dist/cache/lru-cache.d.ts +47 -0
  3. package/dist/cache/lru-cache.d.ts.map +1 -0
  4. package/dist/cache/lru-cache.js +110 -0
  5. package/dist/cache/lru-cache.js.map +1 -0
  6. package/dist/core/bucket-db.d.ts +20 -0
  7. package/dist/core/bucket-db.d.ts.map +1 -0
  8. package/dist/core/bucket-db.js +32 -0
  9. package/dist/core/bucket-db.js.map +1 -0
  10. package/dist/core/collection.d.ts +26 -0
  11. package/dist/core/collection.d.ts.map +1 -0
  12. package/dist/core/collection.js +213 -0
  13. package/dist/core/collection.js.map +1 -0
  14. package/dist/core/wal.d.ts +45 -0
  15. package/dist/core/wal.d.ts.map +1 -0
  16. package/dist/core/wal.js +80 -0
  17. package/dist/core/wal.js.map +1 -0
  18. package/dist/index/shard-manager.d.ts +36 -0
  19. package/dist/index/shard-manager.d.ts.map +1 -0
  20. package/dist/index/shard-manager.js +177 -0
  21. package/dist/index/shard-manager.js.map +1 -0
  22. package/dist/index.d.ts +9 -0
  23. package/dist/index.d.ts.map +1 -0
  24. package/dist/index.js +10 -0
  25. package/dist/index.js.map +1 -0
  26. package/dist/query/evaluator.d.ts +10 -0
  27. package/dist/query/evaluator.d.ts.map +1 -0
  28. package/dist/query/evaluator.js +127 -0
  29. package/dist/query/evaluator.js.map +1 -0
  30. package/dist/storage/fs-adapter.d.ts +24 -0
  31. package/dist/storage/fs-adapter.d.ts.map +1 -0
  32. package/dist/storage/fs-adapter.js +208 -0
  33. package/dist/storage/fs-adapter.js.map +1 -0
  34. package/dist/storage/memory-adapter.d.ts +22 -0
  35. package/dist/storage/memory-adapter.d.ts.map +1 -0
  36. package/dist/storage/memory-adapter.js +78 -0
  37. package/dist/storage/memory-adapter.js.map +1 -0
  38. package/dist/storage/oss-adapter.d.ts +26 -0
  39. package/dist/storage/oss-adapter.d.ts.map +1 -0
  40. package/dist/storage/oss-adapter.js +145 -0
  41. package/dist/storage/oss-adapter.js.map +1 -0
  42. package/dist/storage/s3-adapter.d.ts +26 -0
  43. package/dist/storage/s3-adapter.d.ts.map +1 -0
  44. package/dist/storage/s3-adapter.js +170 -0
  45. package/dist/storage/s3-adapter.js.map +1 -0
  46. package/dist/types/collection.d.ts +40 -0
  47. package/dist/types/collection.d.ts.map +1 -0
  48. package/dist/types/collection.js +2 -0
  49. package/dist/types/collection.js.map +1 -0
  50. package/{src/types/document.ts → dist/types/document.d.ts} +6 -10
  51. package/dist/types/document.d.ts.map +1 -0
  52. package/dist/types/document.js +2 -0
  53. package/dist/types/document.js.map +1 -0
  54. package/dist/types/errors.d.ts +32 -0
  55. package/dist/types/errors.d.ts.map +1 -0
  56. package/dist/types/errors.js +48 -0
  57. package/dist/types/errors.js.map +1 -0
  58. package/dist/types/index.d.ts +6 -0
  59. package/dist/types/index.d.ts.map +1 -0
  60. package/dist/types/index.js +3 -0
  61. package/dist/types/index.js.map +1 -0
  62. package/dist/types/query.d.ts +37 -0
  63. package/dist/types/query.d.ts.map +1 -0
  64. package/dist/types/query.js +2 -0
  65. package/dist/types/query.js.map +1 -0
  66. package/dist/types/storage.d.ts +45 -0
  67. package/dist/types/storage.d.ts.map +1 -0
  68. package/dist/types/storage.js +2 -0
  69. package/dist/types/storage.js.map +1 -0
  70. package/dist/utils/hash.d.ts +13 -0
  71. package/dist/utils/hash.d.ts.map +1 -0
  72. package/dist/utils/hash.js +23 -0
  73. package/dist/utils/hash.js.map +1 -0
  74. package/dist/utils/path-validator.d.ts +46 -0
  75. package/dist/utils/path-validator.d.ts.map +1 -0
  76. package/dist/utils/path-validator.js +99 -0
  77. package/dist/utils/path-validator.js.map +1 -0
  78. package/llms.txt +46 -0
  79. package/package.json +14 -3
  80. package/src/cache/lru-cache.ts +0 -131
  81. package/src/core/bucket-db.ts +0 -47
  82. package/src/core/collection.ts +0 -278
  83. package/src/core/wal.ts +0 -106
  84. package/src/index/shard-manager.ts +0 -197
  85. package/src/index.ts +0 -37
  86. package/src/query/evaluator.ts +0 -143
  87. package/src/storage/ali-oss.d.ts +0 -43
  88. package/src/storage/fs-adapter.ts +0 -248
  89. package/src/storage/memory-adapter.ts +0 -105
  90. package/src/storage/oss-adapter.ts +0 -164
  91. package/src/storage/s3-adapter.ts +0 -200
  92. package/src/types/collection.ts +0 -43
  93. package/src/types/errors.ts +0 -49
  94. package/src/types/index.ts +0 -37
  95. package/src/types/query.ts +0 -50
  96. package/src/types/storage.ts +0 -46
  97. package/src/utils/hash.ts +0 -24
  98. package/src/utils/path-validator.ts +0 -119
  99. package/tests/core/bucket-db.test.d.ts +0 -2
  100. package/tests/core/bucket-db.test.d.ts.map +0 -1
  101. package/tests/core/bucket-db.test.js +0 -34
  102. package/tests/core/bucket-db.test.js.map +0 -1
  103. package/tests/core/bucket-db.test.ts +0 -50
  104. package/tests/core/collection.test.d.ts +0 -2
  105. package/tests/core/collection.test.d.ts.map +0 -1
  106. package/tests/core/collection.test.js +0 -148
  107. package/tests/core/collection.test.js.map +0 -1
  108. package/tests/core/collection.test.ts +0 -195
  109. package/tests/core/dbpath.test.ts +0 -52
  110. package/tests/index/shard-manager.test.d.ts +0 -2
  111. package/tests/index/shard-manager.test.d.ts.map +0 -1
  112. package/tests/index/shard-manager.test.js +0 -106
  113. package/tests/index/shard-manager.test.js.map +0 -1
  114. package/tests/index/shard-manager.test.ts +0 -129
  115. package/tests/query/evaluator.test.d.ts +0 -2
  116. package/tests/query/evaluator.test.d.ts.map +0 -1
  117. package/tests/query/evaluator.test.js +0 -87
  118. package/tests/query/evaluator.test.js.map +0 -1
  119. package/tests/query/evaluator.test.ts +0 -104
  120. package/tests/storage/fs-adapter.test.ts +0 -111
  121. package/tests/storage/memory-adapter.test.d.ts +0 -2
  122. package/tests/storage/memory-adapter.test.d.ts.map +0 -1
  123. package/tests/storage/memory-adapter.test.js +0 -65
  124. package/tests/storage/memory-adapter.test.js.map +0 -1
  125. package/tests/storage/memory-adapter.test.ts +0 -84
  126. package/tests/storage/oss-adapter.test.d.ts +0 -2
  127. package/tests/storage/oss-adapter.test.d.ts.map +0 -1
  128. package/tests/storage/oss-adapter.test.js +0 -34
  129. package/tests/storage/oss-adapter.test.js.map +0 -1
  130. package/tests/storage/oss-adapter.test.ts +0 -40
  131. package/tests/storage/s3-adapter.test.d.ts +0 -2
  132. package/tests/storage/s3-adapter.test.d.ts.map +0 -1
  133. package/tests/storage/s3-adapter.test.js +0 -51
  134. package/tests/storage/s3-adapter.test.js.map +0 -1
  135. package/tests/storage/s3-adapter.test.ts +0 -62
  136. package/tests/utils/hash.test.d.ts +0 -2
  137. package/tests/utils/hash.test.d.ts.map +0 -1
  138. package/tests/utils/hash.test.js +0 -36
  139. package/tests/utils/hash.test.js.map +0 -1
  140. package/tests/utils/hash.test.ts +0 -42
  141. package/tsconfig.json +0 -12
@@ -1,248 +0,0 @@
1
- import { readFileSync, writeFileSync, unlinkSync, existsSync, mkdirSync, statSync, readdirSync, renameSync } from 'fs';
2
- import { join, dirname, relative } from 'path';
3
- import { createHash } from 'crypto';
4
- import type {
5
- StorageAdapter,
6
- StorageObject,
7
- PutOptions,
8
- } from '../types/index.js';
9
- import { StorageError } from '../types/index.js';
10
- import { safeKeyPath, validatePathWithinBase } from '../utils/path-validator.js';
11
-
12
- export interface FileSystemAdapterConfig {
13
- basePath: string;
14
- }
15
-
16
- interface FileMetadata {
17
- etag: string;
18
- data: any;
19
- }
20
-
21
- /**
22
- * Local file system storage adapter
23
- */
24
- export class FileSystemAdapter implements StorageAdapter {
25
- private basePath: string;
26
-
27
- constructor(config: FileSystemAdapterConfig) {
28
- this.basePath = config.basePath;
29
-
30
- // Ensure base directory exists
31
- if (!existsSync(this.basePath)) {
32
- mkdirSync(this.basePath, { recursive: true });
33
- }
34
- }
35
-
36
- private getFilePath(key: string): string {
37
- return safeKeyPath(this.basePath, key);
38
- }
39
-
40
- private getMetadataPath(key: string): string {
41
- return safeKeyPath(this.basePath, key + '.meta');
42
- }
43
-
44
- private generateETag(data: any): string {
45
- const content = JSON.stringify(data);
46
- return createHash('md5').update(content).digest('hex');
47
- }
48
-
49
- private ensureDirectoryExists(filePath: string): void {
50
- const dir = dirname(filePath);
51
- if (!existsSync(dir)) {
52
- mkdirSync(dir, { recursive: true });
53
- }
54
- }
55
-
56
- async get(key: string): Promise<StorageObject> {
57
- const filePath = this.getFilePath(key);
58
- const metaPath = this.getMetadataPath(key);
59
-
60
- try {
61
- if (!existsSync(filePath)) {
62
- throw new StorageError(`Object not found: ${key}`);
63
- }
64
-
65
- const content = readFileSync(filePath, 'utf-8');
66
- const data = JSON.parse(content);
67
-
68
- let etag: string;
69
- if (existsSync(metaPath)) {
70
- const metadata: FileMetadata = JSON.parse(readFileSync(metaPath, 'utf-8'));
71
- etag = metadata.etag;
72
- } else {
73
- etag = this.generateETag(data);
74
- }
75
-
76
- const stats = statSync(filePath);
77
-
78
- return {
79
- data,
80
- etag,
81
- lastModified: stats.mtime,
82
- };
83
- } catch (error: any) {
84
- if (error instanceof StorageError) {
85
- throw error;
86
- }
87
- throw new StorageError(`Failed to get object: ${error.message}`, error);
88
- }
89
- }
90
-
91
- async put(key: string, data: any, options?: PutOptions): Promise<{ etag: string }> {
92
- const filePath = this.getFilePath(key);
93
- const metaPath = this.getMetadataPath(key);
94
-
95
- try {
96
- // Check conditional writes
97
- if (options?.ifMatch || options?.ifNoneMatch) {
98
- if (existsSync(metaPath)) {
99
- const currentMeta: FileMetadata = JSON.parse(readFileSync(metaPath, 'utf-8'));
100
-
101
- if (options.ifMatch && currentMeta.etag !== options.ifMatch) {
102
- throw new StorageError('PreconditionFailed: ETag mismatch');
103
- }
104
-
105
- if (options.ifNoneMatch && existsSync(filePath)) {
106
- throw new StorageError('PreconditionFailed: Object already exists');
107
- }
108
- } else if (options.ifMatch) {
109
- // ifMatch provided but no existing metadata
110
- throw new StorageError('PreconditionFailed: ETag mismatch');
111
- }
112
- }
113
-
114
- // Ensure directory exists
115
- this.ensureDirectoryExists(filePath);
116
-
117
- // Generate ETag
118
- const etag = this.generateETag(data);
119
-
120
- // Use temporary files for atomic writes
121
- const tmpFilePath = filePath + '.tmp';
122
- const tmpMetaPath = metaPath + '.tmp';
123
-
124
- try {
125
- // Write to temporary files first
126
- writeFileSync(tmpFilePath, JSON.stringify(data, null, 2), 'utf-8');
127
-
128
- const metadata: FileMetadata = { etag, data };
129
- writeFileSync(tmpMetaPath, JSON.stringify(metadata), 'utf-8');
130
-
131
- // Atomic rename - both files become visible together
132
- renameSync(tmpFilePath, filePath);
133
- renameSync(tmpMetaPath, metaPath);
134
-
135
- return { etag };
136
- } catch (writeError: any) {
137
- // Clean up temporary files on failure
138
- try {
139
- if (existsSync(tmpFilePath)) unlinkSync(tmpFilePath);
140
- if (existsSync(tmpMetaPath)) unlinkSync(tmpMetaPath);
141
- } catch {
142
- // Ignore cleanup errors
143
- }
144
- throw writeError;
145
- }
146
- } catch (error: any) {
147
- if (error instanceof StorageError) {
148
- throw error;
149
- }
150
- throw new StorageError(`Failed to put object: ${error.message}`, error);
151
- }
152
- }
153
-
154
- async delete(key: string): Promise<void> {
155
- const filePath = this.getFilePath(key);
156
- const metaPath = this.getMetadataPath(key);
157
-
158
- try {
159
- if (existsSync(filePath)) {
160
- unlinkSync(filePath);
161
- }
162
- if (existsSync(metaPath)) {
163
- unlinkSync(metaPath);
164
- }
165
- } catch (error: any) {
166
- throw new StorageError(`Failed to delete object: ${error.message}`, error);
167
- }
168
- }
169
-
170
- async exists(key: string): Promise<boolean> {
171
- const filePath = this.getFilePath(key);
172
- return existsSync(filePath);
173
- }
174
-
175
- async listKeys(prefix: string): Promise<string[]> {
176
- const keys: string[] = [];
177
- const prefixPath = join(this.basePath, prefix);
178
-
179
- // Validate that prefix path doesn't escape base directory
180
- try {
181
- validatePathWithinBase(this.basePath, prefixPath);
182
- } catch (error) {
183
- throw new StorageError(`Invalid prefix path: ${(error as Error).message}`);
184
- }
185
-
186
- try {
187
- const walk = (dir: string): void => {
188
- if (!existsSync(dir)) {
189
- return;
190
- }
191
-
192
- const files = readdirSync(dir, { withFileTypes: true });
193
-
194
- for (const file of files) {
195
- const fullPath = join(dir, file.name);
196
-
197
- if (file.isDirectory()) {
198
- walk(fullPath);
199
- } else if (file.isFile() && !file.name.endsWith('.meta')) {
200
- // Get relative path from base
201
- const relativePath = relative(this.basePath, fullPath);
202
-
203
- // Check if it starts with the prefix
204
- if (relativePath.startsWith(prefix)) {
205
- keys.push(relativePath);
206
- }
207
- }
208
- }
209
- };
210
-
211
- // If prefix path exists as a directory, walk it
212
- if (existsSync(prefixPath) && statSync(prefixPath).isDirectory()) {
213
- walk(prefixPath);
214
- } else {
215
- // Walk from base and filter by prefix
216
- walk(this.basePath);
217
- }
218
-
219
- return keys.sort();
220
- } catch (error: any) {
221
- throw new StorageError(`Failed to list keys: ${error.message}`, error);
222
- }
223
- }
224
-
225
- async batchGet(keys: string[]): Promise<Map<string, StorageObject>> {
226
- const results = new Map<string, StorageObject>();
227
-
228
- // Fetch in parallel
229
- const promises = keys.map(async (key) => {
230
- try {
231
- const obj = await this.get(key);
232
- return { key, obj };
233
- } catch (error) {
234
- return null;
235
- }
236
- });
237
-
238
- const settled = await Promise.all(promises);
239
-
240
- for (const result of settled) {
241
- if (result) {
242
- results.set(result.key, result.obj);
243
- }
244
- }
245
-
246
- return results;
247
- }
248
- }
@@ -1,105 +0,0 @@
1
- import type {
2
- StorageAdapter,
3
- StorageObject,
4
- PutOptions,
5
- } from '../types/index.js';
6
- import { StorageError } from '../types/index.js';
7
-
8
- interface StorageEntry {
9
- data: any;
10
- etag: string;
11
- lastModified: Date;
12
- }
13
-
14
- /**
15
- * In-memory storage adapter for testing
16
- */
17
- export class MemoryStorageAdapter implements StorageAdapter {
18
- private storage = new Map<string, StorageEntry>();
19
- private etagCounter = 0;
20
-
21
- private generateETag(): string {
22
- return `etag-${++this.etagCounter}-${Date.now()}`;
23
- }
24
-
25
- async get(key: string): Promise<StorageObject> {
26
- const entry = this.storage.get(key);
27
- if (!entry) {
28
- throw new StorageError(`Object not found: ${key}`);
29
- }
30
- return {
31
- data: JSON.parse(JSON.stringify(entry.data)), // Deep clone
32
- etag: entry.etag,
33
- lastModified: entry.lastModified,
34
- };
35
- }
36
-
37
- async put(
38
- key: string,
39
- data: any,
40
- options?: PutOptions
41
- ): Promise<{ etag: string }> {
42
- const existing = this.storage.get(key);
43
-
44
- // Check ifMatch condition
45
- if (options?.ifMatch) {
46
- if (!existing || existing.etag !== options.ifMatch) {
47
- throw new StorageError('PreconditionFailed: ETag mismatch');
48
- }
49
- }
50
-
51
- // Check ifNoneMatch condition
52
- if (options?.ifNoneMatch && existing) {
53
- throw new StorageError('PreconditionFailed: Object already exists');
54
- }
55
-
56
- const etag = this.generateETag();
57
- this.storage.set(key, {
58
- data: JSON.parse(JSON.stringify(data)), // Deep clone
59
- etag,
60
- lastModified: new Date(),
61
- });
62
-
63
- return { etag };
64
- }
65
-
66
- async delete(key: string): Promise<void> {
67
- this.storage.delete(key);
68
- }
69
-
70
- async exists(key: string): Promise<boolean> {
71
- return this.storage.has(key);
72
- }
73
-
74
- async listKeys(prefix: string): Promise<string[]> {
75
- const keys: string[] = [];
76
- for (const key of this.storage.keys()) {
77
- if (key.startsWith(prefix)) {
78
- keys.push(key);
79
- }
80
- }
81
- return keys.sort();
82
- }
83
-
84
- async batchGet(keys: string[]): Promise<Map<string, StorageObject>> {
85
- const results = new Map<string, StorageObject>();
86
- for (const key of keys) {
87
- const entry = this.storage.get(key);
88
- if (entry) {
89
- results.set(key, {
90
- data: JSON.parse(JSON.stringify(entry.data)),
91
- etag: entry.etag,
92
- lastModified: entry.lastModified,
93
- });
94
- }
95
- }
96
- return results;
97
- }
98
-
99
- /**
100
- * Clear all data (for testing)
101
- */
102
- clear(): void {
103
- this.storage.clear();
104
- }
105
- }
@@ -1,164 +0,0 @@
1
- import OSS from 'ali-oss';
2
- import type {
3
- StorageAdapter,
4
- StorageObject,
5
- PutOptions,
6
- StorageAdapterConfig,
7
- } from '../types/index.js';
8
- import { StorageError } from '../types/index.js';
9
-
10
- /**
11
- * Alibaba Cloud OSS storage adapter with retry logic and timeout configuration
12
- */
13
- export class OSSAdapter implements StorageAdapter {
14
- private client: OSS;
15
- private bucket: string;
16
-
17
- constructor(config: StorageAdapterConfig) {
18
- this.bucket = config.bucket;
19
- this.client = new OSS({
20
- region: config.region,
21
- accessKeyId: config.credentials.accessKeyId,
22
- accessKeySecret: config.credentials.secretAccessKey,
23
- bucket: config.bucket,
24
- endpoint: config.endpoint,
25
- });
26
- }
27
-
28
- /**
29
- * Check if an error is retryable (temporary network issue or rate limit)
30
- */
31
- private isRetryableError(error: any): boolean {
32
- // OSS error codes for retryable errors
33
- const retryableCodes = ['RequestTimeout', 'ServiceUnavailable', 'Throttling', 'RequestTimeTooSkewed'];
34
- const retryableStatusCodes = [429, 500, 502, 503, 504];
35
-
36
- return (
37
- retryableCodes.includes(error.code) ||
38
- retryableStatusCodes.includes(error.status) ||
39
- error.code === 'ETIMEDOUT' ||
40
- error.code === 'ECONNRESET'
41
- );
42
- }
43
-
44
- /**
45
- * Retry wrapper with exponential backoff
46
- */
47
- private async withRetry<T>(fn: () => Promise<T>): Promise<T> {
48
- const maxRetries = 3;
49
- for (let attempt = 0; attempt < maxRetries; attempt++) {
50
- try {
51
- return await fn();
52
- } catch (error: any) {
53
- if (this.isRetryableError(error) && attempt < maxRetries - 1) {
54
- // Exponential backoff with jitter: 100ms * 2^attempt + random(0-50ms)
55
- const baseDelay = 100 * Math.pow(2, attempt);
56
- const jitter = Math.random() * 50;
57
- await new Promise(resolve => setTimeout(resolve, baseDelay + jitter));
58
- continue;
59
- }
60
- throw error;
61
- }
62
- }
63
- throw new Error('Unreachable');
64
- }
65
-
66
- async get(key: string): Promise<StorageObject> {
67
- return this.withRetry(async () => {
68
- try {
69
- const result = await this.client.get(key);
70
-
71
- return {
72
- data: JSON.parse(result.content.toString()),
73
- etag: result.res.headers.etag?.replace(/"/g, '') || '',
74
- lastModified: new Date(result.res.headers['last-modified']),
75
- };
76
- } catch (error: any) {
77
- if (error.code === 'NoSuchKey') {
78
- throw new StorageError(`Object not found: ${key}`);
79
- }
80
- throw new StorageError(`Failed to get object: ${error.message}`, error);
81
- }
82
- });
83
- }
84
-
85
- async put(key: string, data: any, options?: PutOptions): Promise<{ etag: string }> {
86
- return this.withRetry(async () => {
87
- try {
88
- const headers: Record<string, string> = {};
89
-
90
- if (options?.ifMatch) {
91
- headers['If-Match'] = options.ifMatch;
92
- }
93
- if (options?.ifNoneMatch) {
94
- headers['If-None-Match'] = options.ifNoneMatch;
95
- }
96
-
97
- const result = await this.client.put(key, Buffer.from(JSON.stringify(data)), {
98
- headers,
99
- });
100
-
101
- return {
102
- etag: result.res.headers.etag?.replace(/"/g, '') || '',
103
- };
104
- } catch (error: any) {
105
- if (error.code === 'PreconditionFailed') {
106
- throw new StorageError('PreconditionFailed: ETag mismatch');
107
- }
108
- throw new StorageError(`Failed to put object: ${error.message}`, error);
109
- }
110
- });
111
- }
112
-
113
- async delete(key: string): Promise<void> {
114
- try {
115
- await this.client.delete(key);
116
- } catch (error: any) {
117
- throw new StorageError(`Failed to delete object: ${error.message}`, error);
118
- }
119
- }
120
-
121
- async exists(key: string): Promise<boolean> {
122
- try {
123
- await this.client.head(key);
124
- return true;
125
- } catch (error: any) {
126
- if (error.code === 'NoSuchKey') {
127
- return false;
128
- }
129
- throw new StorageError(`Failed to check existence: ${error.message}`, error);
130
- }
131
- }
132
-
133
- async listKeys(prefix: string): Promise<string[]> {
134
- try {
135
- const result = await this.client.list({ prefix });
136
- return (result.objects || []).map(obj => obj.name);
137
- } catch (error: any) {
138
- throw new StorageError(`Failed to list keys: ${error.message}`, error);
139
- }
140
- }
141
-
142
- async batchGet(keys: string[]): Promise<Map<string, StorageObject>> {
143
- const results = new Map<string, StorageObject>();
144
-
145
- const promises = keys.map(async (key) => {
146
- try {
147
- const obj = await this.get(key);
148
- return { key, obj };
149
- } catch (error) {
150
- return null;
151
- }
152
- });
153
-
154
- const settled = await Promise.all(promises);
155
-
156
- for (const result of settled) {
157
- if (result) {
158
- results.set(result.key, result.obj);
159
- }
160
- }
161
-
162
- return results;
163
- }
164
- }
@@ -1,200 +0,0 @@
1
- import {
2
- S3Client,
3
- GetObjectCommand,
4
- PutObjectCommand,
5
- DeleteObjectCommand,
6
- HeadObjectCommand,
7
- ListObjectsV2Command,
8
- } from '@aws-sdk/client-s3';
9
- import type {
10
- StorageAdapter,
11
- StorageObject,
12
- PutOptions,
13
- StorageAdapterConfig,
14
- } from '../types/index.js';
15
- import { StorageError } from '../types/index.js';
16
-
17
- /**
18
- * S3 storage adapter with retry logic and timeout configuration
19
- */
20
- export class S3Adapter implements StorageAdapter {
21
- private client: S3Client;
22
- private bucket: string;
23
-
24
- constructor(config: StorageAdapterConfig) {
25
- this.bucket = config.bucket;
26
- this.client = new S3Client({
27
- region: config.region,
28
- credentials: config.credentials,
29
- endpoint: config.endpoint,
30
- maxAttempts: 1, // Disable SDK's built-in retry, use our custom retry logic
31
- requestHandler: {
32
- connectionTimeout: 3000,
33
- requestTimeout: 3000,
34
- },
35
- });
36
- }
37
-
38
- /**
39
- * Check if an error is retryable (temporary network issue or rate limit)
40
- */
41
- private isRetryableError(error: any): boolean {
42
- // AWS SDK error codes for retryable errors
43
- const retryableCodes = ['RequestTimeout', 'ServiceUnavailable', 'ThrottlingException', 'RequestThrottled'];
44
- const retryableStatusCodes = [429, 500, 502, 503, 504];
45
-
46
- return (
47
- retryableCodes.includes(error.name) ||
48
- retryableStatusCodes.includes(error.$metadata?.httpStatusCode) ||
49
- error.code === 'ETIMEDOUT' ||
50
- error.code === 'ECONNRESET'
51
- );
52
- }
53
-
54
- /**
55
- * Retry wrapper with exponential backoff
56
- */
57
- private async withRetry<T>(fn: () => Promise<T>): Promise<T> {
58
- const maxRetries = 3;
59
- for (let attempt = 0; attempt < maxRetries; attempt++) {
60
- try {
61
- return await fn();
62
- } catch (error: any) {
63
- if (this.isRetryableError(error) && attempt < maxRetries - 1) {
64
- // Exponential backoff with jitter: 100ms * 2^attempt + random(0-50ms)
65
- const baseDelay = 100 * Math.pow(2, attempt);
66
- const jitter = Math.random() * 50;
67
- await new Promise(resolve => setTimeout(resolve, baseDelay + jitter));
68
- continue;
69
- }
70
- throw error;
71
- }
72
- }
73
- throw new Error('Unreachable');
74
- }
75
-
76
- async get(key: string): Promise<StorageObject> {
77
- return this.withRetry(async () => {
78
- try {
79
- const command = new GetObjectCommand({
80
- Bucket: this.bucket,
81
- Key: key,
82
- });
83
-
84
- const response = await this.client.send(command);
85
- const body = await response.Body?.transformToString();
86
-
87
- if (!body) {
88
- throw new StorageError(`Empty response for key: ${key}`);
89
- }
90
-
91
- return {
92
- data: JSON.parse(body),
93
- etag: response.ETag?.replace(/"/g, '') || '',
94
- lastModified: response.LastModified || new Date(),
95
- };
96
- } catch (error: any) {
97
- if (error.name === 'NoSuchKey') {
98
- throw new StorageError(`Object not found: ${key}`);
99
- }
100
- throw new StorageError(`Failed to get object: ${error.message}`, error);
101
- }
102
- });
103
- }
104
-
105
- async put(key: string, data: any, options?: PutOptions): Promise<{ etag: string }> {
106
- return this.withRetry(async () => {
107
- try {
108
- const command = new PutObjectCommand({
109
- Bucket: this.bucket,
110
- Key: key,
111
- Body: JSON.stringify(data),
112
- ContentType: 'application/json',
113
- IfMatch: options?.ifMatch,
114
- IfNoneMatch: options?.ifNoneMatch,
115
- });
116
-
117
- const response = await this.client.send(command);
118
-
119
- return {
120
- etag: response.ETag?.replace(/"/g, '') || '',
121
- };
122
- } catch (error: any) {
123
- if (error.name === 'PreconditionFailed') {
124
- throw new StorageError('PreconditionFailed: ETag mismatch');
125
- }
126
- throw new StorageError(`Failed to put object: ${error.message}`, error);
127
- }
128
- });
129
- }
130
-
131
- async delete(key: string): Promise<void> {
132
- try {
133
- const command = new DeleteObjectCommand({
134
- Bucket: this.bucket,
135
- Key: key,
136
- });
137
-
138
- await this.client.send(command);
139
- } catch (error: any) {
140
- throw new StorageError(`Failed to delete object: ${error.message}`, error);
141
- }
142
- }
143
-
144
- async exists(key: string): Promise<boolean> {
145
- try {
146
- const command = new HeadObjectCommand({
147
- Bucket: this.bucket,
148
- Key: key,
149
- });
150
-
151
- await this.client.send(command);
152
- return true;
153
- } catch (error: any) {
154
- if (error.name === 'NotFound' || error.name === 'NoSuchKey') {
155
- return false;
156
- }
157
- throw new StorageError(`Failed to check existence: ${error.message}`, error);
158
- }
159
- }
160
-
161
- async listKeys(prefix: string): Promise<string[]> {
162
- return this.withRetry(async () => {
163
- try {
164
- const command = new ListObjectsV2Command({
165
- Bucket: this.bucket,
166
- Prefix: prefix,
167
- });
168
-
169
- const response = await this.client.send(command);
170
- return (response.Contents || []).map(obj => obj.Key!).filter(Boolean);
171
- } catch (error: any) {
172
- throw new StorageError(`Failed to list keys: ${error.message}`, error);
173
- }
174
- });
175
- }
176
-
177
- async batchGet(keys: string[]): Promise<Map<string, StorageObject>> {
178
- const results = new Map<string, StorageObject>();
179
-
180
- // Fetch in parallel
181
- const promises = keys.map(async (key) => {
182
- try {
183
- const obj = await this.get(key);
184
- return { key, obj };
185
- } catch (error) {
186
- return null;
187
- }
188
- });
189
-
190
- const settled = await Promise.all(promises);
191
-
192
- for (const result of settled) {
193
- if (result) {
194
- results.set(result.key, result.obj);
195
- }
196
- }
197
-
198
- return results;
199
- }
200
- }