@agirails/sdk 2.0.4 → 2.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +536 -87
- package/dist/adapters/BasicAdapter.d.ts.map +1 -1
- package/dist/adapters/BasicAdapter.js +8 -0
- package/dist/adapters/BasicAdapter.js.map +1 -1
- package/dist/adapters/StandardAdapter.d.ts +10 -5
- package/dist/adapters/StandardAdapter.d.ts.map +1 -1
- package/dist/adapters/StandardAdapter.js +19 -6
- package/dist/adapters/StandardAdapter.js.map +1 -1
- package/dist/config/networks.d.ts +9 -0
- package/dist/config/networks.d.ts.map +1 -1
- package/dist/config/networks.js +25 -10
- package/dist/config/networks.js.map +1 -1
- package/dist/index.d.ts +6 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +31 -1
- package/dist/index.js.map +1 -1
- package/dist/level0/provide.d.ts.map +1 -1
- package/dist/level0/provide.js +2 -1
- package/dist/level0/provide.js.map +1 -1
- package/dist/level1/Agent.d.ts.map +1 -1
- package/dist/level1/Agent.js +11 -3
- package/dist/level1/Agent.js.map +1 -1
- package/dist/protocol/ACTPKernel.d.ts.map +1 -1
- package/dist/protocol/ACTPKernel.js +7 -5
- package/dist/protocol/ACTPKernel.js.map +1 -1
- package/dist/protocol/DIDResolver.js +1 -1
- package/dist/protocol/DIDResolver.js.map +1 -1
- package/dist/protocol/EASHelper.d.ts.map +1 -1
- package/dist/protocol/EASHelper.js +2 -3
- package/dist/protocol/EASHelper.js.map +1 -1
- package/dist/protocol/MessageSigner.d.ts.map +1 -1
- package/dist/protocol/MessageSigner.js +8 -8
- package/dist/protocol/MessageSigner.js.map +1 -1
- package/dist/runtime/BlockchainRuntime.d.ts +7 -0
- package/dist/runtime/BlockchainRuntime.d.ts.map +1 -1
- package/dist/runtime/BlockchainRuntime.js +38 -22
- package/dist/runtime/BlockchainRuntime.js.map +1 -1
- package/dist/runtime/IACTPRuntime.d.ts +15 -0
- package/dist/runtime/IACTPRuntime.d.ts.map +1 -1
- package/dist/runtime/MockRuntime.d.ts +7 -0
- package/dist/runtime/MockRuntime.d.ts.map +1 -1
- package/dist/runtime/MockRuntime.js +15 -4
- package/dist/runtime/MockRuntime.js.map +1 -1
- package/dist/runtime/types/MockState.d.ts +5 -2
- package/dist/runtime/types/MockState.d.ts.map +1 -1
- package/dist/runtime/types/MockState.js.map +1 -1
- package/dist/storage/ArchiveBundleBuilder.d.ts +150 -0
- package/dist/storage/ArchiveBundleBuilder.d.ts.map +1 -0
- package/dist/storage/ArchiveBundleBuilder.js +468 -0
- package/dist/storage/ArchiveBundleBuilder.js.map +1 -0
- package/dist/storage/ArweaveClient.d.ts +271 -0
- package/dist/storage/ArweaveClient.d.ts.map +1 -0
- package/dist/storage/ArweaveClient.js +761 -0
- package/dist/storage/ArweaveClient.js.map +1 -0
- package/dist/storage/FilebaseClient.d.ts +193 -0
- package/dist/storage/FilebaseClient.d.ts.map +1 -0
- package/dist/storage/FilebaseClient.js +643 -0
- package/dist/storage/FilebaseClient.js.map +1 -0
- package/dist/storage/index.d.ts +47 -0
- package/dist/storage/index.d.ts.map +1 -0
- package/dist/storage/index.js +64 -0
- package/dist/storage/index.js.map +1 -0
- package/dist/storage/types.d.ts +291 -0
- package/dist/storage/types.d.ts.map +1 -0
- package/dist/storage/types.js +18 -0
- package/dist/storage/types.js.map +1 -0
- package/dist/types/state.d.ts +5 -4
- package/dist/types/state.d.ts.map +1 -1
- package/dist/types/state.js +10 -9
- package/dist/types/state.js.map +1 -1
- package/dist/utils/IPFSClient.d.ts.map +1 -1
- package/dist/utils/IPFSClient.js +5 -2
- package/dist/utils/IPFSClient.js.map +1 -1
- package/dist/utils/NonceManager.d.ts.map +1 -1
- package/dist/utils/NonceManager.js +3 -2
- package/dist/utils/NonceManager.js.map +1 -1
- package/dist/utils/UsedAttestationTracker.d.ts.map +1 -1
- package/dist/utils/UsedAttestationTracker.js +3 -3
- package/dist/utils/UsedAttestationTracker.js.map +1 -1
- package/dist/utils/circuitBreaker.d.ts +136 -0
- package/dist/utils/circuitBreaker.d.ts.map +1 -0
- package/dist/utils/circuitBreaker.js +253 -0
- package/dist/utils/circuitBreaker.js.map +1 -0
- package/dist/utils/retry.d.ts +120 -0
- package/dist/utils/retry.d.ts.map +1 -0
- package/dist/utils/retry.js +260 -0
- package/dist/utils/retry.js.map +1 -0
- package/dist/utils/validation.d.ts +100 -0
- package/dist/utils/validation.d.ts.map +1 -1
- package/dist/utils/validation.js +248 -1
- package/dist/utils/validation.js.map +1 -1
- package/package.json +14 -2
- package/src/adapters/BasicAdapter.ts +11 -0
- package/src/adapters/StandardAdapter.ts +26 -6
- package/src/config/networks.ts +34 -10
- package/src/index.ts +54 -0
- package/src/level0/provide.ts +2 -1
- package/src/level1/Agent.ts +13 -3
- package/src/protocol/ACTPKernel.ts +7 -5
- package/src/protocol/DIDResolver.ts +1 -1
- package/src/protocol/EASHelper.ts +2 -5
- package/src/protocol/MessageSigner.ts +8 -14
- package/src/runtime/BlockchainRuntime.ts +39 -45
- package/src/runtime/IACTPRuntime.ts +16 -0
- package/src/runtime/MockRuntime.ts +16 -4
- package/src/runtime/types/MockState.ts +5 -2
- package/src/storage/ArchiveBundleBuilder.ts +563 -0
- package/src/storage/ArweaveClient.ts +945 -0
- package/src/storage/FilebaseClient.ts +790 -0
- package/src/storage/index.ts +96 -0
- package/src/storage/types.ts +348 -0
- package/src/types/state.ts +10 -9
- package/src/utils/IPFSClient.ts +5 -4
- package/src/utils/NonceManager.ts +3 -2
- package/src/utils/UsedAttestationTracker.ts +3 -5
- package/src/utils/circuitBreaker.ts +324 -0
- package/src/utils/fsSafe.ts +5 -0
- package/src/utils/retry.ts +365 -0
- package/src/utils/validation.ts +295 -1
|
@@ -0,0 +1,790 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* FilebaseClient - IPFS Storage via Filebase S3 API (AIP-7 §4.2)
|
|
3
|
+
*
|
|
4
|
+
* Provides S3-compatible IPFS pinning via Filebase.
|
|
5
|
+
* Used for hot storage of request metadata and delivery proofs.
|
|
6
|
+
*
|
|
7
|
+
* Security Features (Post-Audit):
|
|
8
|
+
* - Gateway URL whitelist (SSRF protection)
|
|
9
|
+
* - Download size limits (DoS protection)
|
|
10
|
+
* - Credential sanitization in errors
|
|
11
|
+
* - Retry with exponential backoff
|
|
12
|
+
* - Circuit breaker for gateway health tracking
|
|
13
|
+
*
|
|
14
|
+
* @module storage/FilebaseClient
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
import { S3Client, PutObjectCommand, GetObjectCommand, HeadObjectCommand } from '@aws-sdk/client-s3';
|
|
18
|
+
import {
|
|
19
|
+
StorageError,
|
|
20
|
+
StorageAuthenticationError,
|
|
21
|
+
StorageRateLimitError,
|
|
22
|
+
UploadTimeoutError,
|
|
23
|
+
DownloadTimeoutError,
|
|
24
|
+
ContentNotFoundError,
|
|
25
|
+
FileSizeLimitExceededError,
|
|
26
|
+
ValidationError
|
|
27
|
+
} from '../errors';
|
|
28
|
+
import {
|
|
29
|
+
FilebaseConfig,
|
|
30
|
+
IPFSUploadResult,
|
|
31
|
+
DownloadResult
|
|
32
|
+
} from './types';
|
|
33
|
+
import {
|
|
34
|
+
validateCID,
|
|
35
|
+
validateGatewayURL,
|
|
36
|
+
sanitizeErrorMessage,
|
|
37
|
+
ALLOWED_IPFS_GATEWAYS
|
|
38
|
+
} from '../utils/validation';
|
|
39
|
+
import { withRetry, RetryOptions } from '../utils/retry';
|
|
40
|
+
import {
|
|
41
|
+
GatewayCircuitBreaker,
|
|
42
|
+
CircuitBreakerConfig,
|
|
43
|
+
globalCircuitBreaker
|
|
44
|
+
} from '../utils/circuitBreaker';
|
|
45
|
+
|
|
46
|
+
// ============================================================================
|
|
47
|
+
// Constants
|
|
48
|
+
// ============================================================================
|
|
49
|
+
|
|
50
|
+
const DEFAULT_ENDPOINT = 'https://s3.filebase.com';
|
|
51
|
+
const DEFAULT_GATEWAY = 'https://ipfs.filebase.io/ipfs/';
|
|
52
|
+
const DEFAULT_BUCKET = 'agirails-storage';
|
|
53
|
+
const DEFAULT_TIMEOUT = 30000; // 30 seconds
|
|
54
|
+
const DEFAULT_MAX_FILE_SIZE = 100 * 1024 * 1024; // 100MB
|
|
55
|
+
const DEFAULT_MAX_DOWNLOAD_SIZE = 50 * 1024 * 1024; // 50MB (P1-1: DoS protection)
|
|
56
|
+
const DEFAULT_REGION = 'us-east-1';
|
|
57
|
+
|
|
58
|
+
// ============================================================================
|
|
59
|
+
// FilebaseClient Class
|
|
60
|
+
// ============================================================================
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* FilebaseClient - IPFS storage via Filebase S3 API
|
|
64
|
+
*
|
|
65
|
+
* Provides hot storage for:
|
|
66
|
+
* - AIP-1 request metadata
|
|
67
|
+
* - AIP-4 delivery proofs
|
|
68
|
+
* - Service descriptors
|
|
69
|
+
*
|
|
70
|
+
* @example
|
|
71
|
+
* ```typescript
|
|
72
|
+
* const client = new FilebaseClient({
|
|
73
|
+
* accessKey: process.env.FILEBASE_ACCESS_KEY!,
|
|
74
|
+
* secretKey: process.env.FILEBASE_SECRET_KEY!,
|
|
75
|
+
* bucket: 'my-agirails-bucket'
|
|
76
|
+
* });
|
|
77
|
+
*
|
|
78
|
+
* // Upload JSON
|
|
79
|
+
* const result = await client.uploadJSON({ message: 'Hello IPFS!' });
|
|
80
|
+
* console.log('CID:', result.cid);
|
|
81
|
+
*
|
|
82
|
+
* // Download JSON
|
|
83
|
+
* const data = await client.downloadJSON(result.cid);
|
|
84
|
+
* ```
|
|
85
|
+
*/
|
|
86
|
+
export class FilebaseClient {
|
|
87
|
+
private readonly s3: S3Client;
|
|
88
|
+
private readonly bucket: string;
|
|
89
|
+
private readonly gatewayUrl: string;
|
|
90
|
+
private readonly timeout: number;
|
|
91
|
+
private readonly maxFileSize: number;
|
|
92
|
+
private readonly maxDownloadSize: number;
|
|
93
|
+
private readonly retryOptions: RetryOptions;
|
|
94
|
+
private readonly circuitBreaker: GatewayCircuitBreaker | null;
|
|
95
|
+
private readonly circuitBreakerEnabled: boolean;
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* Create a new FilebaseClient
|
|
99
|
+
*
|
|
100
|
+
* @param config - Filebase configuration
|
|
101
|
+
* @throws {ValidationError} If required config is missing or gateway URL not whitelisted
|
|
102
|
+
*/
|
|
103
|
+
constructor(config: FilebaseConfig) {
|
|
104
|
+
// Validate required config
|
|
105
|
+
if (!config.accessKey) {
|
|
106
|
+
throw new ValidationError('accessKey', 'Filebase access key is required');
|
|
107
|
+
}
|
|
108
|
+
if (!config.secretKey) {
|
|
109
|
+
throw new ValidationError('secretKey', 'Filebase secret key is required');
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
// P0-1: Validate gateway URL against whitelist (SSRF protection)
|
|
113
|
+
const gatewayUrl = config.gatewayUrl || DEFAULT_GATEWAY;
|
|
114
|
+
validateGatewayURL(gatewayUrl, ALLOWED_IPFS_GATEWAYS, 'gatewayUrl');
|
|
115
|
+
|
|
116
|
+
// Initialize S3 client
|
|
117
|
+
this.s3 = new S3Client({
|
|
118
|
+
endpoint: config.endpoint || DEFAULT_ENDPOINT,
|
|
119
|
+
region: DEFAULT_REGION,
|
|
120
|
+
credentials: {
|
|
121
|
+
accessKeyId: config.accessKey,
|
|
122
|
+
secretAccessKey: config.secretKey
|
|
123
|
+
},
|
|
124
|
+
forcePathStyle: true // Required for S3-compatible services
|
|
125
|
+
});
|
|
126
|
+
|
|
127
|
+
this.bucket = config.bucket || DEFAULT_BUCKET;
|
|
128
|
+
this.gatewayUrl = gatewayUrl;
|
|
129
|
+
this.timeout = config.timeout || DEFAULT_TIMEOUT;
|
|
130
|
+
this.maxFileSize = config.maxFileSize || DEFAULT_MAX_FILE_SIZE;
|
|
131
|
+
this.maxDownloadSize = config.maxDownloadSize || DEFAULT_MAX_DOWNLOAD_SIZE;
|
|
132
|
+
|
|
133
|
+
// P1-2: Default retry options
|
|
134
|
+
this.retryOptions = {
|
|
135
|
+
maxAttempts: 3,
|
|
136
|
+
initialDelayMs: 1000,
|
|
137
|
+
maxDelayMs: 10000,
|
|
138
|
+
backoffMultiplier: 2
|
|
139
|
+
};
|
|
140
|
+
|
|
141
|
+
// Circuit breaker for gateway health tracking
|
|
142
|
+
this.circuitBreakerEnabled = config.circuitBreaker?.enabled !== false;
|
|
143
|
+
if (this.circuitBreakerEnabled) {
|
|
144
|
+
this.circuitBreaker = new GatewayCircuitBreaker({
|
|
145
|
+
failureThreshold: config.circuitBreaker?.failureThreshold,
|
|
146
|
+
resetTimeoutMs: config.circuitBreaker?.resetTimeoutMs,
|
|
147
|
+
failureWindowMs: config.circuitBreaker?.failureWindowMs,
|
|
148
|
+
successThreshold: config.circuitBreaker?.successThreshold
|
|
149
|
+
});
|
|
150
|
+
} else {
|
|
151
|
+
this.circuitBreaker = null;
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
// ==========================================================================
|
|
156
|
+
// Public Methods
|
|
157
|
+
// ==========================================================================
|
|
158
|
+
|
|
159
|
+
/**
|
|
160
|
+
* Upload JSON to IPFS via Filebase (automatic pinning)
|
|
161
|
+
*
|
|
162
|
+
* @param data - JSON-serializable data to upload
|
|
163
|
+
* @param options - Upload options
|
|
164
|
+
* @returns Upload result with CID
|
|
165
|
+
* @throws {FileSizeLimitExceededError} If data exceeds max size
|
|
166
|
+
* @throws {StorageAuthenticationError} If credentials are invalid
|
|
167
|
+
* @throws {UploadTimeoutError} If upload times out
|
|
168
|
+
* @throws {StorageError} If upload fails
|
|
169
|
+
*
|
|
170
|
+
* @example
|
|
171
|
+
* ```typescript
|
|
172
|
+
* const result = await client.uploadJSON({
|
|
173
|
+
* version: '1.0.0',
|
|
174
|
+
* serviceType: 'text-generation',
|
|
175
|
+
* inputData: { prompt: 'Hello!' }
|
|
176
|
+
* });
|
|
177
|
+
* console.log('Uploaded to IPFS:', result.cid);
|
|
178
|
+
* ```
|
|
179
|
+
*/
|
|
180
|
+
async uploadJSON(
|
|
181
|
+
data: unknown,
|
|
182
|
+
options?: { key?: string; metadata?: Record<string, string> }
|
|
183
|
+
): Promise<IPFSUploadResult> {
|
|
184
|
+
// Serialize to JSON
|
|
185
|
+
const jsonString = JSON.stringify(data);
|
|
186
|
+
const buffer = Buffer.from(jsonString, 'utf-8');
|
|
187
|
+
|
|
188
|
+
// Check file size
|
|
189
|
+
if (buffer.length > this.maxFileSize) {
|
|
190
|
+
throw new FileSizeLimitExceededError(buffer.length, this.maxFileSize);
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
// Generate unique key if not provided
|
|
194
|
+
const key = options?.key || this.generateKey('.json');
|
|
195
|
+
|
|
196
|
+
try {
|
|
197
|
+
// Upload to Filebase (auto-pins to IPFS)
|
|
198
|
+
const command = new PutObjectCommand({
|
|
199
|
+
Bucket: this.bucket,
|
|
200
|
+
Key: key,
|
|
201
|
+
Body: buffer,
|
|
202
|
+
ContentType: 'application/json',
|
|
203
|
+
Metadata: options?.metadata
|
|
204
|
+
});
|
|
205
|
+
|
|
206
|
+
const response = await this.withTimeout(
|
|
207
|
+
this.s3.send(command),
|
|
208
|
+
this.timeout,
|
|
209
|
+
'upload'
|
|
210
|
+
);
|
|
211
|
+
|
|
212
|
+
// Extract CID from response headers
|
|
213
|
+
// Filebase returns CID in x-amz-meta-cid header
|
|
214
|
+
const cid = (response as any).$metadata?.httpStatusCode === 200
|
|
215
|
+
? await this.getCIDFromKey(key)
|
|
216
|
+
: undefined;
|
|
217
|
+
|
|
218
|
+
if (!cid) {
|
|
219
|
+
throw new StorageError('upload', 'Failed to retrieve CID after upload');
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
return {
|
|
223
|
+
cid,
|
|
224
|
+
size: buffer.length,
|
|
225
|
+
uploadedAt: new Date()
|
|
226
|
+
};
|
|
227
|
+
} catch (error: any) {
|
|
228
|
+
// Handle specific errors
|
|
229
|
+
if (error instanceof FileSizeLimitExceededError) throw error;
|
|
230
|
+
if (error instanceof UploadTimeoutError) throw error;
|
|
231
|
+
|
|
232
|
+
if (error.name === 'CredentialsProviderError' || error.Code === 'InvalidAccessKeyId') {
|
|
233
|
+
throw new StorageAuthenticationError('Filebase');
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
if (error.Code === 'SlowDown' || error.statusCode === 429) {
|
|
237
|
+
const retryAfter = error.$metadata?.httpHeaders?.['retry-after'];
|
|
238
|
+
throw new StorageRateLimitError(retryAfter ? parseInt(retryAfter, 10) : undefined);
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
// P0-2: Sanitize error message (may contain AWS credentials)
|
|
242
|
+
throw new StorageError('upload', sanitizeErrorMessage(error), {
|
|
243
|
+
originalError: error.name,
|
|
244
|
+
key
|
|
245
|
+
});
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
/**
|
|
250
|
+
* Download JSON from IPFS via gateway
|
|
251
|
+
*
|
|
252
|
+
* Security Features:
|
|
253
|
+
* - Size limit enforcement (P1-1: DoS protection)
|
|
254
|
+
* - Retry with exponential backoff (P1-2)
|
|
255
|
+
* - Centralized CID validation (P1-3)
|
|
256
|
+
* - Credential sanitization in errors (P0-2)
|
|
257
|
+
*
|
|
258
|
+
* @param cid - IPFS CID to download
|
|
259
|
+
* @returns Downloaded and parsed JSON data
|
|
260
|
+
* @throws {InvalidCIDError} If CID format is invalid
|
|
261
|
+
* @throws {ContentNotFoundError} If content not found
|
|
262
|
+
* @throws {FileSizeLimitExceededError} If content exceeds size limit
|
|
263
|
+
* @throws {DownloadTimeoutError} If download times out
|
|
264
|
+
* @throws {StorageError} If download fails
|
|
265
|
+
*
|
|
266
|
+
* @example
|
|
267
|
+
* ```typescript
|
|
268
|
+
* const data = await client.downloadJSON('bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi');
|
|
269
|
+
* console.log('Downloaded:', data);
|
|
270
|
+
* ```
|
|
271
|
+
*/
|
|
272
|
+
async downloadJSON<T = unknown>(cid: string): Promise<DownloadResult<T>> {
|
|
273
|
+
// P1-3: Use centralized CID validation
|
|
274
|
+
validateCID(cid);
|
|
275
|
+
|
|
276
|
+
const url = `${this.gatewayUrl}${cid}`;
|
|
277
|
+
|
|
278
|
+
// Circuit breaker: check gateway health before attempting download
|
|
279
|
+
if (this.circuitBreaker && !this.circuitBreaker.isHealthy(this.gatewayUrl)) {
|
|
280
|
+
const state = this.circuitBreaker.getState(this.gatewayUrl);
|
|
281
|
+
const failures = this.circuitBreaker.getFailureCount(this.gatewayUrl);
|
|
282
|
+
throw new StorageError(
|
|
283
|
+
'download',
|
|
284
|
+
`Gateway circuit breaker OPEN for ${this.gatewayUrl}. ` +
|
|
285
|
+
`State: ${state}, Failures: ${failures}. ` +
|
|
286
|
+
`Please wait for cooldown period before retrying.`,
|
|
287
|
+
{ cid, circuitState: state }
|
|
288
|
+
);
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
// P1-2: Wrap in retry logic
|
|
292
|
+
return withRetry(
|
|
293
|
+
async () => {
|
|
294
|
+
const controller = new AbortController();
|
|
295
|
+
const timeoutId = setTimeout(() => controller.abort(), this.timeout);
|
|
296
|
+
|
|
297
|
+
try {
|
|
298
|
+
const response = await fetch(url, {
|
|
299
|
+
signal: controller.signal,
|
|
300
|
+
headers: {
|
|
301
|
+
'Accept': 'application/json'
|
|
302
|
+
}
|
|
303
|
+
});
|
|
304
|
+
|
|
305
|
+
clearTimeout(timeoutId);
|
|
306
|
+
|
|
307
|
+
if (!response.ok) {
|
|
308
|
+
if (response.status === 404) {
|
|
309
|
+
throw new ContentNotFoundError(cid);
|
|
310
|
+
}
|
|
311
|
+
throw new StorageError('download', `HTTP ${response.status}: ${response.statusText}`, { cid });
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
// P1-1: Check Content-Length header before downloading
|
|
315
|
+
const contentLength = response.headers.get('content-length');
|
|
316
|
+
if (contentLength && parseInt(contentLength, 10) > this.maxDownloadSize) {
|
|
317
|
+
throw new FileSizeLimitExceededError(
|
|
318
|
+
parseInt(contentLength, 10),
|
|
319
|
+
this.maxDownloadSize
|
|
320
|
+
);
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
// P1-1: Stream response with size limit enforcement
|
|
324
|
+
const reader = response.body?.getReader();
|
|
325
|
+
if (!reader) {
|
|
326
|
+
throw new StorageError('download', 'No response body', { cid });
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
const chunks: Uint8Array[] = [];
|
|
330
|
+
let totalSize = 0;
|
|
331
|
+
|
|
332
|
+
while (true) {
|
|
333
|
+
const { done, value } = await reader.read();
|
|
334
|
+
if (done) break;
|
|
335
|
+
|
|
336
|
+
totalSize += value.length;
|
|
337
|
+
|
|
338
|
+
// P1-1: Enforce size limit during streaming
|
|
339
|
+
if (totalSize > this.maxDownloadSize) {
|
|
340
|
+
reader.cancel();
|
|
341
|
+
throw new FileSizeLimitExceededError(totalSize, this.maxDownloadSize);
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
chunks.push(value);
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
// Combine chunks into text
|
|
348
|
+
const decoder = new TextDecoder();
|
|
349
|
+
const text = chunks.map(chunk => decoder.decode(chunk, { stream: true })).join('') +
|
|
350
|
+
decoder.decode();
|
|
351
|
+
|
|
352
|
+
const data = JSON.parse(text) as T;
|
|
353
|
+
|
|
354
|
+
// Circuit breaker: record success
|
|
355
|
+
if (this.circuitBreaker) {
|
|
356
|
+
this.circuitBreaker.recordSuccess(this.gatewayUrl);
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
return {
|
|
360
|
+
data,
|
|
361
|
+
size: totalSize,
|
|
362
|
+
downloadedAt: new Date()
|
|
363
|
+
};
|
|
364
|
+
} catch (error: any) {
|
|
365
|
+
clearTimeout(timeoutId);
|
|
366
|
+
|
|
367
|
+
// Circuit breaker: record failure for retryable errors
|
|
368
|
+
// (non-retryable errors like 404 shouldn't count as gateway failures)
|
|
369
|
+
if (this.circuitBreaker && this.isGatewayFailure(error)) {
|
|
370
|
+
this.circuitBreaker.recordFailure(this.gatewayUrl);
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
if (error instanceof ContentNotFoundError) throw error;
|
|
374
|
+
if (error instanceof FileSizeLimitExceededError) throw error;
|
|
375
|
+
|
|
376
|
+
if (error.name === 'AbortError') {
|
|
377
|
+
throw new DownloadTimeoutError(cid, this.timeout);
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
if (error instanceof SyntaxError) {
|
|
381
|
+
throw new StorageError('download', `Invalid JSON content for CID: ${cid}`, { cid });
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
// P0-2: Sanitize error message
|
|
385
|
+
throw new StorageError('download', sanitizeErrorMessage(error), {
|
|
386
|
+
cid,
|
|
387
|
+
originalError: error.name
|
|
388
|
+
});
|
|
389
|
+
}
|
|
390
|
+
},
|
|
391
|
+
this.retryOptions
|
|
392
|
+
);
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
/**
|
|
396
|
+
* Upload binary data to IPFS
|
|
397
|
+
*
|
|
398
|
+
* @param data - Binary data to upload
|
|
399
|
+
* @param contentType - MIME type of the content
|
|
400
|
+
* @param options - Upload options
|
|
401
|
+
* @returns Upload result with CID
|
|
402
|
+
*/
|
|
403
|
+
async uploadBinary(
|
|
404
|
+
data: Buffer | Uint8Array,
|
|
405
|
+
contentType: string,
|
|
406
|
+
options?: { key?: string; metadata?: Record<string, string> }
|
|
407
|
+
): Promise<IPFSUploadResult> {
|
|
408
|
+
const buffer = Buffer.from(data);
|
|
409
|
+
|
|
410
|
+
// Check file size
|
|
411
|
+
if (buffer.length > this.maxFileSize) {
|
|
412
|
+
throw new FileSizeLimitExceededError(buffer.length, this.maxFileSize);
|
|
413
|
+
}
|
|
414
|
+
|
|
415
|
+
// Generate key with appropriate extension
|
|
416
|
+
const extension = this.getExtensionFromContentType(contentType);
|
|
417
|
+
const key = options?.key || this.generateKey(extension);
|
|
418
|
+
|
|
419
|
+
try {
|
|
420
|
+
const command = new PutObjectCommand({
|
|
421
|
+
Bucket: this.bucket,
|
|
422
|
+
Key: key,
|
|
423
|
+
Body: buffer,
|
|
424
|
+
ContentType: contentType,
|
|
425
|
+
Metadata: options?.metadata
|
|
426
|
+
});
|
|
427
|
+
|
|
428
|
+
await this.withTimeout(this.s3.send(command), this.timeout, 'upload');
|
|
429
|
+
|
|
430
|
+
const cid = await this.getCIDFromKey(key);
|
|
431
|
+
if (!cid) {
|
|
432
|
+
throw new StorageError('upload', 'Failed to retrieve CID after upload');
|
|
433
|
+
}
|
|
434
|
+
|
|
435
|
+
return {
|
|
436
|
+
cid,
|
|
437
|
+
size: buffer.length,
|
|
438
|
+
uploadedAt: new Date()
|
|
439
|
+
};
|
|
440
|
+
} catch (error: any) {
|
|
441
|
+
if (error instanceof FileSizeLimitExceededError) throw error;
|
|
442
|
+
if (error instanceof UploadTimeoutError) throw error;
|
|
443
|
+
if (error instanceof StorageError) throw error;
|
|
444
|
+
|
|
445
|
+
// P0-2: Sanitize error message (may contain AWS credentials)
|
|
446
|
+
throw new StorageError('upload', sanitizeErrorMessage(error), {
|
|
447
|
+
originalError: error.name,
|
|
448
|
+
key
|
|
449
|
+
});
|
|
450
|
+
}
|
|
451
|
+
}
|
|
452
|
+
|
|
453
|
+
/**
|
|
454
|
+
* Download binary data from IPFS
|
|
455
|
+
*
|
|
456
|
+
* Security Features:
|
|
457
|
+
* - Size limit enforcement (P1-1: DoS protection)
|
|
458
|
+
* - Retry with exponential backoff (P1-2)
|
|
459
|
+
* - Centralized CID validation (P1-3)
|
|
460
|
+
*
|
|
461
|
+
* @param cid - IPFS CID to download
|
|
462
|
+
* @returns Downloaded binary data
|
|
463
|
+
*/
|
|
464
|
+
async downloadBinary(cid: string): Promise<DownloadResult<Buffer>> {
|
|
465
|
+
// P1-3: Use centralized CID validation
|
|
466
|
+
validateCID(cid);
|
|
467
|
+
|
|
468
|
+
const url = `${this.gatewayUrl}${cid}`;
|
|
469
|
+
|
|
470
|
+
// Circuit breaker: check gateway health before attempting download
|
|
471
|
+
if (this.circuitBreaker && !this.circuitBreaker.isHealthy(this.gatewayUrl)) {
|
|
472
|
+
const state = this.circuitBreaker.getState(this.gatewayUrl);
|
|
473
|
+
const failures = this.circuitBreaker.getFailureCount(this.gatewayUrl);
|
|
474
|
+
throw new StorageError(
|
|
475
|
+
'download',
|
|
476
|
+
`Gateway circuit breaker OPEN for ${this.gatewayUrl}. ` +
|
|
477
|
+
`State: ${state}, Failures: ${failures}. ` +
|
|
478
|
+
`Please wait for cooldown period before retrying.`,
|
|
479
|
+
{ cid, circuitState: state }
|
|
480
|
+
);
|
|
481
|
+
}
|
|
482
|
+
|
|
483
|
+
// P1-2: Wrap in retry logic
|
|
484
|
+
return withRetry(
|
|
485
|
+
async () => {
|
|
486
|
+
const controller = new AbortController();
|
|
487
|
+
const timeoutId = setTimeout(() => controller.abort(), this.timeout);
|
|
488
|
+
|
|
489
|
+
try {
|
|
490
|
+
const response = await fetch(url, { signal: controller.signal });
|
|
491
|
+
|
|
492
|
+
clearTimeout(timeoutId);
|
|
493
|
+
|
|
494
|
+
if (!response.ok) {
|
|
495
|
+
if (response.status === 404) {
|
|
496
|
+
throw new ContentNotFoundError(cid);
|
|
497
|
+
}
|
|
498
|
+
throw new StorageError('download', `HTTP ${response.status}: ${response.statusText}`, { cid });
|
|
499
|
+
}
|
|
500
|
+
|
|
501
|
+
// P1-1: Check Content-Length header before downloading
|
|
502
|
+
const contentLength = response.headers.get('content-length');
|
|
503
|
+
if (contentLength && parseInt(contentLength, 10) > this.maxDownloadSize) {
|
|
504
|
+
throw new FileSizeLimitExceededError(
|
|
505
|
+
parseInt(contentLength, 10),
|
|
506
|
+
this.maxDownloadSize
|
|
507
|
+
);
|
|
508
|
+
}
|
|
509
|
+
|
|
510
|
+
// P1-1: Stream response with size limit enforcement
|
|
511
|
+
const reader = response.body?.getReader();
|
|
512
|
+
if (!reader) {
|
|
513
|
+
throw new StorageError('download', 'No response body', { cid });
|
|
514
|
+
}
|
|
515
|
+
|
|
516
|
+
const chunks: Uint8Array[] = [];
|
|
517
|
+
let totalSize = 0;
|
|
518
|
+
|
|
519
|
+
while (true) {
|
|
520
|
+
const { done, value } = await reader.read();
|
|
521
|
+
if (done) break;
|
|
522
|
+
|
|
523
|
+
totalSize += value.length;
|
|
524
|
+
|
|
525
|
+
// P1-1: Enforce size limit during streaming
|
|
526
|
+
if (totalSize > this.maxDownloadSize) {
|
|
527
|
+
reader.cancel();
|
|
528
|
+
throw new FileSizeLimitExceededError(totalSize, this.maxDownloadSize);
|
|
529
|
+
}
|
|
530
|
+
|
|
531
|
+
chunks.push(value);
|
|
532
|
+
}
|
|
533
|
+
|
|
534
|
+
// Combine chunks into Buffer
|
|
535
|
+
const buffer = Buffer.concat(chunks);
|
|
536
|
+
|
|
537
|
+
// Circuit breaker: record success
|
|
538
|
+
if (this.circuitBreaker) {
|
|
539
|
+
this.circuitBreaker.recordSuccess(this.gatewayUrl);
|
|
540
|
+
}
|
|
541
|
+
|
|
542
|
+
return {
|
|
543
|
+
data: buffer,
|
|
544
|
+
size: buffer.length,
|
|
545
|
+
downloadedAt: new Date()
|
|
546
|
+
};
|
|
547
|
+
} catch (error: any) {
|
|
548
|
+
clearTimeout(timeoutId);
|
|
549
|
+
|
|
550
|
+
// Circuit breaker: record failure for retryable errors
|
|
551
|
+
if (this.circuitBreaker && this.isGatewayFailure(error)) {
|
|
552
|
+
this.circuitBreaker.recordFailure(this.gatewayUrl);
|
|
553
|
+
}
|
|
554
|
+
|
|
555
|
+
if (error instanceof ContentNotFoundError) throw error;
|
|
556
|
+
if (error instanceof FileSizeLimitExceededError) throw error;
|
|
557
|
+
|
|
558
|
+
if (error.name === 'AbortError') {
|
|
559
|
+
throw new DownloadTimeoutError(cid, this.timeout);
|
|
560
|
+
}
|
|
561
|
+
|
|
562
|
+
// P0-2: Sanitize error message
|
|
563
|
+
throw new StorageError('download', sanitizeErrorMessage(error), {
|
|
564
|
+
cid,
|
|
565
|
+
originalError: error.name
|
|
566
|
+
});
|
|
567
|
+
}
|
|
568
|
+
},
|
|
569
|
+
this.retryOptions
|
|
570
|
+
);
|
|
571
|
+
}
|
|
572
|
+
|
|
573
|
+
/**
|
|
574
|
+
* Check if content exists on IPFS
|
|
575
|
+
*
|
|
576
|
+
* @param cid - IPFS CID to check
|
|
577
|
+
* @returns True if content exists
|
|
578
|
+
*/
|
|
579
|
+
async exists(cid: string): Promise<boolean> {
|
|
580
|
+
// P1-3: Use centralized CID validation
|
|
581
|
+
validateCID(cid);
|
|
582
|
+
|
|
583
|
+
const url = `${this.gatewayUrl}${cid}`;
|
|
584
|
+
|
|
585
|
+
try {
|
|
586
|
+
const controller = new AbortController();
|
|
587
|
+
const timeoutId = setTimeout(() => controller.abort(), this.timeout);
|
|
588
|
+
|
|
589
|
+
const response = await fetch(url, {
|
|
590
|
+
method: 'HEAD',
|
|
591
|
+
signal: controller.signal
|
|
592
|
+
});
|
|
593
|
+
|
|
594
|
+
clearTimeout(timeoutId);
|
|
595
|
+
|
|
596
|
+
return response.ok;
|
|
597
|
+
} catch {
|
|
598
|
+
return false;
|
|
599
|
+
}
|
|
600
|
+
}
|
|
601
|
+
|
|
602
|
+
/**
|
|
603
|
+
* Get the bucket name
|
|
604
|
+
*/
|
|
605
|
+
getBucket(): string {
|
|
606
|
+
return this.bucket;
|
|
607
|
+
}
|
|
608
|
+
|
|
609
|
+
/**
|
|
610
|
+
* Get the gateway URL
|
|
611
|
+
*/
|
|
612
|
+
getGatewayUrl(): string {
|
|
613
|
+
return this.gatewayUrl;
|
|
614
|
+
}
|
|
615
|
+
|
|
616
|
+
// ==========================================================================
|
|
617
|
+
// Private Methods
|
|
618
|
+
// ==========================================================================
|
|
619
|
+
|
|
620
|
+
/**
|
|
621
|
+
* Generate unique key for S3 object
|
|
622
|
+
*/
|
|
623
|
+
private generateKey(extension: string): string {
|
|
624
|
+
const timestamp = Date.now();
|
|
625
|
+
const random = Math.random().toString(36).substring(2, 10);
|
|
626
|
+
return `${timestamp}-${random}${extension}`;
|
|
627
|
+
}
|
|
628
|
+
|
|
629
|
+
/**
|
|
630
|
+
* Get CID from uploaded object via HeadObject
|
|
631
|
+
* Filebase returns CID in x-amz-meta-cid header
|
|
632
|
+
*/
|
|
633
|
+
private async getCIDFromKey(key: string): Promise<string | undefined> {
|
|
634
|
+
try {
|
|
635
|
+
const command = new HeadObjectCommand({
|
|
636
|
+
Bucket: this.bucket,
|
|
637
|
+
Key: key
|
|
638
|
+
});
|
|
639
|
+
|
|
640
|
+
const response = await this.s3.send(command);
|
|
641
|
+
|
|
642
|
+
// Filebase stores CID in metadata
|
|
643
|
+
return response.Metadata?.cid;
|
|
644
|
+
} catch {
|
|
645
|
+
return undefined;
|
|
646
|
+
}
|
|
647
|
+
}
|
|
648
|
+
|
|
649
|
+
|
|
650
|
+
/**
|
|
651
|
+
* Get file extension from content type
|
|
652
|
+
*/
|
|
653
|
+
private getExtensionFromContentType(contentType: string): string {
|
|
654
|
+
const types: Record<string, string> = {
|
|
655
|
+
'application/json': '.json',
|
|
656
|
+
'text/plain': '.txt',
|
|
657
|
+
'image/png': '.png',
|
|
658
|
+
'image/jpeg': '.jpg',
|
|
659
|
+
'application/pdf': '.pdf',
|
|
660
|
+
'application/octet-stream': '.bin'
|
|
661
|
+
};
|
|
662
|
+
|
|
663
|
+
return types[contentType] || '.bin';
|
|
664
|
+
}
|
|
665
|
+
|
|
666
|
+
/**
|
|
667
|
+
* Wrap promise with timeout
|
|
668
|
+
*/
|
|
669
|
+
private async withTimeout<T>(
|
|
670
|
+
promise: Promise<T>,
|
|
671
|
+
timeoutMs: number,
|
|
672
|
+
operation: 'upload' | 'download'
|
|
673
|
+
): Promise<T> {
|
|
674
|
+
let timeoutId: NodeJS.Timeout;
|
|
675
|
+
|
|
676
|
+
const timeoutPromise = new Promise<never>((_, reject) => {
|
|
677
|
+
timeoutId = setTimeout(() => {
|
|
678
|
+
if (operation === 'upload') {
|
|
679
|
+
reject(new UploadTimeoutError(timeoutMs));
|
|
680
|
+
} else {
|
|
681
|
+
reject(new DownloadTimeoutError('unknown', timeoutMs));
|
|
682
|
+
}
|
|
683
|
+
}, timeoutMs);
|
|
684
|
+
});
|
|
685
|
+
|
|
686
|
+
try {
|
|
687
|
+
return await Promise.race([promise, timeoutPromise]);
|
|
688
|
+
} finally {
|
|
689
|
+
clearTimeout(timeoutId!);
|
|
690
|
+
}
|
|
691
|
+
}
|
|
692
|
+
|
|
693
|
+
/**
|
|
694
|
+
* Determine if an error represents a gateway failure
|
|
695
|
+
* (should be counted by circuit breaker)
|
|
696
|
+
*
|
|
697
|
+
* Gateway failures are network/server errors that indicate
|
|
698
|
+
* the gateway is unhealthy. Content-specific errors (404, invalid JSON)
|
|
699
|
+
* are NOT gateway failures.
|
|
700
|
+
*
|
|
701
|
+
* @param error - Error to check
|
|
702
|
+
* @returns true if error is a gateway failure
|
|
703
|
+
*/
|
|
704
|
+
private isGatewayFailure(error: unknown): boolean {
|
|
705
|
+
if (!error) return false;
|
|
706
|
+
|
|
707
|
+
const e = error as any;
|
|
708
|
+
|
|
709
|
+
// 404 Not Found - content doesn't exist, not a gateway failure
|
|
710
|
+
if (error instanceof ContentNotFoundError) return false;
|
|
711
|
+
|
|
712
|
+
// File size exceeded - content too large, not a gateway failure
|
|
713
|
+
if (error instanceof FileSizeLimitExceededError) return false;
|
|
714
|
+
|
|
715
|
+
// Invalid JSON - content is corrupted, not a gateway failure
|
|
716
|
+
if (error instanceof SyntaxError) return false;
|
|
717
|
+
|
|
718
|
+
// Timeout - gateway is slow/unresponsive, IS a failure
|
|
719
|
+
if (e.name === 'AbortError' || e.name === 'TimeoutError') return true;
|
|
720
|
+
|
|
721
|
+
// 5xx server errors - gateway issue
|
|
722
|
+
if (e.status >= 500 && e.status < 600) return true;
|
|
723
|
+
if (e.statusCode >= 500 && e.statusCode < 600) return true;
|
|
724
|
+
|
|
725
|
+
// Network errors - gateway unreachable
|
|
726
|
+
const networkErrors = ['ECONNRESET', 'ECONNREFUSED', 'ETIMEDOUT', 'ENOTFOUND'];
|
|
727
|
+
if (e.code && networkErrors.includes(e.code)) return true;
|
|
728
|
+
|
|
729
|
+
// Rate limiting - gateway overloaded
|
|
730
|
+
if (e.status === 429 || e.statusCode === 429) return true;
|
|
731
|
+
|
|
732
|
+
// Generic StorageError from network issues or HTTP errors
|
|
733
|
+
if (error instanceof StorageError) {
|
|
734
|
+
const message = e.message?.toLowerCase() || '';
|
|
735
|
+
// Network-related errors
|
|
736
|
+
if (message.includes('timeout') ||
|
|
737
|
+
message.includes('network') ||
|
|
738
|
+
message.includes('connection')) {
|
|
739
|
+
return true;
|
|
740
|
+
}
|
|
741
|
+
// HTTP 5xx errors in message (e.g., "HTTP 500: Internal Server Error")
|
|
742
|
+
if (/http 5\d{2}/.test(message)) {
|
|
743
|
+
return true;
|
|
744
|
+
}
|
|
745
|
+
// HTTP 429 rate limiting in message
|
|
746
|
+
if (/http 429/.test(message)) {
|
|
747
|
+
return true;
|
|
748
|
+
}
|
|
749
|
+
}
|
|
750
|
+
|
|
751
|
+
return false;
|
|
752
|
+
}
|
|
753
|
+
|
|
754
|
+
// ==========================================================================
|
|
755
|
+
// Circuit Breaker Status (Public API)
|
|
756
|
+
// ==========================================================================
|
|
757
|
+
|
|
758
|
+
/**
|
|
759
|
+
* Get circuit breaker status for the gateway
|
|
760
|
+
*
|
|
761
|
+
* @returns Circuit breaker status or null if disabled
|
|
762
|
+
*/
|
|
763
|
+
getCircuitBreakerStatus(): {
|
|
764
|
+
enabled: boolean;
|
|
765
|
+
state: 'CLOSED' | 'OPEN' | 'HALF_OPEN';
|
|
766
|
+
failures: number;
|
|
767
|
+
isHealthy: boolean;
|
|
768
|
+
} | null {
|
|
769
|
+
if (!this.circuitBreaker) {
|
|
770
|
+
return null;
|
|
771
|
+
}
|
|
772
|
+
|
|
773
|
+
return {
|
|
774
|
+
enabled: true,
|
|
775
|
+
state: this.circuitBreaker.getState(this.gatewayUrl),
|
|
776
|
+
failures: this.circuitBreaker.getFailureCount(this.gatewayUrl),
|
|
777
|
+
isHealthy: this.circuitBreaker.isHealthy(this.gatewayUrl)
|
|
778
|
+
};
|
|
779
|
+
}
|
|
780
|
+
|
|
781
|
+
/**
|
|
782
|
+
* Reset circuit breaker for the gateway
|
|
783
|
+
* Use this to manually reset after a known temporary outage
|
|
784
|
+
*/
|
|
785
|
+
resetCircuitBreaker(): void {
|
|
786
|
+
if (this.circuitBreaker) {
|
|
787
|
+
this.circuitBreaker.reset(this.gatewayUrl);
|
|
788
|
+
}
|
|
789
|
+
}
|
|
790
|
+
}
|