@aigne/afs-s3 1.11.0-beta.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE.md +26 -0
- package/dist/index.d.mts +282 -0
- package/dist/index.d.mts.map +1 -0
- package/dist/index.mjs +1260 -0
- package/dist/index.mjs.map +1 -0
- package/package.json +60 -0
package/dist/index.mjs
ADDED
|
@@ -0,0 +1,1260 @@
|
|
|
1
|
+
import { AFSBaseProvider, AFSError, AFSNotFoundError, Actions, Delete, List, Meta, Read, Stat, Write } from "@aigne/afs";
|
|
2
|
+
import { camelize, optionalize, zodParse } from "@aigne/afs/utils/zod";
|
|
3
|
+
import { AbortMultipartUploadCommand, CompleteMultipartUploadCommand, CreateMultipartUploadCommand, DeleteObjectCommand, GetObjectCommand, HeadObjectCommand, ListObjectVersionsCommand, ListObjectsV2Command, PutObjectCommand, S3Client, SelectObjectContentCommand, UploadPartCommand } from "@aws-sdk/client-s3";
|
|
4
|
+
import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
|
|
5
|
+
import { z } from "zod";
|
|
6
|
+
|
|
7
|
+
//#region src/cache.ts
|
|
8
|
+
/**
|
|
9
|
+
* LRU Cache with TTL support
|
|
10
|
+
*/
|
|
11
|
+
var LRUCache = class {
|
|
12
|
+
cache = /* @__PURE__ */ new Map();
|
|
13
|
+
maxSize;
|
|
14
|
+
defaultTtl;
|
|
15
|
+
/**
|
|
16
|
+
* Create a new LRU cache
|
|
17
|
+
*
|
|
18
|
+
* @param maxSize - Maximum number of entries (default: 1000)
|
|
19
|
+
* @param defaultTtl - Default TTL in seconds (default: 60)
|
|
20
|
+
*/
|
|
21
|
+
constructor(maxSize = 1e3, defaultTtl = 60) {
|
|
22
|
+
this.maxSize = maxSize;
|
|
23
|
+
this.defaultTtl = defaultTtl;
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* Get a value from the cache
|
|
27
|
+
*
|
|
28
|
+
* @param key - Cache key
|
|
29
|
+
* @returns Cached value or undefined if not found/expired
|
|
30
|
+
*/
|
|
31
|
+
get(key) {
|
|
32
|
+
const entry = this.cache.get(key);
|
|
33
|
+
if (!entry) return;
|
|
34
|
+
if (Date.now() > entry.expiresAt) {
|
|
35
|
+
this.cache.delete(key);
|
|
36
|
+
return;
|
|
37
|
+
}
|
|
38
|
+
this.cache.delete(key);
|
|
39
|
+
this.cache.set(key, entry);
|
|
40
|
+
return entry.value;
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* Set a value in the cache
|
|
44
|
+
*
|
|
45
|
+
* @param key - Cache key
|
|
46
|
+
* @param value - Value to cache
|
|
47
|
+
* @param ttl - TTL in seconds (optional, uses default)
|
|
48
|
+
*/
|
|
49
|
+
set(key, value, ttl) {
|
|
50
|
+
if (this.cache.has(key)) this.cache.delete(key);
|
|
51
|
+
while (this.cache.size >= this.maxSize) {
|
|
52
|
+
const oldestKey = this.cache.keys().next().value;
|
|
53
|
+
if (oldestKey) this.cache.delete(oldestKey);
|
|
54
|
+
}
|
|
55
|
+
const expiresAt = Date.now() + (ttl ?? this.defaultTtl) * 1e3;
|
|
56
|
+
this.cache.set(key, {
|
|
57
|
+
value,
|
|
58
|
+
expiresAt
|
|
59
|
+
});
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* Delete a value from the cache
|
|
63
|
+
*
|
|
64
|
+
* @param key - Cache key
|
|
65
|
+
*/
|
|
66
|
+
delete(key) {
|
|
67
|
+
this.cache.delete(key);
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* Delete all entries matching a prefix
|
|
71
|
+
*
|
|
72
|
+
* @param prefix - Key prefix to match
|
|
73
|
+
*/
|
|
74
|
+
deleteByPrefix(prefix) {
|
|
75
|
+
for (const key of this.cache.keys()) if (key.startsWith(prefix)) this.cache.delete(key);
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* Clear all entries
|
|
79
|
+
*/
|
|
80
|
+
clear() {
|
|
81
|
+
this.cache.clear();
|
|
82
|
+
}
|
|
83
|
+
/**
|
|
84
|
+
* Get the number of entries in the cache
|
|
85
|
+
*/
|
|
86
|
+
get size() {
|
|
87
|
+
return this.cache.size;
|
|
88
|
+
}
|
|
89
|
+
/**
|
|
90
|
+
* Prune expired entries
|
|
91
|
+
*/
|
|
92
|
+
prune() {
|
|
93
|
+
const now = Date.now();
|
|
94
|
+
for (const [key, entry] of this.cache.entries()) if (now > entry.expiresAt) this.cache.delete(key);
|
|
95
|
+
}
|
|
96
|
+
};
|
|
97
|
+
/**
|
|
98
|
+
* Create a cache key from bucket, prefix, and path
|
|
99
|
+
*/
|
|
100
|
+
function createCacheKey(bucket, prefix, path, suffix) {
|
|
101
|
+
const base = `${bucket}:${prefix}:${path}`;
|
|
102
|
+
return suffix ? `${base}:${suffix}` : base;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
//#endregion
|
|
106
|
+
//#region src/client.ts
|
|
107
|
+
/**
|
|
108
|
+
* S3 Client Factory
|
|
109
|
+
*
|
|
110
|
+
* Creates and configures AWS S3 clients.
|
|
111
|
+
*/
|
|
112
|
+
/**
|
|
113
|
+
* Create an S3 client with the given options
|
|
114
|
+
*
|
|
115
|
+
* Uses AWS SDK's default credential chain:
|
|
116
|
+
* 1. Environment variables (AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY)
|
|
117
|
+
* 2. Shared credentials file (~/.aws/credentials)
|
|
118
|
+
* 3. IAM role (EC2/ECS/Lambda)
|
|
119
|
+
* 4. SSO credentials
|
|
120
|
+
*
|
|
121
|
+
* @param options - S3 provider options
|
|
122
|
+
* @returns Configured S3 client
|
|
123
|
+
*/
|
|
124
|
+
function createS3Client(options) {
|
|
125
|
+
const config = {};
|
|
126
|
+
if (options.region) config.region = options.region;
|
|
127
|
+
else if (options.endpoint) config.region = "us-east-1";
|
|
128
|
+
if (options.endpoint) config.endpoint = options.endpoint;
|
|
129
|
+
if (options.forcePathStyle) config.forcePathStyle = true;
|
|
130
|
+
if (options.credentials) config.credentials = {
|
|
131
|
+
accessKeyId: options.credentials.accessKeyId,
|
|
132
|
+
secretAccessKey: options.credentials.secretAccessKey,
|
|
133
|
+
sessionToken: options.credentials.sessionToken
|
|
134
|
+
};
|
|
135
|
+
return new S3Client(config);
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
//#endregion
|
|
139
|
+
//#region src/errors.ts
|
|
140
|
+
/**
|
|
141
|
+
* S3 Error Handling
|
|
142
|
+
*
|
|
143
|
+
* Maps AWS S3 errors to AFS errors.
|
|
144
|
+
*/
|
|
145
|
+
/**
|
|
146
|
+
* AFS error codes
|
|
147
|
+
*/
|
|
148
|
+
const AFSErrorCode = {
|
|
149
|
+
ENTRY_NOT_FOUND: "ENTRY_NOT_FOUND",
|
|
150
|
+
MODULE_NOT_FOUND: "MODULE_NOT_FOUND",
|
|
151
|
+
PERMISSION_DENIED: "PERMISSION_DENIED",
|
|
152
|
+
AUTH_ERROR: "AUTH_ERROR",
|
|
153
|
+
RATE_LIMITED: "RATE_LIMITED",
|
|
154
|
+
TYPE_MISMATCH: "TYPE_MISMATCH",
|
|
155
|
+
INTERNAL_ERROR: "INTERNAL_ERROR"
|
|
156
|
+
};
|
|
157
|
+
/**
|
|
158
|
+
* AFS Error class
|
|
159
|
+
*/
|
|
160
|
+
var AFSError$1 = class extends Error {
|
|
161
|
+
code;
|
|
162
|
+
retryAfter;
|
|
163
|
+
constructor(code, message, options) {
|
|
164
|
+
super(message);
|
|
165
|
+
this.name = "AFSError";
|
|
166
|
+
this.code = code;
|
|
167
|
+
this.retryAfter = options?.retryAfter;
|
|
168
|
+
}
|
|
169
|
+
};
|
|
170
|
+
/**
|
|
171
|
+
* Map S3 error to AFS error
|
|
172
|
+
*
|
|
173
|
+
* @param error - AWS S3 error
|
|
174
|
+
* @param path - Optional path for AFSNotFoundError (with leading slash)
|
|
175
|
+
* @returns AFS error (or throws the original error if it's already an AFS error)
|
|
176
|
+
*/
|
|
177
|
+
function mapS3Error(error, path) {
|
|
178
|
+
if (error && typeof error === "object" && "name" in error) {
|
|
179
|
+
const errorObj = error;
|
|
180
|
+
if (errorObj.name === "AFSNotFoundError" || errorObj.name === "AFSError" || errorObj.code === "AFS_NOT_FOUND") throw error;
|
|
181
|
+
}
|
|
182
|
+
if (error && typeof error === "object" && "name" in error) {
|
|
183
|
+
const awsError = error;
|
|
184
|
+
const message = awsError.message ?? "Unknown S3 error";
|
|
185
|
+
switch (awsError.name) {
|
|
186
|
+
case "NoSuchBucket": return new AFSError$1(AFSErrorCode.MODULE_NOT_FOUND, `Bucket not found: ${message}`);
|
|
187
|
+
case "NoSuchKey":
|
|
188
|
+
case "NotFound":
|
|
189
|
+
if (path) return new AFSNotFoundError(path);
|
|
190
|
+
return new AFSError$1(AFSErrorCode.ENTRY_NOT_FOUND, `Object not found: ${message}`);
|
|
191
|
+
case "AccessDenied":
|
|
192
|
+
case "Forbidden": return new AFSError$1(AFSErrorCode.PERMISSION_DENIED, `Access denied: ${message}`);
|
|
193
|
+
case "InvalidAccessKeyId":
|
|
194
|
+
case "SignatureDoesNotMatch":
|
|
195
|
+
case "ExpiredToken":
|
|
196
|
+
case "TokenRefreshRequired": return new AFSError$1(AFSErrorCode.AUTH_ERROR, `Authentication failed: ${message}`);
|
|
197
|
+
case "SlowDown":
|
|
198
|
+
case "ServiceUnavailable": return new AFSError$1(AFSErrorCode.RATE_LIMITED, `Rate limited: ${message}`, { retryAfter: 1e3 });
|
|
199
|
+
default: {
|
|
200
|
+
const statusCode = awsError.$metadata?.httpStatusCode;
|
|
201
|
+
if (statusCode === 404) {
|
|
202
|
+
if (path) return new AFSNotFoundError(path);
|
|
203
|
+
return new AFSError$1(AFSErrorCode.ENTRY_NOT_FOUND, message);
|
|
204
|
+
}
|
|
205
|
+
if (statusCode === 403) return new AFSError$1(AFSErrorCode.PERMISSION_DENIED, message);
|
|
206
|
+
if (statusCode === 401) return new AFSError$1(AFSErrorCode.AUTH_ERROR, message);
|
|
207
|
+
if (statusCode === 429 || statusCode === 503) return new AFSError$1(AFSErrorCode.RATE_LIMITED, message, { retryAfter: 1e3 });
|
|
208
|
+
return new AFSError$1(AFSErrorCode.INTERNAL_ERROR, message);
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
if (error instanceof Error) return new AFSError$1(AFSErrorCode.INTERNAL_ERROR, error.message);
|
|
213
|
+
return new AFSError$1(AFSErrorCode.INTERNAL_ERROR, String(error));
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
//#endregion
|
|
217
|
+
//#region src/operations/multipart.ts
|
|
218
|
+
/**
|
|
219
|
+
* S3 Multipart Upload (Phase 2)
|
|
220
|
+
*
|
|
221
|
+
* Handles multipart upload for large files (>5GB).
|
|
222
|
+
* S3 requires multipart upload for objects larger than 5GB.
|
|
223
|
+
*/
|
|
224
|
+
/**
|
|
225
|
+
* Minimum part size (5MB) - AWS S3 requirement
|
|
226
|
+
*/
|
|
227
|
+
const MIN_PART_SIZE = 5 * 1024 * 1024;
|
|
228
|
+
/**
|
|
229
|
+
* Maximum part size (5GB) - AWS S3 requirement
|
|
230
|
+
*/
|
|
231
|
+
const MAX_PART_SIZE = 5 * 1024 * 1024 * 1024;
|
|
232
|
+
/**
|
|
233
|
+
* Threshold for switching to multipart upload (100MB)
|
|
234
|
+
* Files larger than this will use multipart upload
|
|
235
|
+
*/
|
|
236
|
+
const MULTIPART_THRESHOLD = 100 * 1024 * 1024;
|
|
237
|
+
/**
|
|
238
|
+
* Default part size (10MB)
|
|
239
|
+
*/
|
|
240
|
+
const DEFAULT_PART_SIZE = 10 * 1024 * 1024;
|
|
241
|
+
/**
|
|
242
|
+
* Maximum number of parts (10,000) - AWS S3 requirement
|
|
243
|
+
*/
|
|
244
|
+
const MAX_PARTS = 1e4;
|
|
245
|
+
/**
|
|
246
|
+
* Calculate optimal part size based on file size
|
|
247
|
+
*/
|
|
248
|
+
function calculatePartSize(fileSize, requestedPartSize) {
|
|
249
|
+
let partSize = requestedPartSize ?? DEFAULT_PART_SIZE;
|
|
250
|
+
if (partSize < MIN_PART_SIZE) partSize = MIN_PART_SIZE;
|
|
251
|
+
if (partSize > MAX_PART_SIZE) partSize = MAX_PART_SIZE;
|
|
252
|
+
if (Math.ceil(fileSize / partSize) > MAX_PARTS) {
|
|
253
|
+
partSize = Math.ceil(fileSize / MAX_PARTS);
|
|
254
|
+
partSize = Math.ceil(partSize / (1024 * 1024)) * 1024 * 1024;
|
|
255
|
+
}
|
|
256
|
+
return partSize;
|
|
257
|
+
}
|
|
258
|
+
/**
|
|
259
|
+
* Upload a large file using multipart upload
|
|
260
|
+
*
|
|
261
|
+
* @param client - S3 client
|
|
262
|
+
* @param bucket - Bucket name
|
|
263
|
+
* @param key - Object key
|
|
264
|
+
* @param data - File content as Buffer
|
|
265
|
+
* @param options - Upload options
|
|
266
|
+
* @returns Upload result with ETag
|
|
267
|
+
*/
|
|
268
|
+
async function multipartUpload(client, bucket, key, data, options) {
|
|
269
|
+
const fileSize = data.length;
|
|
270
|
+
const partSize = calculatePartSize(fileSize, options?.partSize);
|
|
271
|
+
const createCommand = new CreateMultipartUploadCommand({
|
|
272
|
+
Bucket: bucket,
|
|
273
|
+
Key: key,
|
|
274
|
+
ContentType: options?.contentType ?? "application/octet-stream",
|
|
275
|
+
Metadata: options?.metadata
|
|
276
|
+
});
|
|
277
|
+
const uploadId = (await client.send(createCommand)).UploadId;
|
|
278
|
+
if (!uploadId) throw new Error("Failed to initiate multipart upload: no uploadId returned");
|
|
279
|
+
const parts = [];
|
|
280
|
+
try {
|
|
281
|
+
const totalParts = Math.ceil(fileSize / partSize);
|
|
282
|
+
for (let partNumber = 1; partNumber <= totalParts; partNumber++) {
|
|
283
|
+
const start = (partNumber - 1) * partSize;
|
|
284
|
+
const end = Math.min(start + partSize, fileSize);
|
|
285
|
+
const partData = data.subarray(start, end);
|
|
286
|
+
const uploadPartCommand = new UploadPartCommand({
|
|
287
|
+
Bucket: bucket,
|
|
288
|
+
Key: key,
|
|
289
|
+
UploadId: uploadId,
|
|
290
|
+
PartNumber: partNumber,
|
|
291
|
+
Body: partData
|
|
292
|
+
});
|
|
293
|
+
const uploadPartResponse = await client.send(uploadPartCommand);
|
|
294
|
+
if (!uploadPartResponse.ETag) throw new Error(`Failed to upload part ${partNumber}: no ETag returned`);
|
|
295
|
+
parts.push({
|
|
296
|
+
ETag: uploadPartResponse.ETag,
|
|
297
|
+
PartNumber: partNumber
|
|
298
|
+
});
|
|
299
|
+
}
|
|
300
|
+
const completeCommand = new CompleteMultipartUploadCommand({
|
|
301
|
+
Bucket: bucket,
|
|
302
|
+
Key: key,
|
|
303
|
+
UploadId: uploadId,
|
|
304
|
+
MultipartUpload: { Parts: parts }
|
|
305
|
+
});
|
|
306
|
+
const completeResponse = await client.send(completeCommand);
|
|
307
|
+
return {
|
|
308
|
+
etag: completeResponse.ETag?.replace(/"/g, "") ?? "",
|
|
309
|
+
versionId: completeResponse.VersionId
|
|
310
|
+
};
|
|
311
|
+
} catch (error) {
|
|
312
|
+
try {
|
|
313
|
+
const abortCommand = new AbortMultipartUploadCommand({
|
|
314
|
+
Bucket: bucket,
|
|
315
|
+
Key: key,
|
|
316
|
+
UploadId: uploadId
|
|
317
|
+
});
|
|
318
|
+
await client.send(abortCommand);
|
|
319
|
+
} catch {}
|
|
320
|
+
throw mapS3Error(error);
|
|
321
|
+
}
|
|
322
|
+
}
|
|
323
|
+
/**
|
|
324
|
+
* Check if a file should use multipart upload
|
|
325
|
+
*/
|
|
326
|
+
function shouldUseMultipart(size) {
|
|
327
|
+
return size >= MULTIPART_THRESHOLD;
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
//#endregion
|
|
331
|
+
//#region src/operations/select.ts
|
|
332
|
+
/**
|
|
333
|
+
* S3 Select Support (Phase 3)
|
|
334
|
+
*
|
|
335
|
+
* Runs SQL-like queries on CSV and JSON files stored in S3.
|
|
336
|
+
* This enables server-side filtering, reducing data transfer.
|
|
337
|
+
*/
|
|
338
|
+
/**
|
|
339
|
+
* Detect input format from file extension
|
|
340
|
+
*/
|
|
341
|
+
function detectInputFormat(path) {
|
|
342
|
+
switch (path.toLowerCase().split(".").pop()) {
|
|
343
|
+
case "csv":
|
|
344
|
+
case "tsv": return "CSV";
|
|
345
|
+
case "json":
|
|
346
|
+
case "jsonl":
|
|
347
|
+
case "ndjson": return "JSON";
|
|
348
|
+
case "parquet": return "Parquet";
|
|
349
|
+
default: return "JSON";
|
|
350
|
+
}
|
|
351
|
+
}
|
|
352
|
+
/**
|
|
353
|
+
* Run a SQL-like query on an S3 object
|
|
354
|
+
*
|
|
355
|
+
* @param client - S3 client
|
|
356
|
+
* @param bucket - Bucket name
|
|
357
|
+
* @param mountPrefix - Mount prefix from options
|
|
358
|
+
* @param path - Path relative to mount point
|
|
359
|
+
* @param query - SQL query (e.g., "SELECT * FROM s3object WHERE age > 21")
|
|
360
|
+
* @param options - Select options
|
|
361
|
+
* @returns Query results
|
|
362
|
+
*/
|
|
363
|
+
async function selectQuery(client, bucket, mountPrefix, path, query, options) {
|
|
364
|
+
try {
|
|
365
|
+
const normalizedPath = path.replace(/^\/+/, "").replace(/\/+$/, "");
|
|
366
|
+
const key = mountPrefix ? `${mountPrefix}/${normalizedPath}` : normalizedPath;
|
|
367
|
+
const inputFormat = options?.inputFormat ?? detectInputFormat(path);
|
|
368
|
+
const outputFormat = options?.outputFormat ?? "JSON";
|
|
369
|
+
const inputSerialization = {};
|
|
370
|
+
if (inputFormat === "CSV") inputSerialization.CSV = {
|
|
371
|
+
FieldDelimiter: options?.csv?.fieldDelimiter ?? ",",
|
|
372
|
+
RecordDelimiter: options?.csv?.recordDelimiter ?? "\n",
|
|
373
|
+
FileHeaderInfo: options?.csv?.fileHeaderInfo ?? "USE",
|
|
374
|
+
QuoteCharacter: options?.csv?.quoteCharacter ?? "\"",
|
|
375
|
+
Comments: options?.csv?.comments
|
|
376
|
+
};
|
|
377
|
+
else if (inputFormat === "JSON") inputSerialization.JSON = { Type: options?.json?.type ?? "DOCUMENT" };
|
|
378
|
+
else if (inputFormat === "Parquet") inputSerialization.Parquet = {};
|
|
379
|
+
const outputSerialization = {};
|
|
380
|
+
if (outputFormat === "JSON") outputSerialization.JSON = {};
|
|
381
|
+
else outputSerialization.CSV = {};
|
|
382
|
+
const command = new SelectObjectContentCommand({
|
|
383
|
+
Bucket: bucket,
|
|
384
|
+
Key: key,
|
|
385
|
+
ExpressionType: "SQL",
|
|
386
|
+
Expression: query,
|
|
387
|
+
InputSerialization: inputSerialization,
|
|
388
|
+
OutputSerialization: outputSerialization
|
|
389
|
+
});
|
|
390
|
+
const response = await client.send(command);
|
|
391
|
+
const records = [];
|
|
392
|
+
let stats;
|
|
393
|
+
if (response.Payload) for await (const event of response.Payload) {
|
|
394
|
+
if (event.Records?.Payload) {
|
|
395
|
+
const lines = new TextDecoder().decode(event.Records.Payload).split("\n").filter((line) => line.trim());
|
|
396
|
+
for (const line of lines) try {
|
|
397
|
+
records.push(JSON.parse(line));
|
|
398
|
+
} catch {
|
|
399
|
+
records.push(line);
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
if (event.Stats?.Details) stats = {
|
|
403
|
+
bytesScanned: Number(event.Stats.Details.BytesScanned ?? 0),
|
|
404
|
+
bytesProcessed: Number(event.Stats.Details.BytesProcessed ?? 0),
|
|
405
|
+
bytesReturned: Number(event.Stats.Details.BytesReturned ?? 0)
|
|
406
|
+
};
|
|
407
|
+
}
|
|
408
|
+
return {
|
|
409
|
+
records,
|
|
410
|
+
stats
|
|
411
|
+
};
|
|
412
|
+
} catch (error) {
|
|
413
|
+
throw mapS3Error(error);
|
|
414
|
+
}
|
|
415
|
+
}
|
|
416
|
+
|
|
417
|
+
//#endregion
|
|
418
|
+
//#region src/platform-ref.ts
|
|
419
|
+
/**
|
|
420
|
+
* Generate platform reference with AWS Console URL
|
|
421
|
+
*
|
|
422
|
+
* @param bucket - S3 bucket name
|
|
423
|
+
* @param region - AWS region (defaults to us-east-1)
|
|
424
|
+
* @param key - S3 object key (without leading slash)
|
|
425
|
+
* @param isDirectory - Whether this is a directory (prefix)
|
|
426
|
+
* @returns Platform reference with console URL
|
|
427
|
+
*/
|
|
428
|
+
function generatePlatformRef(bucket, region, key, isDirectory) {
|
|
429
|
+
const effectiveRegion = region || "us-east-1";
|
|
430
|
+
if (isDirectory) return { consoleUrl: `https://s3.console.aws.amazon.com/s3/buckets/${bucket}?region=${effectiveRegion}&prefix=${key.endsWith("/") ? key : key ? `${key}/` : ""}` };
|
|
431
|
+
return { consoleUrl: `https://s3.console.aws.amazon.com/s3/object/${bucket}?region=${effectiveRegion}&prefix=${encodeURIComponent(key).replace(/%2F/g, "/")}` };
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
//#endregion
|
|
435
|
+
//#region src/types.ts
|
|
436
|
+
/**
|
|
437
|
+
* AFS S3 Provider Types
|
|
438
|
+
*/
|
|
439
|
+
/**
|
|
440
|
+
* S3 bucket name validation regex
|
|
441
|
+
* - 3-63 characters
|
|
442
|
+
* - lowercase letters, numbers, hyphens, dots
|
|
443
|
+
* - must start and end with letter or number
|
|
444
|
+
*/
|
|
445
|
+
const BUCKET_NAME_REGEX = /^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$/;
|
|
446
|
+
/**
|
|
447
|
+
* Zod schema for options validation
|
|
448
|
+
*/
|
|
449
|
+
const afss3OptionsSchema = camelize(z.object({
|
|
450
|
+
name: optionalize(z.string()),
|
|
451
|
+
description: optionalize(z.string()),
|
|
452
|
+
bucket: z.string().regex(BUCKET_NAME_REGEX, "Invalid S3 bucket name"),
|
|
453
|
+
prefix: optionalize(z.string()),
|
|
454
|
+
region: optionalize(z.string()),
|
|
455
|
+
accessMode: optionalize(z.enum(["readonly", "readwrite"])),
|
|
456
|
+
endpoint: optionalize(z.string().url()),
|
|
457
|
+
forcePathStyle: optionalize(z.boolean()),
|
|
458
|
+
profile: optionalize(z.string()),
|
|
459
|
+
credentials: optionalize(z.object({
|
|
460
|
+
accessKeyId: z.string(),
|
|
461
|
+
secretAccessKey: z.string(),
|
|
462
|
+
sessionToken: optionalize(z.string())
|
|
463
|
+
})),
|
|
464
|
+
cacheTtl: optionalize(z.number().int().min(0))
|
|
465
|
+
}).strict());
|
|
466
|
+
|
|
467
|
+
//#endregion
|
|
468
|
+
//#region \0@oxc-project+runtime@0.108.0/helpers/decorate.js
|
|
469
|
+
function __decorate(decorators, target, key, desc) {
|
|
470
|
+
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
471
|
+
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
472
|
+
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
473
|
+
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
474
|
+
}
|
|
475
|
+
|
|
476
|
+
//#endregion
|
|
477
|
+
//#region src/s3-afs.ts
|
|
478
|
+
/**
|
|
479
|
+
* AFS S3 Provider
|
|
480
|
+
*
|
|
481
|
+
* S3 provider using AFSBaseProvider decorator routing pattern.
|
|
482
|
+
* Provides access to AWS S3 and S3-compatible storage (MinIO, R2, B2).
|
|
483
|
+
*/
|
|
484
|
+
/**
|
|
485
|
+
* Default URL expiration time (1 hour)
|
|
486
|
+
*/
|
|
487
|
+
const DEFAULT_EXPIRES_IN = 3600;
|
|
488
|
+
/**
|
|
489
|
+
* Maximum expiration time (7 days)
|
|
490
|
+
*/
|
|
491
|
+
const MAX_EXPIRES_IN = 604800;
|
|
492
|
+
/**
|
|
493
|
+
* AFSS3 Provider using Base Provider pattern
|
|
494
|
+
*
|
|
495
|
+
* Provides access to AWS S3 and S3-compatible storage through AFS.
|
|
496
|
+
* Uses decorator routing (@List, @Read, @Write, @Delete, @Meta, @Actions).
|
|
497
|
+
*
|
|
498
|
+
* @example
|
|
499
|
+
* ```typescript
|
|
500
|
+
* const s3 = new AFSS3({
|
|
501
|
+
* bucket: "my-bucket",
|
|
502
|
+
* prefix: "data",
|
|
503
|
+
* region: "us-east-1",
|
|
504
|
+
* });
|
|
505
|
+
*
|
|
506
|
+
* // Mount to AFS
|
|
507
|
+
* afs.mount(s3);
|
|
508
|
+
*
|
|
509
|
+
* // List objects
|
|
510
|
+
* const result = await afs.list("/modules/my-bucket/data");
|
|
511
|
+
*
|
|
512
|
+
* // Read object
|
|
513
|
+
* const content = await afs.read("/modules/my-bucket/data/file.json");
|
|
514
|
+
* ```
|
|
515
|
+
*/
|
|
516
|
+
var AFSS3 = class AFSS3 extends AFSBaseProvider {
|
|
517
|
+
name;
|
|
518
|
+
description;
|
|
519
|
+
accessMode;
|
|
520
|
+
options;
|
|
521
|
+
client;
|
|
522
|
+
listCache;
|
|
523
|
+
statCache;
|
|
524
|
+
constructor(options) {
|
|
525
|
+
super();
|
|
526
|
+
const { client, ...restOptions } = options;
|
|
527
|
+
const parsed = afss3OptionsSchema.parse(restOptions);
|
|
528
|
+
this.options = {
|
|
529
|
+
...parsed,
|
|
530
|
+
bucket: parsed.bucket,
|
|
531
|
+
prefix: parsed.prefix ?? "",
|
|
532
|
+
accessMode: parsed.accessMode ?? "readonly"
|
|
533
|
+
};
|
|
534
|
+
this.name = parsed.name ?? parsed.bucket;
|
|
535
|
+
this.description = parsed.description ?? `S3 bucket: ${parsed.bucket}`;
|
|
536
|
+
this.accessMode = this.options.accessMode ?? "readonly";
|
|
537
|
+
this.client = client ?? createS3Client(this.options);
|
|
538
|
+
if (parsed.cacheTtl && parsed.cacheTtl > 0) {
|
|
539
|
+
this.listCache = new LRUCache(1e3, parsed.cacheTtl);
|
|
540
|
+
this.statCache = new LRUCache(5e3, parsed.cacheTtl);
|
|
541
|
+
}
|
|
542
|
+
}
|
|
543
|
+
/**
|
|
544
|
+
* Schema for configuration validation
|
|
545
|
+
*/
|
|
546
|
+
static schema() {
|
|
547
|
+
return afss3OptionsSchema;
|
|
548
|
+
}
|
|
549
|
+
/**
|
|
550
|
+
* Load from configuration file
|
|
551
|
+
*/
|
|
552
|
+
static async load(params) {
|
|
553
|
+
return new AFSS3(zodParse(afss3OptionsSchema, params.parsed, { prefix: params.filepath }));
|
|
554
|
+
}
|
|
555
|
+
/**
|
|
556
|
+
* Build the full S3 key from a path
|
|
557
|
+
*/
|
|
558
|
+
buildS3Key(path) {
|
|
559
|
+
const normalizedPath = path.replace(/^\/+/, "").replace(/\/+$/, "");
|
|
560
|
+
return this.options.prefix ? `${this.options.prefix}/${normalizedPath}` : normalizedPath;
|
|
561
|
+
}
|
|
562
|
+
/**
|
|
563
|
+
* Generate a unique ID for an S3 object
|
|
564
|
+
*/
|
|
565
|
+
generateId(key) {
|
|
566
|
+
return `s3://${this.options.bucket}/${key}`;
|
|
567
|
+
}
|
|
568
|
+
/**
|
|
569
|
+
* Invalidate caches for a given path
|
|
570
|
+
*/
|
|
571
|
+
invalidateCache(path) {
|
|
572
|
+
if (this.statCache) {
|
|
573
|
+
const statKey = createCacheKey(this.options.bucket, this.options.prefix ?? "", path);
|
|
574
|
+
this.statCache.delete(statKey);
|
|
575
|
+
}
|
|
576
|
+
if (this.listCache) {
|
|
577
|
+
const parentPath = path.split("/").slice(0, -1).join("/") || "/";
|
|
578
|
+
const listPrefix = createCacheKey(this.options.bucket, this.options.prefix ?? "", parentPath);
|
|
579
|
+
this.listCache.deleteByPrefix(listPrefix);
|
|
580
|
+
}
|
|
581
|
+
}
|
|
582
|
+
/**
|
|
583
|
+
* Clear all caches
|
|
584
|
+
*/
|
|
585
|
+
clearCache() {
|
|
586
|
+
this.listCache?.clear();
|
|
587
|
+
this.statCache?.clear();
|
|
588
|
+
}
|
|
589
|
+
async listHandler(ctx) {
|
|
590
|
+
try {
|
|
591
|
+
const normalizedPath = (ctx.params.path ?? "").replace(/^\/+/, "").replace(/\/+$/, "");
|
|
592
|
+
const fullPrefix = this.options.prefix ? normalizedPath ? `${this.options.prefix}/${normalizedPath}/` : `${this.options.prefix}/` : normalizedPath ? `${normalizedPath}/` : "";
|
|
593
|
+
const opts = ctx.options;
|
|
594
|
+
const maxChildren = opts?.limit ?? opts?.maxChildren ?? 1e3;
|
|
595
|
+
if (this.listCache) {
|
|
596
|
+
const cacheKey = createCacheKey(this.options.bucket, this.options.prefix ?? "", normalizedPath, JSON.stringify(ctx.options ?? {}));
|
|
597
|
+
const cached = this.listCache.get(cacheKey);
|
|
598
|
+
if (cached) return cached;
|
|
599
|
+
}
|
|
600
|
+
const result = await this.listWithDelimiter(fullPrefix, normalizedPath, maxChildren);
|
|
601
|
+
if (this.listCache) {
|
|
602
|
+
const cacheKey = createCacheKey(this.options.bucket, this.options.prefix ?? "", normalizedPath, JSON.stringify(ctx.options ?? {}));
|
|
603
|
+
this.listCache.set(cacheKey, result);
|
|
604
|
+
if (this.statCache) for (const entry of result.data) {
|
|
605
|
+
const statKey = createCacheKey(this.options.bucket, this.options.prefix ?? "", entry.path);
|
|
606
|
+
this.statCache.set(statKey, entry);
|
|
607
|
+
}
|
|
608
|
+
}
|
|
609
|
+
return result;
|
|
610
|
+
} catch (error) {
|
|
611
|
+
throw mapS3Error(error, ctx.params.path ? ctx.params.path.startsWith("/") ? ctx.params.path : `/${ctx.params.path}` : "/");
|
|
612
|
+
}
|
|
613
|
+
}
|
|
614
|
+
/**
|
|
615
|
+
* List with delimiter (single level)
|
|
616
|
+
*/
|
|
617
|
+
async listWithDelimiter(prefix, basePath, maxChildren) {
|
|
618
|
+
const childEntries = [];
|
|
619
|
+
let continuationToken;
|
|
620
|
+
do {
|
|
621
|
+
const command = new ListObjectsV2Command({
|
|
622
|
+
Bucket: this.options.bucket,
|
|
623
|
+
Prefix: prefix,
|
|
624
|
+
Delimiter: "/",
|
|
625
|
+
MaxKeys: Math.min(maxChildren - childEntries.length, 1e3),
|
|
626
|
+
ContinuationToken: continuationToken
|
|
627
|
+
});
|
|
628
|
+
const response = await this.client.send(command);
|
|
629
|
+
if (response.CommonPrefixes) for (const commonPrefix of response.CommonPrefixes) {
|
|
630
|
+
if (!commonPrefix.Prefix) continue;
|
|
631
|
+
const dirName = commonPrefix.Prefix.slice(prefix.length).replace(/\/$/, "");
|
|
632
|
+
if (!dirName) continue;
|
|
633
|
+
const entryPath = basePath ? `${basePath}/${dirName}` : dirName;
|
|
634
|
+
const normalizedPath = entryPath.startsWith("/") ? entryPath : `/${entryPath}`;
|
|
635
|
+
childEntries.push({
|
|
636
|
+
id: this.generateId(commonPrefix.Prefix),
|
|
637
|
+
path: normalizedPath,
|
|
638
|
+
metadata: {
|
|
639
|
+
childrenCount: 0,
|
|
640
|
+
platformRef: generatePlatformRef(this.options.bucket, this.options.region, commonPrefix.Prefix, true)
|
|
641
|
+
}
|
|
642
|
+
});
|
|
643
|
+
if (childEntries.length >= maxChildren) break;
|
|
644
|
+
}
|
|
645
|
+
if (response.Contents) for (const object of response.Contents) {
|
|
646
|
+
if (!object.Key) continue;
|
|
647
|
+
if (object.Key === prefix) continue;
|
|
648
|
+
const fileName = object.Key.slice(prefix.length);
|
|
649
|
+
if (!fileName || fileName.includes("/")) continue;
|
|
650
|
+
const entryPath = basePath ? `${basePath}/${fileName}` : fileName;
|
|
651
|
+
const normalizedPath = entryPath.startsWith("/") ? entryPath : `/${entryPath}`;
|
|
652
|
+
childEntries.push({
|
|
653
|
+
id: this.generateId(object.Key),
|
|
654
|
+
path: normalizedPath,
|
|
655
|
+
updatedAt: object.LastModified,
|
|
656
|
+
metadata: {
|
|
657
|
+
size: object.Size,
|
|
658
|
+
lastModified: object.LastModified?.toISOString(),
|
|
659
|
+
etag: object.ETag?.replace(/"/g, ""),
|
|
660
|
+
storageClass: object.StorageClass,
|
|
661
|
+
platformRef: generatePlatformRef(this.options.bucket, this.options.region, object.Key, false)
|
|
662
|
+
}
|
|
663
|
+
});
|
|
664
|
+
if (childEntries.length >= maxChildren) break;
|
|
665
|
+
}
|
|
666
|
+
continuationToken = response.IsTruncated ? response.NextContinuationToken : void 0;
|
|
667
|
+
} while (continuationToken && childEntries.length < maxChildren);
|
|
668
|
+
const selfPath = basePath ? `/${basePath}` : "/";
|
|
669
|
+
if (childEntries.length === 0 && basePath) {
|
|
670
|
+
const key = this.options.prefix ? `${this.options.prefix}/${basePath}` : basePath;
|
|
671
|
+
try {
|
|
672
|
+
const headCommand = new HeadObjectCommand({
|
|
673
|
+
Bucket: this.options.bucket,
|
|
674
|
+
Key: key
|
|
675
|
+
});
|
|
676
|
+
const headResponse = await this.client.send(headCommand);
|
|
677
|
+
return { data: [{
|
|
678
|
+
id: this.generateId(key),
|
|
679
|
+
path: selfPath,
|
|
680
|
+
updatedAt: headResponse.LastModified,
|
|
681
|
+
metadata: {
|
|
682
|
+
size: headResponse.ContentLength,
|
|
683
|
+
contentType: headResponse.ContentType,
|
|
684
|
+
lastModified: headResponse.LastModified?.toISOString(),
|
|
685
|
+
etag: headResponse.ETag?.replace(/"/g, ""),
|
|
686
|
+
childrenCount: 0,
|
|
687
|
+
platformRef: generatePlatformRef(this.options.bucket, this.options.region, key, false)
|
|
688
|
+
}
|
|
689
|
+
}] };
|
|
690
|
+
} catch (error) {
|
|
691
|
+
if (error?.name === "NotFound" || error?.$metadata?.httpStatusCode === 404) try {
|
|
692
|
+
const dirMarkerCommand = new HeadObjectCommand({
|
|
693
|
+
Bucket: this.options.bucket,
|
|
694
|
+
Key: `${key}/`
|
|
695
|
+
});
|
|
696
|
+
await this.client.send(dirMarkerCommand);
|
|
697
|
+
} catch {
|
|
698
|
+
throw new AFSNotFoundError(selfPath);
|
|
699
|
+
}
|
|
700
|
+
else throw error;
|
|
701
|
+
}
|
|
702
|
+
}
|
|
703
|
+
return { data: [{
|
|
704
|
+
id: this.generateId(prefix || "/"),
|
|
705
|
+
path: selfPath,
|
|
706
|
+
metadata: {
|
|
707
|
+
kind: "afs:node",
|
|
708
|
+
childrenCount: childEntries.length,
|
|
709
|
+
platformRef: generatePlatformRef(this.options.bucket, this.options.region, prefix, true)
|
|
710
|
+
}
|
|
711
|
+
}, ...childEntries] };
|
|
712
|
+
}
|
|
713
|
+
async listVersionsHandler(ctx) {
|
|
714
|
+
try {
|
|
715
|
+
const normalizedPath = ctx.params.path.replace(/^\/+/, "").replace(/\/+$/, "");
|
|
716
|
+
const key = this.options.prefix ? `${this.options.prefix}/${normalizedPath}` : normalizedPath;
|
|
717
|
+
const command = new ListObjectVersionsCommand({
|
|
718
|
+
Bucket: this.options.bucket,
|
|
719
|
+
Prefix: key,
|
|
720
|
+
MaxKeys: 1e3
|
|
721
|
+
});
|
|
722
|
+
const response = await this.client.send(command);
|
|
723
|
+
const entries = [];
|
|
724
|
+
if (response.Versions) {
|
|
725
|
+
for (const version of response.Versions) if (version.Key === key && version.VersionId) {
|
|
726
|
+
const versionPath = `/${normalizedPath}/@versions/${version.VersionId}`;
|
|
727
|
+
entries.push({
|
|
728
|
+
id: `${this.generateId(key)}:${version.VersionId}`,
|
|
729
|
+
path: versionPath,
|
|
730
|
+
updatedAt: version.LastModified,
|
|
731
|
+
metadata: {
|
|
732
|
+
versionId: version.VersionId,
|
|
733
|
+
isLatest: version.IsLatest ?? false,
|
|
734
|
+
lastModified: version.LastModified?.toISOString(),
|
|
735
|
+
size: version.Size ?? 0,
|
|
736
|
+
etag: version.ETag?.replace(/"/g, "")
|
|
737
|
+
}
|
|
738
|
+
});
|
|
739
|
+
}
|
|
740
|
+
}
|
|
741
|
+
return { data: entries };
|
|
742
|
+
} catch (error) {
|
|
743
|
+
throw mapS3Error(error, `/${ctx.params.path}/@versions`);
|
|
744
|
+
}
|
|
745
|
+
}
|
|
746
|
+
async readHandler(ctx) {
|
|
747
|
+
const normalizedOutputPath = ctx.path.startsWith("/") ? ctx.path : `/${ctx.path}`;
|
|
748
|
+
try {
|
|
749
|
+
const key = this.buildS3Key(ctx.params.path);
|
|
750
|
+
if (!key) {
|
|
751
|
+
const listCommand = new ListObjectsV2Command({
|
|
752
|
+
Bucket: this.options.bucket,
|
|
753
|
+
Prefix: this.options.prefix ? `${this.options.prefix}/` : "",
|
|
754
|
+
Delimiter: "/",
|
|
755
|
+
MaxKeys: 1e3
|
|
756
|
+
});
|
|
757
|
+
const listResponse = await this.client.send(listCommand);
|
|
758
|
+
const childrenCount = (listResponse.CommonPrefixes?.length ?? 0) + (listResponse.Contents?.length ?? 0);
|
|
759
|
+
return {
|
|
760
|
+
id: this.generateId("/"),
|
|
761
|
+
path: "/",
|
|
762
|
+
content: "",
|
|
763
|
+
metadata: {
|
|
764
|
+
kind: "afs:node",
|
|
765
|
+
childrenCount,
|
|
766
|
+
platformRef: generatePlatformRef(this.options.bucket, this.options.region, "", true)
|
|
767
|
+
}
|
|
768
|
+
};
|
|
769
|
+
}
|
|
770
|
+
try {
|
|
771
|
+
const command = new GetObjectCommand({
|
|
772
|
+
Bucket: this.options.bucket,
|
|
773
|
+
Key: key
|
|
774
|
+
});
|
|
775
|
+
const response = await this.client.send(command);
|
|
776
|
+
if (key.endsWith("/") || response.ContentType === "application/x-directory") return {
|
|
777
|
+
id: this.generateId(key),
|
|
778
|
+
path: normalizedOutputPath,
|
|
779
|
+
content: "",
|
|
780
|
+
updatedAt: response.LastModified,
|
|
781
|
+
metadata: {
|
|
782
|
+
kind: "afs:node",
|
|
783
|
+
childrenCount: void 0,
|
|
784
|
+
platformRef: generatePlatformRef(this.options.bucket, this.options.region, key, true)
|
|
785
|
+
}
|
|
786
|
+
};
|
|
787
|
+
let content;
|
|
788
|
+
if (response.Body) {
|
|
789
|
+
const bytes = await response.Body.transformToByteArray();
|
|
790
|
+
content = new TextDecoder().decode(bytes);
|
|
791
|
+
} else content = "";
|
|
792
|
+
return {
|
|
793
|
+
id: this.generateId(key),
|
|
794
|
+
path: normalizedOutputPath,
|
|
795
|
+
content,
|
|
796
|
+
updatedAt: response.LastModified,
|
|
797
|
+
metadata: {
|
|
798
|
+
size: response.ContentLength,
|
|
799
|
+
mimeType: response.ContentType,
|
|
800
|
+
contentType: response.ContentType,
|
|
801
|
+
contentLength: response.ContentLength,
|
|
802
|
+
lastModified: response.LastModified?.toISOString(),
|
|
803
|
+
etag: response.ETag?.replace(/"/g, ""),
|
|
804
|
+
contentRange: response.ContentRange
|
|
805
|
+
}
|
|
806
|
+
};
|
|
807
|
+
} catch (error) {
|
|
808
|
+
if (error?.name === "NotFound" || error?.$metadata?.httpStatusCode === 404) {
|
|
809
|
+
const listCommand = new ListObjectsV2Command({
|
|
810
|
+
Bucket: this.options.bucket,
|
|
811
|
+
Prefix: `${key}/`,
|
|
812
|
+
Delimiter: "/",
|
|
813
|
+
MaxKeys: 1e3
|
|
814
|
+
});
|
|
815
|
+
const listResponse = await this.client.send(listCommand);
|
|
816
|
+
if (listResponse.Contents && listResponse.Contents.length > 0 || listResponse.CommonPrefixes && listResponse.CommonPrefixes.length > 0) {
|
|
817
|
+
const childrenCount = (listResponse.CommonPrefixes?.length ?? 0) + (listResponse.Contents?.length ?? 0);
|
|
818
|
+
return {
|
|
819
|
+
id: this.generateId(`${key}/`),
|
|
820
|
+
path: normalizedOutputPath,
|
|
821
|
+
content: "",
|
|
822
|
+
metadata: {
|
|
823
|
+
kind: "afs:node",
|
|
824
|
+
childrenCount,
|
|
825
|
+
platformRef: generatePlatformRef(this.options.bucket, this.options.region, key, true)
|
|
826
|
+
}
|
|
827
|
+
};
|
|
828
|
+
}
|
|
829
|
+
try {
|
|
830
|
+
const markerCommand = new HeadObjectCommand({
|
|
831
|
+
Bucket: this.options.bucket,
|
|
832
|
+
Key: `${key}/`
|
|
833
|
+
});
|
|
834
|
+
await this.client.send(markerCommand);
|
|
835
|
+
return {
|
|
836
|
+
id: this.generateId(`${key}/`),
|
|
837
|
+
path: normalizedOutputPath,
|
|
838
|
+
content: "",
|
|
839
|
+
metadata: {
|
|
840
|
+
kind: "afs:node",
|
|
841
|
+
childrenCount: 0,
|
|
842
|
+
platformRef: generatePlatformRef(this.options.bucket, this.options.region, key, true)
|
|
843
|
+
}
|
|
844
|
+
};
|
|
845
|
+
} catch {
|
|
846
|
+
throw new AFSNotFoundError(normalizedOutputPath);
|
|
847
|
+
}
|
|
848
|
+
}
|
|
849
|
+
throw error;
|
|
850
|
+
}
|
|
851
|
+
} catch (error) {
|
|
852
|
+
if (error instanceof AFSNotFoundError) throw error;
|
|
853
|
+
throw mapS3Error(error, normalizedOutputPath);
|
|
854
|
+
}
|
|
855
|
+
}
|
|
856
|
+
async readVersionHandler(ctx) {
|
|
857
|
+
try {
|
|
858
|
+
const normalizedPath = ctx.params.path.replace(/^\/+/, "").replace(/\/+$/, "");
|
|
859
|
+
const key = this.options.prefix ? `${this.options.prefix}/${normalizedPath}` : normalizedPath;
|
|
860
|
+
const command = new GetObjectCommand({
|
|
861
|
+
Bucket: this.options.bucket,
|
|
862
|
+
Key: key,
|
|
863
|
+
VersionId: ctx.params.versionId
|
|
864
|
+
});
|
|
865
|
+
const response = await this.client.send(command);
|
|
866
|
+
const content = response.Body ? await response.Body.transformToString() : "";
|
|
867
|
+
return {
|
|
868
|
+
id: `${this.generateId(key)}:${ctx.params.versionId}`,
|
|
869
|
+
path: ctx.path,
|
|
870
|
+
content,
|
|
871
|
+
updatedAt: response.LastModified,
|
|
872
|
+
metadata: {
|
|
873
|
+
size: response.ContentLength,
|
|
874
|
+
contentType: response.ContentType,
|
|
875
|
+
lastModified: response.LastModified?.toISOString(),
|
|
876
|
+
etag: response.ETag?.replace(/"/g, ""),
|
|
877
|
+
versionId: response.VersionId
|
|
878
|
+
}
|
|
879
|
+
};
|
|
880
|
+
} catch (error) {
|
|
881
|
+
throw mapS3Error(error, `/${ctx.params.path}/@versions/${ctx.params.versionId}`);
|
|
882
|
+
}
|
|
883
|
+
}
|
|
884
|
+
async metaHandler(ctx) {
|
|
885
|
+
try {
|
|
886
|
+
const path = ctx.params.path ?? "";
|
|
887
|
+
const normalizedPath = path.replace(/^\/+/, "").replace(/\/+$/, "");
|
|
888
|
+
const key = this.options.prefix ? normalizedPath ? `${this.options.prefix}/${normalizedPath}` : this.options.prefix : normalizedPath;
|
|
889
|
+
if (!key) {
|
|
890
|
+
const listCommand = new ListObjectsV2Command({
|
|
891
|
+
Bucket: this.options.bucket,
|
|
892
|
+
Prefix: this.options.prefix ? `${this.options.prefix}/` : "",
|
|
893
|
+
Delimiter: "/",
|
|
894
|
+
MaxKeys: 1e3
|
|
895
|
+
});
|
|
896
|
+
const listResponse = await this.client.send(listCommand);
|
|
897
|
+
const childrenCount = (listResponse.CommonPrefixes?.length ?? 0) + (listResponse.Contents?.length ?? 0);
|
|
898
|
+
return {
|
|
899
|
+
id: this.generateId("/"),
|
|
900
|
+
path: "/.meta",
|
|
901
|
+
metadata: {
|
|
902
|
+
kind: "afs:node",
|
|
903
|
+
childrenCount,
|
|
904
|
+
platformRef: generatePlatformRef(this.options.bucket, this.options.region, "", true)
|
|
905
|
+
}
|
|
906
|
+
};
|
|
907
|
+
}
|
|
908
|
+
if (this.statCache) {
|
|
909
|
+
const cacheKey = createCacheKey(this.options.bucket, this.options.prefix ?? "", path);
|
|
910
|
+
const cached = this.statCache.get(cacheKey);
|
|
911
|
+
if (cached) return {
|
|
912
|
+
id: cached.id,
|
|
913
|
+
path: ctx.path,
|
|
914
|
+
metadata: cached.metadata
|
|
915
|
+
};
|
|
916
|
+
}
|
|
917
|
+
try {
|
|
918
|
+
const command = new HeadObjectCommand({
|
|
919
|
+
Bucket: this.options.bucket,
|
|
920
|
+
Key: key
|
|
921
|
+
});
|
|
922
|
+
const response = await this.client.send(command);
|
|
923
|
+
if (key.endsWith("/") || response.ContentType === "application/x-directory") return {
|
|
924
|
+
id: this.generateId(key),
|
|
925
|
+
path: ctx.path,
|
|
926
|
+
updatedAt: response.LastModified,
|
|
927
|
+
metadata: {
|
|
928
|
+
kind: "afs:node",
|
|
929
|
+
childrenCount: void 0,
|
|
930
|
+
platformRef: generatePlatformRef(this.options.bucket, this.options.region, key, true)
|
|
931
|
+
}
|
|
932
|
+
};
|
|
933
|
+
const result = {
|
|
934
|
+
id: this.generateId(key),
|
|
935
|
+
path: ctx.path,
|
|
936
|
+
updatedAt: response.LastModified,
|
|
937
|
+
metadata: {
|
|
938
|
+
kind: "afs:document",
|
|
939
|
+
size: response.ContentLength,
|
|
940
|
+
contentType: response.ContentType,
|
|
941
|
+
lastModified: response.LastModified?.toISOString(),
|
|
942
|
+
etag: response.ETag?.replace(/"/g, ""),
|
|
943
|
+
storageClass: response.StorageClass,
|
|
944
|
+
platformRef: generatePlatformRef(this.options.bucket, this.options.region, key, false)
|
|
945
|
+
}
|
|
946
|
+
};
|
|
947
|
+
if (this.statCache) {
|
|
948
|
+
const cacheKey = createCacheKey(this.options.bucket, this.options.prefix ?? "", path);
|
|
949
|
+
this.statCache.set(cacheKey, result);
|
|
950
|
+
}
|
|
951
|
+
return result;
|
|
952
|
+
} catch (error) {
|
|
953
|
+
if (error?.name === "NotFound" || error?.$metadata?.httpStatusCode === 404) {
|
|
954
|
+
const listCommand = new ListObjectsV2Command({
|
|
955
|
+
Bucket: this.options.bucket,
|
|
956
|
+
Prefix: `${key}/`,
|
|
957
|
+
MaxKeys: 1
|
|
958
|
+
});
|
|
959
|
+
const listResponse = await this.client.send(listCommand);
|
|
960
|
+
if (listResponse.Contents && listResponse.Contents.length > 0) return {
|
|
961
|
+
id: this.generateId(`${key}/`),
|
|
962
|
+
path: ctx.path,
|
|
963
|
+
metadata: {
|
|
964
|
+
kind: "afs:node",
|
|
965
|
+
childrenCount: void 0,
|
|
966
|
+
platformRef: generatePlatformRef(this.options.bucket, this.options.region, key, true)
|
|
967
|
+
}
|
|
968
|
+
};
|
|
969
|
+
try {
|
|
970
|
+
const markerCommand = new HeadObjectCommand({
|
|
971
|
+
Bucket: this.options.bucket,
|
|
972
|
+
Key: `${key}/`
|
|
973
|
+
});
|
|
974
|
+
const markerResponse = await this.client.send(markerCommand);
|
|
975
|
+
return {
|
|
976
|
+
id: this.generateId(`${key}/`),
|
|
977
|
+
path: ctx.path,
|
|
978
|
+
updatedAt: markerResponse.LastModified,
|
|
979
|
+
metadata: {
|
|
980
|
+
kind: "afs:node",
|
|
981
|
+
childrenCount: void 0,
|
|
982
|
+
platformRef: generatePlatformRef(this.options.bucket, this.options.region, key, true)
|
|
983
|
+
}
|
|
984
|
+
};
|
|
985
|
+
} catch {
|
|
986
|
+
throw new AFSNotFoundError(path.startsWith("/") ? path : `/${path}`);
|
|
987
|
+
}
|
|
988
|
+
}
|
|
989
|
+
throw error;
|
|
990
|
+
}
|
|
991
|
+
} catch (error) {
|
|
992
|
+
if (error instanceof AFSNotFoundError) throw error;
|
|
993
|
+
throw mapS3Error(error, (ctx.params.path ?? "").startsWith("/") ? ctx.params.path ?? "/" : `/${ctx.params.path ?? ""}`);
|
|
994
|
+
}
|
|
995
|
+
}
|
|
996
|
+
async statHandler(ctx) {
|
|
997
|
+
const metaEntry = await this.metaHandler({
|
|
998
|
+
...ctx,
|
|
999
|
+
path: ctx.path.endsWith("/.meta") ? ctx.path : `${ctx.path}/.meta`
|
|
1000
|
+
});
|
|
1001
|
+
return { data: {
|
|
1002
|
+
path: ctx.path,
|
|
1003
|
+
size: metaEntry.metadata?.size,
|
|
1004
|
+
childrenCount: metaEntry.metadata?.childrenCount,
|
|
1005
|
+
meta: metaEntry.metadata
|
|
1006
|
+
} };
|
|
1007
|
+
}
|
|
1008
|
+
async writeHandler(ctx, payload) {
|
|
1009
|
+
try {
|
|
1010
|
+
const key = this.buildS3Key(ctx.params.path);
|
|
1011
|
+
let body;
|
|
1012
|
+
if (typeof payload.content === "string") body = Buffer.from(payload.content, "utf-8");
|
|
1013
|
+
else if (Buffer.isBuffer(payload.content)) body = payload.content;
|
|
1014
|
+
else if (payload.content !== void 0) body = Buffer.from(JSON.stringify(payload.content), "utf-8");
|
|
1015
|
+
else body = Buffer.from("");
|
|
1016
|
+
const contentType = payload.metadata?.mimeType ?? payload.metadata?.contentType ?? "application/octet-stream";
|
|
1017
|
+
let etag;
|
|
1018
|
+
let versionId;
|
|
1019
|
+
if (shouldUseMultipart(body.length)) {
|
|
1020
|
+
const result = await multipartUpload(this.client, this.options.bucket, key, body, { contentType });
|
|
1021
|
+
etag = result.etag;
|
|
1022
|
+
versionId = result.versionId;
|
|
1023
|
+
} else {
|
|
1024
|
+
const command = new PutObjectCommand({
|
|
1025
|
+
Bucket: this.options.bucket,
|
|
1026
|
+
Key: key,
|
|
1027
|
+
Body: body,
|
|
1028
|
+
ContentType: contentType
|
|
1029
|
+
});
|
|
1030
|
+
const response = await this.client.send(command);
|
|
1031
|
+
etag = response.ETag?.replace(/"/g, "");
|
|
1032
|
+
versionId = response.VersionId;
|
|
1033
|
+
}
|
|
1034
|
+
const normalizedOutputPath = ctx.path.startsWith("/") ? ctx.path : `/${ctx.path}`;
|
|
1035
|
+
this.invalidateCache(ctx.params.path);
|
|
1036
|
+
return { data: {
|
|
1037
|
+
id: this.generateId(key),
|
|
1038
|
+
path: normalizedOutputPath,
|
|
1039
|
+
content: payload.content,
|
|
1040
|
+
metadata: {
|
|
1041
|
+
size: body.length,
|
|
1042
|
+
etag,
|
|
1043
|
+
versionId,
|
|
1044
|
+
...payload.metadata
|
|
1045
|
+
}
|
|
1046
|
+
} };
|
|
1047
|
+
} catch (error) {
|
|
1048
|
+
throw mapS3Error(error, ctx.params.path.startsWith("/") ? ctx.params.path : `/${ctx.params.path}`);
|
|
1049
|
+
}
|
|
1050
|
+
}
|
|
1051
|
+
async deleteHandler(ctx) {
|
|
1052
|
+
try {
|
|
1053
|
+
const key = this.buildS3Key(ctx.params.path);
|
|
1054
|
+
try {
|
|
1055
|
+
const headCommand = new HeadObjectCommand({
|
|
1056
|
+
Bucket: this.options.bucket,
|
|
1057
|
+
Key: key
|
|
1058
|
+
});
|
|
1059
|
+
await this.client.send(headCommand);
|
|
1060
|
+
} catch (error) {
|
|
1061
|
+
if (error?.name === "NotFound" || error?.$metadata?.httpStatusCode === 404) throw new AFSNotFoundError(`/${ctx.params.path}`);
|
|
1062
|
+
throw error;
|
|
1063
|
+
}
|
|
1064
|
+
const command = new DeleteObjectCommand({
|
|
1065
|
+
Bucket: this.options.bucket,
|
|
1066
|
+
Key: key
|
|
1067
|
+
});
|
|
1068
|
+
await this.client.send(command);
|
|
1069
|
+
this.invalidateCache(ctx.params.path);
|
|
1070
|
+
return { message: `Successfully deleted: ${ctx.params.path}` };
|
|
1071
|
+
} catch (error) {
|
|
1072
|
+
if (error instanceof AFSNotFoundError) throw error;
|
|
1073
|
+
throw mapS3Error(error, `/${ctx.params.path}`);
|
|
1074
|
+
}
|
|
1075
|
+
}
|
|
1076
|
+
async listActionsHandler(ctx) {
|
|
1077
|
+
const basePath = ctx.path.replace(/\/\.actions$/, "");
|
|
1078
|
+
return { data: [
|
|
1079
|
+
{
|
|
1080
|
+
id: "select",
|
|
1081
|
+
path: `${basePath}/.actions/select`,
|
|
1082
|
+
summary: "Query with S3 Select",
|
|
1083
|
+
metadata: {
|
|
1084
|
+
kind: "afs:executable",
|
|
1085
|
+
kinds: ["afs:executable", "afs:node"]
|
|
1086
|
+
}
|
|
1087
|
+
},
|
|
1088
|
+
{
|
|
1089
|
+
id: "presign-download",
|
|
1090
|
+
path: `${basePath}/.actions/presign-download`,
|
|
1091
|
+
summary: "Generate download URL",
|
|
1092
|
+
metadata: {
|
|
1093
|
+
kind: "afs:executable",
|
|
1094
|
+
kinds: ["afs:executable", "afs:node"]
|
|
1095
|
+
}
|
|
1096
|
+
},
|
|
1097
|
+
{
|
|
1098
|
+
id: "presign-upload",
|
|
1099
|
+
path: `${basePath}/.actions/presign-upload`,
|
|
1100
|
+
summary: "Generate upload URL",
|
|
1101
|
+
metadata: {
|
|
1102
|
+
kind: "afs:executable",
|
|
1103
|
+
kinds: ["afs:executable", "afs:node"]
|
|
1104
|
+
}
|
|
1105
|
+
}
|
|
1106
|
+
] };
|
|
1107
|
+
}
|
|
1108
|
+
async selectActionHandler(ctx, args) {
|
|
1109
|
+
const expression = args.expression;
|
|
1110
|
+
if (!expression) throw new AFSError("Missing required argument: expression", "AFS_INVALID_ARGUMENT");
|
|
1111
|
+
const normalizedPath = ctx.params.path.replace(/^\/+/, "").replace(/\/+$/, "");
|
|
1112
|
+
const key = this.options.prefix ? `${this.options.prefix}/${normalizedPath}` : normalizedPath;
|
|
1113
|
+
const result = await selectQuery(this.client, this.options.bucket, "", key, expression, {
|
|
1114
|
+
inputFormat: args.inputFormat,
|
|
1115
|
+
outputFormat: args.outputFormat
|
|
1116
|
+
});
|
|
1117
|
+
return {
|
|
1118
|
+
success: true,
|
|
1119
|
+
data: {
|
|
1120
|
+
records: result.records,
|
|
1121
|
+
stats: result.stats
|
|
1122
|
+
}
|
|
1123
|
+
};
|
|
1124
|
+
}
|
|
1125
|
+
async presignDownloadActionHandler(ctx, args) {
|
|
1126
|
+
const normalizedPath = ctx.params.path.replace(/^\/+/, "").replace(/\/+$/, "");
|
|
1127
|
+
const key = this.options.prefix ? `${this.options.prefix}/${normalizedPath}` : normalizedPath;
|
|
1128
|
+
let expiresIn = args.expiresIn ?? DEFAULT_EXPIRES_IN;
|
|
1129
|
+
if (expiresIn > MAX_EXPIRES_IN) expiresIn = MAX_EXPIRES_IN;
|
|
1130
|
+
if (expiresIn < 1) expiresIn = 1;
|
|
1131
|
+
const command = new GetObjectCommand({
|
|
1132
|
+
Bucket: this.options.bucket,
|
|
1133
|
+
Key: key
|
|
1134
|
+
});
|
|
1135
|
+
return {
|
|
1136
|
+
success: true,
|
|
1137
|
+
data: {
|
|
1138
|
+
url: await getSignedUrl(this.client, command, { expiresIn }),
|
|
1139
|
+
expiresAt: new Date(Date.now() + expiresIn * 1e3).toISOString()
|
|
1140
|
+
}
|
|
1141
|
+
};
|
|
1142
|
+
}
|
|
1143
|
+
async presignUploadActionHandler(ctx, args) {
|
|
1144
|
+
const normalizedPath = ctx.params.path.replace(/^\/+/, "").replace(/\/+$/, "");
|
|
1145
|
+
const key = this.options.prefix ? `${this.options.prefix}/${normalizedPath}` : normalizedPath;
|
|
1146
|
+
let expiresIn = args.expiresIn ?? DEFAULT_EXPIRES_IN;
|
|
1147
|
+
if (expiresIn > MAX_EXPIRES_IN) expiresIn = MAX_EXPIRES_IN;
|
|
1148
|
+
if (expiresIn < 1) expiresIn = 1;
|
|
1149
|
+
const command = new PutObjectCommand({
|
|
1150
|
+
Bucket: this.options.bucket,
|
|
1151
|
+
Key: key,
|
|
1152
|
+
ContentType: args.contentType ?? "application/octet-stream"
|
|
1153
|
+
});
|
|
1154
|
+
return {
|
|
1155
|
+
success: true,
|
|
1156
|
+
data: {
|
|
1157
|
+
url: await getSignedUrl(this.client, command, { expiresIn }),
|
|
1158
|
+
expiresAt: new Date(Date.now() + expiresIn * 1e3).toISOString()
|
|
1159
|
+
}
|
|
1160
|
+
};
|
|
1161
|
+
}
|
|
1162
|
+
/**
|
|
1163
|
+
* Generate a presigned URL for downloading an object
|
|
1164
|
+
* @deprecated Use action /.actions/presign-download instead
|
|
1165
|
+
*/
|
|
1166
|
+
async getPresignedDownloadUrl(path, options) {
|
|
1167
|
+
return (await this.presignDownloadActionHandler({
|
|
1168
|
+
path: `/${path}`,
|
|
1169
|
+
params: { path },
|
|
1170
|
+
options: {}
|
|
1171
|
+
}, { expiresIn: options?.expiresIn })).data.url;
|
|
1172
|
+
}
|
|
1173
|
+
/**
|
|
1174
|
+
* Generate a presigned URL for uploading an object
|
|
1175
|
+
* @deprecated Use action /.actions/presign-upload instead
|
|
1176
|
+
*/
|
|
1177
|
+
async getPresignedUploadUrl(path, options) {
|
|
1178
|
+
return (await this.presignUploadActionHandler({
|
|
1179
|
+
path: `/${path}`,
|
|
1180
|
+
params: { path },
|
|
1181
|
+
options: {}
|
|
1182
|
+
}, {
|
|
1183
|
+
expiresIn: options?.expiresIn,
|
|
1184
|
+
contentType: options?.contentType
|
|
1185
|
+
})).data.url;
|
|
1186
|
+
}
|
|
1187
|
+
/**
|
|
1188
|
+
* List all versions of an object
|
|
1189
|
+
* @deprecated Use list on /:path/@versions instead
|
|
1190
|
+
*/
|
|
1191
|
+
async listVersions(path) {
|
|
1192
|
+
return (await this.listVersionsHandler({
|
|
1193
|
+
path: `/${path}/@versions`,
|
|
1194
|
+
params: { path },
|
|
1195
|
+
options: {}
|
|
1196
|
+
})).data.map((entry) => ({
|
|
1197
|
+
versionId: entry.metadata?.versionId,
|
|
1198
|
+
isLatest: entry.metadata?.isLatest,
|
|
1199
|
+
lastModified: entry.updatedAt,
|
|
1200
|
+
size: entry.metadata?.size ?? 0,
|
|
1201
|
+
etag: entry.metadata?.etag
|
|
1202
|
+
}));
|
|
1203
|
+
}
|
|
1204
|
+
/**
|
|
1205
|
+
* Read a specific version of an object
|
|
1206
|
+
* @deprecated Use read on /:path/@versions/:versionId instead
|
|
1207
|
+
*/
|
|
1208
|
+
async readVersion(path, versionId) {
|
|
1209
|
+
const result = await this.readVersionHandler({
|
|
1210
|
+
path: `/${path}/@versions/${versionId}`,
|
|
1211
|
+
params: {
|
|
1212
|
+
path,
|
|
1213
|
+
versionId
|
|
1214
|
+
},
|
|
1215
|
+
options: {}
|
|
1216
|
+
});
|
|
1217
|
+
return {
|
|
1218
|
+
content: result.content,
|
|
1219
|
+
metadata: result.metadata
|
|
1220
|
+
};
|
|
1221
|
+
}
|
|
1222
|
+
/**
|
|
1223
|
+
* Run a SQL-like query on a CSV/JSON/Parquet file (S3 Select)
|
|
1224
|
+
* @deprecated Use action /.actions/select instead
|
|
1225
|
+
*/
|
|
1226
|
+
async select(path, query, options) {
|
|
1227
|
+
return selectQuery(this.client, this.options.bucket, this.options.prefix ?? "", path, query, options);
|
|
1228
|
+
}
|
|
1229
|
+
/**
|
|
1230
|
+
* Create a directory marker
|
|
1231
|
+
* @deprecated Use write with empty content instead
|
|
1232
|
+
*/
|
|
1233
|
+
async mkdir(path) {
|
|
1234
|
+
const key = this.buildS3Key(path);
|
|
1235
|
+
const dirKey = key.endsWith("/") ? key : `${key}/`;
|
|
1236
|
+
const command = new PutObjectCommand({
|
|
1237
|
+
Bucket: this.options.bucket,
|
|
1238
|
+
Key: dirKey,
|
|
1239
|
+
Body: Buffer.from(""),
|
|
1240
|
+
ContentType: "application/x-directory"
|
|
1241
|
+
});
|
|
1242
|
+
await this.client.send(command);
|
|
1243
|
+
}
|
|
1244
|
+
};
|
|
1245
|
+
__decorate([List("/"), List("/:path*")], AFSS3.prototype, "listHandler", null);
|
|
1246
|
+
__decorate([List("/:path*/@versions")], AFSS3.prototype, "listVersionsHandler", null);
|
|
1247
|
+
__decorate([Read("/:path*")], AFSS3.prototype, "readHandler", null);
|
|
1248
|
+
__decorate([Read("/:path*/@versions/:versionId")], AFSS3.prototype, "readVersionHandler", null);
|
|
1249
|
+
__decorate([Meta("/"), Meta("/:path*")], AFSS3.prototype, "metaHandler", null);
|
|
1250
|
+
__decorate([Stat("/"), Stat("/:path*")], AFSS3.prototype, "statHandler", null);
|
|
1251
|
+
__decorate([Write("/:path*")], AFSS3.prototype, "writeHandler", null);
|
|
1252
|
+
__decorate([Delete("/:path*")], AFSS3.prototype, "deleteHandler", null);
|
|
1253
|
+
__decorate([Actions("/:path*")], AFSS3.prototype, "listActionsHandler", null);
|
|
1254
|
+
__decorate([Actions.Exec("/:path*", "select")], AFSS3.prototype, "selectActionHandler", null);
|
|
1255
|
+
__decorate([Actions.Exec("/:path*", "presign-download")], AFSS3.prototype, "presignDownloadActionHandler", null);
|
|
1256
|
+
__decorate([Actions.Exec("/:path*", "presign-upload")], AFSS3.prototype, "presignUploadActionHandler", null);
|
|
1257
|
+
|
|
1258
|
+
//#endregion
|
|
1259
|
+
export { AFSS3 };
|
|
1260
|
+
//# sourceMappingURL=index.mjs.map
|