@statewalker/webrun-files-s3 0.5.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -1
- package/README.md +81 -148
- package/dist/cjs/index.cjs +171 -432
- package/dist/esm/index.js +160 -421
- package/dist/index.d.ts +1 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/s3-files-api.d.ts +22 -53
- package/dist/s3-files-api.d.ts.map +1 -1
- package/package.json +15 -15
- package/src/index.ts +1 -2
- package/src/s3-files-api.ts +253 -176
- package/dist/s3-file-handle.d.ts +0 -75
- package/dist/s3-file-handle.d.ts.map +0 -1
- package/src/s3-file-handle.ts +0 -491
package/src/s3-files-api.ts
CHANGED
|
@@ -1,33 +1,40 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* S3 implementation of
|
|
2
|
+
* S3 implementation of FilesApi
|
|
3
3
|
*
|
|
4
4
|
* Provides a filesystem-like interface over S3 object storage.
|
|
5
|
-
* Supports all IFilesApi methods including the optional move, copy, and mkdir.
|
|
6
5
|
*/
|
|
7
6
|
|
|
8
7
|
import type { S3Client } from "@aws-sdk/client-s3";
|
|
9
8
|
import {
|
|
9
|
+
AbortMultipartUploadCommand,
|
|
10
|
+
CompleteMultipartUploadCommand,
|
|
10
11
|
CopyObjectCommand,
|
|
12
|
+
CreateMultipartUploadCommand,
|
|
11
13
|
DeleteObjectCommand,
|
|
14
|
+
GetObjectCommand,
|
|
12
15
|
HeadObjectCommand,
|
|
13
16
|
ListObjectsV2Command,
|
|
14
17
|
PutObjectCommand,
|
|
18
|
+
UploadPartCommand,
|
|
15
19
|
} from "@aws-sdk/client-s3";
|
|
16
20
|
import type {
|
|
17
|
-
CopyOptions,
|
|
18
|
-
FileHandle,
|
|
19
21
|
FileInfo,
|
|
20
|
-
|
|
21
|
-
|
|
22
|
+
FileStats,
|
|
23
|
+
FilesApi,
|
|
22
24
|
ListOptions,
|
|
25
|
+
ReadOptions,
|
|
23
26
|
} from "@statewalker/webrun-files";
|
|
24
|
-
import { basename,
|
|
25
|
-
|
|
27
|
+
import { basename, normalizePath } from "@statewalker/webrun-files";
|
|
28
|
+
|
|
29
|
+
/** Default part size for multipart uploads: 5MB (S3 minimum) */
|
|
30
|
+
const DEFAULT_PART_SIZE = 5 * 1024 * 1024;
|
|
31
|
+
|
|
32
|
+
/** Threshold for using multipart upload: 5MB */
|
|
33
|
+
const MULTIPART_THRESHOLD = 5 * 1024 * 1024;
|
|
26
34
|
|
|
27
35
|
export interface S3FilesApiOptions {
|
|
28
36
|
/**
|
|
29
37
|
* Pre-configured S3Client instance.
|
|
30
|
-
* Allows full control over credentials, region, endpoint.
|
|
31
38
|
*/
|
|
32
39
|
client: S3Client;
|
|
33
40
|
|
|
@@ -38,42 +45,40 @@ export interface S3FilesApiOptions {
|
|
|
38
45
|
|
|
39
46
|
/**
|
|
40
47
|
* Optional key prefix (acts as root directory).
|
|
41
|
-
* All paths will be relative to this prefix.
|
|
42
48
|
* @example "projects/my-app/data"
|
|
43
49
|
*/
|
|
44
50
|
prefix?: string;
|
|
45
51
|
|
|
46
52
|
/**
|
|
47
|
-
*
|
|
48
|
-
*
|
|
53
|
+
* Part size for multipart uploads in bytes.
|
|
54
|
+
* S3 requires minimum 5MB for all parts except the last.
|
|
55
|
+
* @default 5242880 (5MB)
|
|
49
56
|
*/
|
|
50
57
|
multipartPartSize?: number;
|
|
51
58
|
}
|
|
52
59
|
|
|
53
|
-
|
|
60
|
+
/**
|
|
61
|
+
* S3 filesystem implementation of FilesApi.
|
|
62
|
+
*
|
|
63
|
+
* Provides a filesystem-like interface over S3 object storage.
|
|
64
|
+
* Works with AWS S3 and S3-compatible services.
|
|
65
|
+
*/
|
|
66
|
+
export class S3FilesApi implements FilesApi {
|
|
54
67
|
private client: S3Client;
|
|
55
68
|
private bucket: string;
|
|
56
69
|
private prefix: string;
|
|
57
|
-
private
|
|
70
|
+
private partSize: number;
|
|
58
71
|
|
|
59
72
|
constructor(options: S3FilesApiOptions) {
|
|
60
73
|
this.client = options.client;
|
|
61
74
|
this.bucket = options.bucket;
|
|
62
|
-
// Normalize prefix: remove leading/trailing slashes
|
|
63
75
|
this.prefix = (options.prefix ?? "").replace(/^\/+|\/+$/g, "");
|
|
64
|
-
this.
|
|
76
|
+
this.partSize = options.multipartPartSize ?? DEFAULT_PART_SIZE;
|
|
65
77
|
}
|
|
66
78
|
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
*/
|
|
71
|
-
private resolveKey(file: FileRef): string {
|
|
72
|
-
const normalized = resolveFileRef(file);
|
|
73
|
-
// Remove leading slash and combine with prefix
|
|
74
|
-
const relativePath = normalized.startsWith("/")
|
|
75
|
-
? normalized.substring(1)
|
|
76
|
-
: normalized;
|
|
79
|
+
private resolveKey(path: string): string {
|
|
80
|
+
const normalized = normalizePath(path);
|
|
81
|
+
const relativePath = normalized.startsWith("/") ? normalized.substring(1) : normalized;
|
|
77
82
|
|
|
78
83
|
if (this.prefix) {
|
|
79
84
|
return relativePath ? `${this.prefix}/${relativePath}` : this.prefix;
|
|
@@ -81,9 +86,6 @@ export class S3FilesApi implements IFilesApi {
|
|
|
81
86
|
return relativePath;
|
|
82
87
|
}
|
|
83
88
|
|
|
84
|
-
/**
|
|
85
|
-
* Converts an S3 key back to a virtual path.
|
|
86
|
-
*/
|
|
87
89
|
private keyToPath(key: string): string {
|
|
88
90
|
let relativePath = key;
|
|
89
91
|
if (this.prefix && key.startsWith(this.prefix)) {
|
|
@@ -92,21 +94,193 @@ export class S3FilesApi implements IFilesApi {
|
|
|
92
94
|
relativePath = relativePath.substring(1);
|
|
93
95
|
}
|
|
94
96
|
}
|
|
95
|
-
return
|
|
97
|
+
return `/${relativePath}`;
|
|
96
98
|
}
|
|
97
99
|
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
100
|
+
async *read(path: string, options?: ReadOptions): AsyncIterable<Uint8Array> {
|
|
101
|
+
const key = this.resolveKey(path);
|
|
102
|
+
|
|
103
|
+
try {
|
|
104
|
+
const start = options?.start ?? 0;
|
|
105
|
+
const length = options?.length;
|
|
106
|
+
|
|
107
|
+
// Return empty if length is explicitly 0
|
|
108
|
+
if (length === 0) {
|
|
109
|
+
return;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
let range: string | undefined;
|
|
113
|
+
if (start > 0 || length !== undefined) {
|
|
114
|
+
const end = length !== undefined ? start + length - 1 : "";
|
|
115
|
+
range = `bytes=${start}-${end}`;
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
const command = new GetObjectCommand({
|
|
119
|
+
Bucket: this.bucket,
|
|
120
|
+
Key: key,
|
|
121
|
+
Range: range,
|
|
122
|
+
});
|
|
123
|
+
|
|
124
|
+
const response = await this.client.send(command);
|
|
125
|
+
|
|
126
|
+
if (response.Body) {
|
|
127
|
+
const stream = response.Body as AsyncIterable<Uint8Array>;
|
|
128
|
+
for await (const chunk of stream) {
|
|
129
|
+
yield chunk;
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
} catch (error: unknown) {
|
|
133
|
+
const err = error as { name?: string; $metadata?: { httpStatusCode?: number } };
|
|
134
|
+
// Handle NotFound (404) and InvalidRange (416) - return empty
|
|
135
|
+
if (
|
|
136
|
+
err.name === "NotFound" ||
|
|
137
|
+
err.name === "InvalidRange" ||
|
|
138
|
+
err.$metadata?.httpStatusCode === 404 ||
|
|
139
|
+
err.$metadata?.httpStatusCode === 416
|
|
140
|
+
) {
|
|
141
|
+
return;
|
|
142
|
+
}
|
|
143
|
+
throw error;
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
async write(
|
|
148
|
+
path: string,
|
|
149
|
+
content: Iterable<Uint8Array> | AsyncIterable<Uint8Array>,
|
|
150
|
+
): Promise<void> {
|
|
151
|
+
const key = this.resolveKey(path);
|
|
152
|
+
|
|
153
|
+
// Buffer for accumulating small writes
|
|
154
|
+
let buffer: Uint8Array[] = [];
|
|
155
|
+
let bufferSize = 0;
|
|
156
|
+
let uploadId: string | undefined;
|
|
157
|
+
const parts: { ETag: string; PartNumber: number }[] = [];
|
|
158
|
+
let partNumber = 1;
|
|
159
|
+
|
|
160
|
+
const flushPart = async (isLast: boolean) => {
|
|
161
|
+
if (buffer.length === 0) return;
|
|
162
|
+
|
|
163
|
+
const partLength = buffer.reduce((sum, c) => sum + c.length, 0);
|
|
164
|
+
const partBuffer = new Uint8Array(partLength);
|
|
165
|
+
let offset = 0;
|
|
166
|
+
for (const chunk of buffer) {
|
|
167
|
+
partBuffer.set(chunk, offset);
|
|
168
|
+
offset += chunk.length;
|
|
169
|
+
}
|
|
170
|
+
buffer = [];
|
|
171
|
+
bufferSize = 0;
|
|
172
|
+
|
|
173
|
+
// For small files (under threshold) and this is the only/last part, use simple PutObject
|
|
174
|
+
if (!uploadId && isLast && partLength < MULTIPART_THRESHOLD) {
|
|
175
|
+
const command = new PutObjectCommand({
|
|
176
|
+
Bucket: this.bucket,
|
|
177
|
+
Key: key,
|
|
178
|
+
Body: partBuffer,
|
|
179
|
+
ContentLength: partLength,
|
|
180
|
+
});
|
|
181
|
+
await this.client.send(command);
|
|
182
|
+
return;
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
// Start multipart upload if not already started
|
|
186
|
+
if (!uploadId) {
|
|
187
|
+
const createCommand = new CreateMultipartUploadCommand({
|
|
188
|
+
Bucket: this.bucket,
|
|
189
|
+
Key: key,
|
|
190
|
+
});
|
|
191
|
+
const { UploadId } = await this.client.send(createCommand);
|
|
192
|
+
if (!UploadId) {
|
|
193
|
+
throw new Error("Failed to create multipart upload");
|
|
194
|
+
}
|
|
195
|
+
uploadId = UploadId;
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
// Upload the part
|
|
199
|
+
const uploadCommand = new UploadPartCommand({
|
|
200
|
+
Bucket: this.bucket,
|
|
201
|
+
Key: key,
|
|
202
|
+
UploadId: uploadId,
|
|
203
|
+
PartNumber: partNumber,
|
|
204
|
+
Body: partBuffer,
|
|
205
|
+
});
|
|
206
|
+
|
|
207
|
+
const response = await this.client.send(uploadCommand);
|
|
208
|
+
if (!response.ETag) {
|
|
209
|
+
throw new Error(`Failed to upload part ${partNumber}`);
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
parts.push({
|
|
213
|
+
ETag: response.ETag,
|
|
214
|
+
PartNumber: partNumber,
|
|
215
|
+
});
|
|
216
|
+
partNumber++;
|
|
217
|
+
};
|
|
218
|
+
|
|
219
|
+
try {
|
|
220
|
+
// Process chunks as they arrive
|
|
221
|
+
for await (const chunk of content) {
|
|
222
|
+
buffer.push(chunk);
|
|
223
|
+
bufferSize += chunk.length;
|
|
224
|
+
|
|
225
|
+
// Flush when buffer reaches part size
|
|
226
|
+
if (bufferSize >= this.partSize) {
|
|
227
|
+
await flushPart(false);
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
// Flush remaining data
|
|
232
|
+
await flushPart(true);
|
|
233
|
+
|
|
234
|
+
// Complete multipart upload if one was started
|
|
235
|
+
if (uploadId) {
|
|
236
|
+
const completeCommand = new CompleteMultipartUploadCommand({
|
|
237
|
+
Bucket: this.bucket,
|
|
238
|
+
Key: key,
|
|
239
|
+
UploadId: uploadId,
|
|
240
|
+
MultipartUpload: {
|
|
241
|
+
Parts: parts.sort((a, b) => a.PartNumber - b.PartNumber),
|
|
242
|
+
},
|
|
243
|
+
});
|
|
244
|
+
await this.client.send(completeCommand);
|
|
245
|
+
}
|
|
246
|
+
} catch (error) {
|
|
247
|
+
// Abort multipart upload on error
|
|
248
|
+
if (uploadId) {
|
|
249
|
+
const abortCommand = new AbortMultipartUploadCommand({
|
|
250
|
+
Bucket: this.bucket,
|
|
251
|
+
Key: key,
|
|
252
|
+
UploadId: uploadId,
|
|
253
|
+
});
|
|
254
|
+
|
|
255
|
+
try {
|
|
256
|
+
await this.client.send(abortCommand);
|
|
257
|
+
} catch {
|
|
258
|
+
// Ignore abort errors
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
throw error;
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
async mkdir(path: string): Promise<void> {
|
|
267
|
+
const key = this.resolveKey(path);
|
|
268
|
+
const dirKey = key.endsWith("/") ? key : `${key}/`;
|
|
269
|
+
|
|
270
|
+
const command = new PutObjectCommand({
|
|
271
|
+
Bucket: this.bucket,
|
|
272
|
+
Key: dirKey,
|
|
273
|
+
Body: new Uint8Array(0),
|
|
274
|
+
ContentLength: 0,
|
|
275
|
+
});
|
|
276
|
+
|
|
277
|
+
await this.client.send(command);
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
async *list(path: string, options?: ListOptions): AsyncIterable<FileInfo> {
|
|
281
|
+
const recursive = options?.recursive ?? false;
|
|
282
|
+
const prefix = this.resolveKey(path);
|
|
283
|
+
const normalizedPrefix = prefix ? `${prefix}/` : this.prefix ? `${this.prefix}/` : "";
|
|
110
284
|
|
|
111
285
|
let continuationToken: string | undefined;
|
|
112
286
|
|
|
@@ -121,45 +295,45 @@ export class S3FilesApi implements IFilesApi {
|
|
|
121
295
|
|
|
122
296
|
const response = await this.client.send(command);
|
|
123
297
|
|
|
124
|
-
// Yield directories (CommonPrefixes)
|
|
125
298
|
if (!recursive && response.CommonPrefixes) {
|
|
126
299
|
for (const cpPrefix of response.CommonPrefixes) {
|
|
127
300
|
if (!cpPrefix.Prefix) continue;
|
|
128
301
|
|
|
129
|
-
// Remove trailing slash
|
|
130
302
|
const dirKey = cpPrefix.Prefix.endsWith("/")
|
|
131
303
|
? cpPrefix.Prefix.slice(0, -1)
|
|
132
304
|
: cpPrefix.Prefix;
|
|
133
305
|
|
|
134
|
-
const
|
|
306
|
+
const dirPath = this.keyToPath(dirKey);
|
|
135
307
|
|
|
308
|
+
// A common prefix IS the directory variant: it exists only as an
|
|
309
|
+
// artefact of key naming, so there is nothing to report about it
|
|
310
|
+
// beyond its being a directory.
|
|
136
311
|
yield {
|
|
137
312
|
kind: "directory",
|
|
138
|
-
name: basename(
|
|
139
|
-
path,
|
|
140
|
-
lastModified: 0,
|
|
313
|
+
name: basename(dirPath),
|
|
314
|
+
path: dirPath,
|
|
141
315
|
};
|
|
142
316
|
}
|
|
143
317
|
}
|
|
144
318
|
|
|
145
|
-
// Yield files (Contents)
|
|
146
319
|
if (response.Contents) {
|
|
147
320
|
for (const obj of response.Contents) {
|
|
148
321
|
if (!obj.Key) continue;
|
|
149
|
-
|
|
150
|
-
// Skip if it's just the prefix itself (directory marker)
|
|
151
322
|
if (obj.Key === normalizedPrefix) continue;
|
|
152
|
-
if (obj.Key
|
|
153
|
-
|
|
154
|
-
//
|
|
323
|
+
if (`${obj.Key}/` === normalizedPrefix) continue;
|
|
324
|
+
// A key ending in "/" is the zero-byte marker `mkdir` writes to make
|
|
325
|
+
// an empty directory visible. It is NOT a file with `size: 0` — it
|
|
326
|
+
// is how this store spells a directory, and it surfaces through
|
|
327
|
+
// `CommonPrefixes` above. Every other key is a file, and a genuinely
|
|
328
|
+
// empty object is the file variant with `size: 0`.
|
|
155
329
|
if (obj.Key.endsWith("/")) continue;
|
|
156
330
|
|
|
157
|
-
const
|
|
331
|
+
const filePath = this.keyToPath(obj.Key);
|
|
158
332
|
|
|
159
333
|
yield {
|
|
160
334
|
kind: "file",
|
|
161
|
-
name: basename(
|
|
162
|
-
path,
|
|
335
|
+
name: basename(filePath),
|
|
336
|
+
path: filePath,
|
|
163
337
|
size: obj.Size ?? 0,
|
|
164
338
|
lastModified: obj.LastModified?.getTime() ?? 0,
|
|
165
339
|
};
|
|
@@ -170,21 +344,12 @@ export class S3FilesApi implements IFilesApi {
|
|
|
170
344
|
} while (continuationToken);
|
|
171
345
|
}
|
|
172
346
|
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
async stats(file: FileRef): Promise<FileInfo | undefined> {
|
|
177
|
-
const normalized = resolveFileRef(file);
|
|
178
|
-
const key = this.resolveKey(file);
|
|
347
|
+
async stats(path: string): Promise<FileStats | undefined> {
|
|
348
|
+
const normalized = normalizePath(path);
|
|
349
|
+
const key = this.resolveKey(path);
|
|
179
350
|
|
|
180
|
-
// Handle root directory
|
|
181
351
|
if (normalized === "/") {
|
|
182
|
-
return {
|
|
183
|
-
kind: "directory",
|
|
184
|
-
name: "",
|
|
185
|
-
path: "/",
|
|
186
|
-
lastModified: 0,
|
|
187
|
-
};
|
|
352
|
+
return { kind: "directory" };
|
|
188
353
|
}
|
|
189
354
|
|
|
190
355
|
// Try as file first
|
|
@@ -198,11 +363,8 @@ export class S3FilesApi implements IFilesApi {
|
|
|
198
363
|
|
|
199
364
|
return {
|
|
200
365
|
kind: "file",
|
|
201
|
-
name: basename(normalized),
|
|
202
|
-
path: normalized,
|
|
203
366
|
size: response.ContentLength ?? 0,
|
|
204
367
|
lastModified: response.LastModified?.getTime() ?? 0,
|
|
205
|
-
type: response.ContentType,
|
|
206
368
|
};
|
|
207
369
|
} catch (error: unknown) {
|
|
208
370
|
const err = error as { name?: string; $metadata?: { httpStatusCode?: number } };
|
|
@@ -211,8 +373,8 @@ export class S3FilesApi implements IFilesApi {
|
|
|
211
373
|
}
|
|
212
374
|
}
|
|
213
375
|
|
|
214
|
-
// Check if it's a virtual directory
|
|
215
|
-
const prefix = key ? key
|
|
376
|
+
// Check if it's a virtual directory
|
|
377
|
+
const prefix = key ? `${key}/` : "";
|
|
216
378
|
const listCommand = new ListObjectsV2Command({
|
|
217
379
|
Bucket: this.bucket,
|
|
218
380
|
Prefix: prefix,
|
|
@@ -225,26 +387,22 @@ export class S3FilesApi implements IFilesApi {
|
|
|
225
387
|
(listResponse.Contents && listResponse.Contents.length > 0) ||
|
|
226
388
|
(listResponse.CommonPrefixes && listResponse.CommonPrefixes.length > 0)
|
|
227
389
|
) {
|
|
228
|
-
return {
|
|
229
|
-
kind: "directory",
|
|
230
|
-
name: basename(normalized),
|
|
231
|
-
path: normalized,
|
|
232
|
-
lastModified: 0,
|
|
233
|
-
};
|
|
390
|
+
return { kind: "directory" };
|
|
234
391
|
}
|
|
235
392
|
|
|
236
393
|
return undefined;
|
|
237
394
|
}
|
|
238
395
|
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
396
|
+
async exists(path: string): Promise<boolean> {
|
|
397
|
+
const stats = await this.stats(path);
|
|
398
|
+
return stats !== undefined;
|
|
399
|
+
}
|
|
400
|
+
|
|
401
|
+
async remove(path: string): Promise<boolean> {
|
|
402
|
+
const key = this.resolveKey(path);
|
|
244
403
|
|
|
245
404
|
// First try to delete as a single object
|
|
246
405
|
try {
|
|
247
|
-
// Check if object exists
|
|
248
406
|
const headCommand = new HeadObjectCommand({
|
|
249
407
|
Bucket: this.bucket,
|
|
250
408
|
Key: key,
|
|
@@ -252,7 +410,6 @@ export class S3FilesApi implements IFilesApi {
|
|
|
252
410
|
|
|
253
411
|
await this.client.send(headCommand);
|
|
254
412
|
|
|
255
|
-
// Object exists, delete it
|
|
256
413
|
const deleteCommand = new DeleteObjectCommand({
|
|
257
414
|
Bucket: this.bucket,
|
|
258
415
|
Key: key,
|
|
@@ -265,11 +422,10 @@ export class S3FilesApi implements IFilesApi {
|
|
|
265
422
|
if (err.name !== "NotFound" && err.$metadata?.httpStatusCode !== 404) {
|
|
266
423
|
throw error;
|
|
267
424
|
}
|
|
268
|
-
// Object doesn't exist, continue to check if it's a directory
|
|
269
425
|
}
|
|
270
426
|
|
|
271
427
|
// Delete all objects under this prefix (directory)
|
|
272
|
-
const prefix = key ? key
|
|
428
|
+
const prefix = key ? `${key}/` : "";
|
|
273
429
|
const keysToDelete: string[] = [];
|
|
274
430
|
|
|
275
431
|
let continuationToken: string | undefined;
|
|
@@ -298,7 +454,6 @@ export class S3FilesApi implements IFilesApi {
|
|
|
298
454
|
return false;
|
|
299
455
|
}
|
|
300
456
|
|
|
301
|
-
// Delete objects one by one (more compatible with MinIO and other S3-compatible stores)
|
|
302
457
|
for (const keyToDelete of keysToDelete) {
|
|
303
458
|
const deleteCommand = new DeleteObjectCommand({
|
|
304
459
|
Bucket: this.bucket,
|
|
@@ -311,69 +466,20 @@ export class S3FilesApi implements IFilesApi {
|
|
|
311
466
|
return true;
|
|
312
467
|
}
|
|
313
468
|
|
|
314
|
-
|
|
315
|
-
* Opens an S3 object for random access.
|
|
316
|
-
* Creates the object if it doesn't exist.
|
|
317
|
-
*/
|
|
318
|
-
async open(file: FileRef): Promise<FileHandle> {
|
|
319
|
-
const key = this.resolveKey(file);
|
|
320
|
-
|
|
321
|
-
let size = 0;
|
|
322
|
-
|
|
323
|
-
// Get current size if object exists
|
|
324
|
-
try {
|
|
325
|
-
const command = new HeadObjectCommand({
|
|
326
|
-
Bucket: this.bucket,
|
|
327
|
-
Key: key,
|
|
328
|
-
});
|
|
329
|
-
|
|
330
|
-
const response = await this.client.send(command);
|
|
331
|
-
size = response.ContentLength ?? 0;
|
|
332
|
-
} catch (error: unknown) {
|
|
333
|
-
const err = error as { name?: string; $metadata?: { httpStatusCode?: number } };
|
|
334
|
-
if (err.name !== "NotFound" && err.$metadata?.httpStatusCode !== 404) {
|
|
335
|
-
throw error;
|
|
336
|
-
}
|
|
337
|
-
// Object doesn't exist, will be created on write
|
|
338
|
-
}
|
|
339
|
-
|
|
340
|
-
return new S3FileHandle({
|
|
341
|
-
client: this.client,
|
|
342
|
-
bucket: this.bucket,
|
|
343
|
-
key,
|
|
344
|
-
size,
|
|
345
|
-
multipartPartSize: this.multipartPartSize,
|
|
346
|
-
});
|
|
347
|
-
}
|
|
348
|
-
|
|
349
|
-
/**
|
|
350
|
-
* Moves an object using Copy + Delete.
|
|
351
|
-
*/
|
|
352
|
-
async move(source: FileRef, target: FileRef): Promise<boolean> {
|
|
469
|
+
async move(source: string, target: string): Promise<boolean> {
|
|
353
470
|
const copied = await this.copy(source, target);
|
|
354
471
|
if (!copied) return false;
|
|
355
|
-
|
|
356
472
|
return this.remove(source);
|
|
357
473
|
}
|
|
358
474
|
|
|
359
|
-
|
|
360
|
-
* Copies an object or directory using S3 CopyObject.
|
|
361
|
-
*/
|
|
362
|
-
async copy(
|
|
363
|
-
source: FileRef,
|
|
364
|
-
target: FileRef,
|
|
365
|
-
options: CopyOptions = {},
|
|
366
|
-
): Promise<boolean> {
|
|
475
|
+
async copy(source: string, target: string): Promise<boolean> {
|
|
367
476
|
const sourceKey = this.resolveKey(source);
|
|
368
477
|
const targetKey = this.resolveKey(target);
|
|
369
|
-
const { recursive = true } = options;
|
|
370
478
|
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
if (!sourceInfo) return false;
|
|
479
|
+
const sourceStats = await this.stats(source);
|
|
480
|
+
if (!sourceStats) return false;
|
|
374
481
|
|
|
375
|
-
if (
|
|
376
|
-
// Copy single object
|
|
482
|
+
if (sourceStats.kind === "file") {
|
|
377
483
|
const command = new CopyObjectCommand({
|
|
378
484
|
Bucket: this.bucket,
|
|
379
485
|
Key: targetKey,
|
|
@@ -385,12 +491,8 @@ export class S3FilesApi implements IFilesApi {
|
|
|
385
491
|
}
|
|
386
492
|
|
|
387
493
|
// Copy directory (all objects under prefix)
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
}
|
|
391
|
-
|
|
392
|
-
const sourcePrefix = sourceKey + "/";
|
|
393
|
-
const targetPrefix = targetKey + "/";
|
|
494
|
+
const sourcePrefix = `${sourceKey}/`;
|
|
495
|
+
const targetPrefix = `${targetKey}/`;
|
|
394
496
|
|
|
395
497
|
let continuationToken: string | undefined;
|
|
396
498
|
let copied = false;
|
|
@@ -427,29 +529,4 @@ export class S3FilesApi implements IFilesApi {
|
|
|
427
529
|
|
|
428
530
|
return copied;
|
|
429
531
|
}
|
|
430
|
-
|
|
431
|
-
/**
|
|
432
|
-
* Creates a directory marker in S3.
|
|
433
|
-
*
|
|
434
|
-
* S3 doesn't have real directories, but some tools expect
|
|
435
|
-
* a zero-byte object with trailing slash as a directory marker.
|
|
436
|
-
* However, for our purposes we don't actually need to create markers
|
|
437
|
-
* since directories are implied by files within them.
|
|
438
|
-
*/
|
|
439
|
-
async mkdir(file: FileRef): Promise<void> {
|
|
440
|
-
// For S3, directories don't really need to be created explicitly
|
|
441
|
-
// They exist implicitly when files exist inside them
|
|
442
|
-
// But we create an empty object as a marker for compatibility
|
|
443
|
-
const key = this.resolveKey(file);
|
|
444
|
-
const dirKey = key.endsWith("/") ? key : key + "/";
|
|
445
|
-
|
|
446
|
-
const command = new PutObjectCommand({
|
|
447
|
-
Bucket: this.bucket,
|
|
448
|
-
Key: dirKey,
|
|
449
|
-
Body: new Uint8Array(0),
|
|
450
|
-
ContentLength: 0,
|
|
451
|
-
});
|
|
452
|
-
|
|
453
|
-
await this.client.send(command);
|
|
454
|
-
}
|
|
455
532
|
}
|
package/dist/s3-file-handle.d.ts
DELETED
|
@@ -1,75 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* S3 FileHandle implementation
|
|
3
|
-
*
|
|
4
|
-
* Provides random access read/write operations for S3 objects.
|
|
5
|
-
* Uses HTTP Range headers for efficient partial reads.
|
|
6
|
-
* Since S3 objects are immutable, writes require re-uploading the entire object.
|
|
7
|
-
*/
|
|
8
|
-
import type { S3Client } from "@aws-sdk/client-s3";
|
|
9
|
-
import type { AppendOptions, BinaryStream, FileHandle, ReadStreamOptions, WriteStreamOptions } from "@statewalker/webrun-files";
|
|
10
|
-
export interface S3FileHandleOptions {
|
|
11
|
-
client: S3Client;
|
|
12
|
-
bucket: string;
|
|
13
|
-
key: string;
|
|
14
|
-
size: number;
|
|
15
|
-
multipartPartSize: number;
|
|
16
|
-
}
|
|
17
|
-
export declare class S3FileHandle implements FileHandle {
|
|
18
|
-
private client;
|
|
19
|
-
private bucket;
|
|
20
|
-
private key;
|
|
21
|
-
private _size;
|
|
22
|
-
private multipartPartSize;
|
|
23
|
-
private closed;
|
|
24
|
-
constructor(options: S3FileHandleOptions);
|
|
25
|
-
get size(): number;
|
|
26
|
-
close(): Promise<void>;
|
|
27
|
-
/**
|
|
28
|
-
* Reads a range of bytes from the S3 object.
|
|
29
|
-
* Uses HTTP Range header for efficient partial reads.
|
|
30
|
-
* Streams data directly from S3 without buffering the entire response.
|
|
31
|
-
*/
|
|
32
|
-
createReadStream(options?: ReadStreamOptions): AsyncGenerator<Uint8Array>;
|
|
33
|
-
/**
|
|
34
|
-
* Writes content to the S3 object using streaming.
|
|
35
|
-
*
|
|
36
|
-
* S3 objects are immutable, so this operation replaces the entire object.
|
|
37
|
-
* Uses streaming multipart upload to avoid buffering the entire file in memory.
|
|
38
|
-
* Only buffers one part at a time (5MB by default).
|
|
39
|
-
*
|
|
40
|
-
* Note: start position > 0 uses UploadPartCopy to preserve existing content
|
|
41
|
-
* without downloading it.
|
|
42
|
-
*/
|
|
43
|
-
createWriteStream(data: BinaryStream, options?: WriteStreamOptions): Promise<number>;
|
|
44
|
-
/**
|
|
45
|
-
* Appends content to the end of the S3 object using streaming.
|
|
46
|
-
*
|
|
47
|
-
* Uses UploadPartCopy to preserve existing content without downloading it,
|
|
48
|
-
* then streams new data as additional parts.
|
|
49
|
-
*/
|
|
50
|
-
appendFile(data: BinaryStream, options?: AppendOptions): Promise<number>;
|
|
51
|
-
/**
|
|
52
|
-
* Streaming multipart upload that buffers only one part at a time.
|
|
53
|
-
* Uses UploadPartCopy to preserve existing content without downloading it
|
|
54
|
-
* for large files. For small existing content (< part size), downloads and
|
|
55
|
-
* combines with new data to meet S3's minimum part size requirement.
|
|
56
|
-
*/
|
|
57
|
-
private streamingMultipartUpload;
|
|
58
|
-
/**
|
|
59
|
-
* Download a range of bytes from the S3 object.
|
|
60
|
-
*/
|
|
61
|
-
private downloadRange;
|
|
62
|
-
/**
|
|
63
|
-
* Upload a single part and record it in the parts array.
|
|
64
|
-
*/
|
|
65
|
-
private uploadPart;
|
|
66
|
-
/**
|
|
67
|
-
* Abort a multipart upload, ignoring errors.
|
|
68
|
-
*/
|
|
69
|
-
private abortMultipartUpload;
|
|
70
|
-
/**
|
|
71
|
-
* Random access read from S3 object using HTTP Range header.
|
|
72
|
-
*/
|
|
73
|
-
read(buffer: Uint8Array, offset: number, length: number, position: number): Promise<number>;
|
|
74
|
-
}
|
|
75
|
-
//# sourceMappingURL=s3-file-handle.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"s3-file-handle.d.ts","sourceRoot":"","sources":["../src/s3-file-handle.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AASnD,OAAO,KAAK,EACV,aAAa,EACb,YAAY,EACZ,UAAU,EACV,iBAAiB,EACjB,kBAAkB,EACnB,MAAM,2BAA2B,CAAC;AAGnC,MAAM,WAAW,mBAAmB;IAClC,MAAM,EAAE,QAAQ,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,CAAC;IACb,iBAAiB,EAAE,MAAM,CAAC;CAC3B;AA+BD,qBAAa,YAAa,YAAW,UAAU;IAC7C,OAAO,CAAC,MAAM,CAAW;IACzB,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,GAAG,CAAS;IACpB,OAAO,CAAC,KAAK,CAAS;IACtB,OAAO,CAAC,iBAAiB,CAAS;IAClC,OAAO,CAAC,MAAM,CAAS;gBAEX,OAAO,EAAE,mBAAmB;IAQxC,IAAI,IAAI,IAAI,MAAM,CAEjB;IAEK,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAI5B;;;;OAIG;IACI,gBAAgB,CACrB,OAAO,GAAE,iBAAsB,GAC9B,cAAc,CAAC,UAAU,CAAC;IA6B7B;;;;;;;;;OASG;IACG,iBAAiB,CACrB,IAAI,EAAE,YAAY,EAClB,OAAO,GAAE,kBAAuB,GAC/B,OAAO,CAAC,MAAM,CAAC;IAWlB;;;;;OAKG;IACG,UAAU,CAAC,IAAI,EAAE,YAAY,EAAE,OAAO,GAAE,aAAkB,GAAG,OAAO,CAAC,MAAM,CAAC;IAgBlF;;;;;OAKG;YACW,wBAAwB;IAgLtC;;OAEG;YACW,aAAa;IAwC3B;;OAEG;YACW,UAAU;IAuBxB;;OAEG;YACW,oBAAoB;IAYlC;;OAEG;IACG,IAAI,CAAC,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;CAoDlG"}
|