@statewalker/webrun-files-s3 0.9.0 → 0.10.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +27 -21
- package/dist/cjs/index.cjs +79 -16
- package/dist/esm/index.js +80 -17
- package/dist/s3-files-api.d.ts +7 -0
- package/dist/s3-files-api.d.ts.map +1 -1
- package/package.json +9 -8
- package/src/s3-files-api.ts +106 -37
package/README.md
CHANGED
|
@@ -50,7 +50,7 @@ console.log(content); // "Hello, S3!"
|
|
|
50
50
|
|
|
51
51
|
// List directory contents
|
|
52
52
|
for await (const entry of files.list('/docs')) {
|
|
53
|
-
console.log(entry.name, entry.kind, entry.size);
|
|
53
|
+
console.log(entry.name, entry.kind, entry.kind === 'file' ? entry.size : '');
|
|
54
54
|
}
|
|
55
55
|
```
|
|
56
56
|
|
|
@@ -142,14 +142,25 @@ S3 doesn't have real directories, but this implementation simulates them using:
|
|
|
142
142
|
- **ListObjectsV2** with `Delimiter="/"` to get "subdirectories" via `CommonPrefixes`
|
|
143
143
|
- Files are returned from `Contents`
|
|
144
144
|
|
|
145
|
+
With `{ recursive: true }` the delimiter is omitted and only file entries are yielded: directories,
|
|
146
|
+
including empty ones created by `mkdir()`, do not appear in a recursive listing.
|
|
147
|
+
|
|
148
|
+
Entries come out in path order (code-point order, as every `FilesApi` lists), and `{ after }` is
|
|
149
|
+
sent to S3 as `StartAfter`, so resuming a listing does not re-list what came before. Keys list in
|
|
150
|
+
UTF-8 byte order, which is already that order for files; but a directory `a` arrives as the common
|
|
151
|
+
prefix `a/` *after* keys such as `a-x` and `a.txt` that must follow it. A non-recursive listing
|
|
152
|
+
therefore holds back the few entries a directory still to come could precede, across page
|
|
153
|
+
boundaries, and releases them in order. When a key `a` and keys under `a/` both exist, the path
|
|
154
|
+
`/a` is listed once, as the file — the same answer `stats()` gives.
|
|
155
|
+
|
|
145
156
|
Because `FileStats` is a discriminated union, this implementation has to commit
|
|
146
157
|
to two things a store without real directories could otherwise leave vague:
|
|
147
158
|
|
|
148
159
|
- A **common prefix is the directory variant** - `{ kind: "directory" }` and
|
|
149
160
|
nothing more. There is no size or modification time to report for something
|
|
150
161
|
that exists only as an artefact of key naming.
|
|
151
|
-
-
|
|
152
|
-
marker `mkdir()` writes so an empty directory is visible
|
|
162
|
+
- **Any key ending in `/` is a directory, not a file** — such as the zero-byte
|
|
163
|
+
marker `mkdir()` writes so an empty directory is visible — and it is skipped
|
|
153
164
|
when reading `Contents`. Every other key is a file, and a genuinely empty
|
|
154
165
|
object is the file variant with `size: 0`.
|
|
155
166
|
|
|
@@ -221,31 +232,26 @@ This implementation works with any S3-compatible storage:
|
|
|
221
232
|
|
|
222
233
|
## Testing
|
|
223
234
|
|
|
224
|
-
|
|
235
|
+
The package has integration tests only. They need Docker and run against a real S3-compatible
|
|
236
|
+
server, [RustFS](https://github.com/rustfs/rustfs), started through
|
|
237
|
+
[testcontainers](https://github.com/testcontainers/testcontainers-node) as a `GenericContainer` of
|
|
238
|
+
`rustfs/rustfs:latest` (override the image with `RUSTFS_IMAGE`). They run the shared
|
|
239
|
+
`createFilesApiTests` and `createBigFilesApiTests` suites from `@statewalker/webrun-files-tests`.
|
|
225
240
|
|
|
226
|
-
```
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
241
|
+
```bash
|
|
242
|
+
pnpm test # unit profile: excludes *.integration.test.ts, so no Docker needed (runs nothing today)
|
|
243
|
+
pnpm test:integration # Docker: RustFS container, all suites including the 256 MiB big-file suite
|
|
244
|
+
```
|
|
230
245
|
|
|
231
|
-
|
|
246
|
+
The client the tests build is the one to use against any S3-compatible server on a custom endpoint:
|
|
232
247
|
|
|
248
|
+
```typescript
|
|
233
249
|
const s3Client = new S3Client({
|
|
234
|
-
endpoint:
|
|
250
|
+
endpoint: `http://${container.getHost()}:${container.getMappedPort(9000)}`,
|
|
235
251
|
region: 'us-east-1',
|
|
236
|
-
credentials: {
|
|
237
|
-
accessKeyId: minioContainer.getUsername(),
|
|
238
|
-
secretAccessKey: minioContainer.getPassword(),
|
|
239
|
-
},
|
|
252
|
+
credentials: { accessKeyId: 'rustfsadmin', secretAccessKey: 'rustfsadmin' },
|
|
240
253
|
forcePathStyle: true,
|
|
241
254
|
});
|
|
242
|
-
|
|
243
|
-
await s3Client.send(new CreateBucketCommand({ Bucket: 'test-bucket' }));
|
|
244
|
-
|
|
245
|
-
const files = new S3FilesApi({
|
|
246
|
-
client: s3Client,
|
|
247
|
-
bucket: 'test-bucket',
|
|
248
|
-
});
|
|
249
255
|
```
|
|
250
256
|
|
|
251
257
|
## License
|
package/dist/cjs/index.cjs
CHANGED
|
@@ -154,10 +154,32 @@ var S3FilesApi = class {
|
|
|
154
154
|
});
|
|
155
155
|
await this.client.send(command);
|
|
156
156
|
}
|
|
157
|
+
/**
|
|
158
|
+
* Keys list in UTF-8 byte order, which is `comparePaths` order for files.
|
|
159
|
+
* A recursive listing is therefore already ordered. A non-recursive one is
|
|
160
|
+
* not quite: the directory `a` arrives as the common prefix `a/`, after keys
|
|
161
|
+
* such as `a-x` and `a.txt` that must follow it, so entries pass through a
|
|
162
|
+
* small reorder buffer (see {@link releaseBound}).
|
|
163
|
+
*/
|
|
157
164
|
async *list(path, options) {
|
|
158
165
|
const recursive = options?.recursive ?? false;
|
|
166
|
+
const after = options?.after;
|
|
167
|
+
const dirPath = (0, _statewalker_webrun_files.normalizePath)(path);
|
|
159
168
|
const prefix = this.resolveKey(path);
|
|
160
169
|
const normalizedPrefix = prefix ? `${prefix}/` : this.prefix ? `${this.prefix}/` : "";
|
|
170
|
+
const dirPrefix = dirPath === "/" ? "" : dirPath;
|
|
171
|
+
let startAfter;
|
|
172
|
+
if (after !== void 0) {
|
|
173
|
+
if (after.startsWith(`${dirPrefix}/`)) startAfter = normalizedPrefix + after.slice(dirPrefix.length + 1);
|
|
174
|
+
else if ((0, _statewalker_webrun_files.comparePaths)(after, `${dirPrefix}/`) >= 0) return;
|
|
175
|
+
}
|
|
176
|
+
const pending = [];
|
|
177
|
+
let released;
|
|
178
|
+
const release = (entry) => {
|
|
179
|
+
if (entry.path === released) return false;
|
|
180
|
+
released = entry.path;
|
|
181
|
+
return after === void 0 || (0, _statewalker_webrun_files.comparePaths)(entry.path, after) > 0;
|
|
182
|
+
};
|
|
161
183
|
let continuationToken;
|
|
162
184
|
do {
|
|
163
185
|
const command = new _aws_sdk_client_s3.ListObjectsV2Command({
|
|
@@ -165,35 +187,60 @@ var S3FilesApi = class {
|
|
|
165
187
|
Prefix: normalizedPrefix,
|
|
166
188
|
Delimiter: recursive ? void 0 : "/",
|
|
167
189
|
ContinuationToken: continuationToken,
|
|
190
|
+
StartAfter: continuationToken ? void 0 : startAfter,
|
|
168
191
|
MaxKeys: 1e3
|
|
169
192
|
});
|
|
170
193
|
const response = await this.client.send(command);
|
|
171
|
-
|
|
194
|
+
const page = [];
|
|
195
|
+
for (const cpPrefix of recursive ? [] : response.CommonPrefixes ?? []) {
|
|
172
196
|
if (!cpPrefix.Prefix) continue;
|
|
173
197
|
const dirKey = cpPrefix.Prefix.endsWith("/") ? cpPrefix.Prefix.slice(0, -1) : cpPrefix.Prefix;
|
|
174
|
-
const
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
198
|
+
const childPath = this.keyToPath(dirKey);
|
|
199
|
+
page.push({
|
|
200
|
+
key: cpPrefix.Prefix,
|
|
201
|
+
entry: {
|
|
202
|
+
kind: "directory",
|
|
203
|
+
name: (0, _statewalker_webrun_files.basename)(childPath),
|
|
204
|
+
path: childPath
|
|
205
|
+
}
|
|
206
|
+
});
|
|
180
207
|
}
|
|
181
|
-
|
|
208
|
+
for (const obj of response.Contents ?? []) {
|
|
182
209
|
if (!obj.Key) continue;
|
|
183
210
|
if (obj.Key === normalizedPrefix) continue;
|
|
184
211
|
if (`${obj.Key}/` === normalizedPrefix) continue;
|
|
185
|
-
if (obj.Key.endsWith("/"))
|
|
212
|
+
if (obj.Key.endsWith("/")) {
|
|
213
|
+
page.push({ key: obj.Key });
|
|
214
|
+
continue;
|
|
215
|
+
}
|
|
186
216
|
const filePath = this.keyToPath(obj.Key);
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
217
|
+
page.push({
|
|
218
|
+
key: obj.Key,
|
|
219
|
+
entry: {
|
|
220
|
+
kind: "file",
|
|
221
|
+
name: (0, _statewalker_webrun_files.basename)(filePath),
|
|
222
|
+
path: filePath,
|
|
223
|
+
size: obj.Size ?? 0,
|
|
224
|
+
lastModified: obj.LastModified?.getTime() ?? 0
|
|
225
|
+
}
|
|
226
|
+
});
|
|
227
|
+
}
|
|
228
|
+
page.sort((x, y) => (0, _statewalker_webrun_files.comparePaths)(x.key, y.key));
|
|
229
|
+
for (const { key, entry } of page) {
|
|
230
|
+
if (recursive) {
|
|
231
|
+
if (entry && release(entry)) yield entry;
|
|
232
|
+
continue;
|
|
233
|
+
}
|
|
234
|
+
if (entry) insertInOrder(pending, entry);
|
|
235
|
+
const bound = releaseBound(this.keyToPath(key));
|
|
236
|
+
while (pending.length > 0 && (0, _statewalker_webrun_files.comparePaths)(pending[0].path, bound) < 0) {
|
|
237
|
+
const next = pending.shift();
|
|
238
|
+
if (release(next)) yield next;
|
|
239
|
+
}
|
|
194
240
|
}
|
|
195
241
|
continuationToken = response.NextContinuationToken;
|
|
196
242
|
} while (continuationToken);
|
|
243
|
+
for (const entry of pending) if (release(entry)) yield entry;
|
|
197
244
|
}
|
|
198
245
|
async stats(path) {
|
|
199
246
|
const normalized = (0, _statewalker_webrun_files.normalizePath)(path);
|
|
@@ -314,5 +361,21 @@ var S3FilesApi = class {
|
|
|
314
361
|
return copied;
|
|
315
362
|
}
|
|
316
363
|
};
|
|
364
|
+
/**
|
|
365
|
+
* Once a delimited listing has reached key position `position` (as a path), a
|
|
366
|
+
* directory still to come — which arrives as `D + "/"` at or after that
|
|
367
|
+
* position — has a path of at least this bound: the shortest prefix of
|
|
368
|
+
* `position` followed by a character that sorts before `/`, or `position`
|
|
369
|
+
* itself if there is none. Buffered entries below the bound are final.
|
|
370
|
+
*/
|
|
371
|
+
function releaseBound(position) {
|
|
372
|
+
for (let i = 0; i < position.length; i++) if (position.charCodeAt(i) < 47) return position.slice(0, i);
|
|
373
|
+
return position;
|
|
374
|
+
}
|
|
375
|
+
function insertInOrder(entries, entry) {
|
|
376
|
+
let i = entries.length;
|
|
377
|
+
while (i > 0 && (0, _statewalker_webrun_files.comparePaths)(entries[i - 1].path, entry.path) > 0) i--;
|
|
378
|
+
entries.splice(i, 0, entry);
|
|
379
|
+
}
|
|
317
380
|
//#endregion
|
|
318
381
|
exports.S3FilesApi = S3FilesApi;
|
package/dist/esm/index.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { AbortMultipartUploadCommand, CompleteMultipartUploadCommand, CopyObjectCommand, CreateMultipartUploadCommand, DeleteObjectCommand, GetObjectCommand, HeadObjectCommand, ListObjectsV2Command, PutObjectCommand, UploadPartCommand } from "@aws-sdk/client-s3";
|
|
2
|
-
import { basename, normalizePath } from "@statewalker/webrun-files";
|
|
2
|
+
import { basename, comparePaths, normalizePath } from "@statewalker/webrun-files";
|
|
3
3
|
//#region src/s3-files-api.ts
|
|
4
4
|
/** Default part size for multipart uploads: 5MB (S3 minimum) */
|
|
5
5
|
const DEFAULT_PART_SIZE = 5242880;
|
|
@@ -153,10 +153,32 @@ var S3FilesApi = class {
|
|
|
153
153
|
});
|
|
154
154
|
await this.client.send(command);
|
|
155
155
|
}
|
|
156
|
+
/**
|
|
157
|
+
* Keys list in UTF-8 byte order, which is `comparePaths` order for files.
|
|
158
|
+
* A recursive listing is therefore already ordered. A non-recursive one is
|
|
159
|
+
* not quite: the directory `a` arrives as the common prefix `a/`, after keys
|
|
160
|
+
* such as `a-x` and `a.txt` that must follow it, so entries pass through a
|
|
161
|
+
* small reorder buffer (see {@link releaseBound}).
|
|
162
|
+
*/
|
|
156
163
|
async *list(path, options) {
|
|
157
164
|
const recursive = options?.recursive ?? false;
|
|
165
|
+
const after = options?.after;
|
|
166
|
+
const dirPath = normalizePath(path);
|
|
158
167
|
const prefix = this.resolveKey(path);
|
|
159
168
|
const normalizedPrefix = prefix ? `${prefix}/` : this.prefix ? `${this.prefix}/` : "";
|
|
169
|
+
const dirPrefix = dirPath === "/" ? "" : dirPath;
|
|
170
|
+
let startAfter;
|
|
171
|
+
if (after !== void 0) {
|
|
172
|
+
if (after.startsWith(`${dirPrefix}/`)) startAfter = normalizedPrefix + after.slice(dirPrefix.length + 1);
|
|
173
|
+
else if (comparePaths(after, `${dirPrefix}/`) >= 0) return;
|
|
174
|
+
}
|
|
175
|
+
const pending = [];
|
|
176
|
+
let released;
|
|
177
|
+
const release = (entry) => {
|
|
178
|
+
if (entry.path === released) return false;
|
|
179
|
+
released = entry.path;
|
|
180
|
+
return after === void 0 || comparePaths(entry.path, after) > 0;
|
|
181
|
+
};
|
|
160
182
|
let continuationToken;
|
|
161
183
|
do {
|
|
162
184
|
const command = new ListObjectsV2Command({
|
|
@@ -164,35 +186,60 @@ var S3FilesApi = class {
|
|
|
164
186
|
Prefix: normalizedPrefix,
|
|
165
187
|
Delimiter: recursive ? void 0 : "/",
|
|
166
188
|
ContinuationToken: continuationToken,
|
|
189
|
+
StartAfter: continuationToken ? void 0 : startAfter,
|
|
167
190
|
MaxKeys: 1e3
|
|
168
191
|
});
|
|
169
192
|
const response = await this.client.send(command);
|
|
170
|
-
|
|
193
|
+
const page = [];
|
|
194
|
+
for (const cpPrefix of recursive ? [] : response.CommonPrefixes ?? []) {
|
|
171
195
|
if (!cpPrefix.Prefix) continue;
|
|
172
196
|
const dirKey = cpPrefix.Prefix.endsWith("/") ? cpPrefix.Prefix.slice(0, -1) : cpPrefix.Prefix;
|
|
173
|
-
const
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
197
|
+
const childPath = this.keyToPath(dirKey);
|
|
198
|
+
page.push({
|
|
199
|
+
key: cpPrefix.Prefix,
|
|
200
|
+
entry: {
|
|
201
|
+
kind: "directory",
|
|
202
|
+
name: basename(childPath),
|
|
203
|
+
path: childPath
|
|
204
|
+
}
|
|
205
|
+
});
|
|
179
206
|
}
|
|
180
|
-
|
|
207
|
+
for (const obj of response.Contents ?? []) {
|
|
181
208
|
if (!obj.Key) continue;
|
|
182
209
|
if (obj.Key === normalizedPrefix) continue;
|
|
183
210
|
if (`${obj.Key}/` === normalizedPrefix) continue;
|
|
184
|
-
if (obj.Key.endsWith("/"))
|
|
211
|
+
if (obj.Key.endsWith("/")) {
|
|
212
|
+
page.push({ key: obj.Key });
|
|
213
|
+
continue;
|
|
214
|
+
}
|
|
185
215
|
const filePath = this.keyToPath(obj.Key);
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
216
|
+
page.push({
|
|
217
|
+
key: obj.Key,
|
|
218
|
+
entry: {
|
|
219
|
+
kind: "file",
|
|
220
|
+
name: basename(filePath),
|
|
221
|
+
path: filePath,
|
|
222
|
+
size: obj.Size ?? 0,
|
|
223
|
+
lastModified: obj.LastModified?.getTime() ?? 0
|
|
224
|
+
}
|
|
225
|
+
});
|
|
226
|
+
}
|
|
227
|
+
page.sort((x, y) => comparePaths(x.key, y.key));
|
|
228
|
+
for (const { key, entry } of page) {
|
|
229
|
+
if (recursive) {
|
|
230
|
+
if (entry && release(entry)) yield entry;
|
|
231
|
+
continue;
|
|
232
|
+
}
|
|
233
|
+
if (entry) insertInOrder(pending, entry);
|
|
234
|
+
const bound = releaseBound(this.keyToPath(key));
|
|
235
|
+
while (pending.length > 0 && comparePaths(pending[0].path, bound) < 0) {
|
|
236
|
+
const next = pending.shift();
|
|
237
|
+
if (release(next)) yield next;
|
|
238
|
+
}
|
|
193
239
|
}
|
|
194
240
|
continuationToken = response.NextContinuationToken;
|
|
195
241
|
} while (continuationToken);
|
|
242
|
+
for (const entry of pending) if (release(entry)) yield entry;
|
|
196
243
|
}
|
|
197
244
|
async stats(path) {
|
|
198
245
|
const normalized = normalizePath(path);
|
|
@@ -313,5 +360,21 @@ var S3FilesApi = class {
|
|
|
313
360
|
return copied;
|
|
314
361
|
}
|
|
315
362
|
};
|
|
363
|
+
/**
|
|
364
|
+
* Once a delimited listing has reached key position `position` (as a path), a
|
|
365
|
+
* directory still to come — which arrives as `D + "/"` at or after that
|
|
366
|
+
* position — has a path of at least this bound: the shortest prefix of
|
|
367
|
+
* `position` followed by a character that sorts before `/`, or `position`
|
|
368
|
+
* itself if there is none. Buffered entries below the bound are final.
|
|
369
|
+
*/
|
|
370
|
+
function releaseBound(position) {
|
|
371
|
+
for (let i = 0; i < position.length; i++) if (position.charCodeAt(i) < 47) return position.slice(0, i);
|
|
372
|
+
return position;
|
|
373
|
+
}
|
|
374
|
+
function insertInOrder(entries, entry) {
|
|
375
|
+
let i = entries.length;
|
|
376
|
+
while (i > 0 && comparePaths(entries[i - 1].path, entry.path) > 0) i--;
|
|
377
|
+
entries.splice(i, 0, entry);
|
|
378
|
+
}
|
|
316
379
|
//#endregion
|
|
317
380
|
export { S3FilesApi };
|
package/dist/s3-files-api.d.ts
CHANGED
|
@@ -43,6 +43,13 @@ export declare class S3FilesApi implements FilesApi {
|
|
|
43
43
|
read(path: string, options?: ReadOptions): AsyncIterable<Uint8Array>;
|
|
44
44
|
write(path: string, content: Iterable<Uint8Array> | AsyncIterable<Uint8Array>): Promise<void>;
|
|
45
45
|
mkdir(path: string): Promise<void>;
|
|
46
|
+
/**
|
|
47
|
+
* Keys list in UTF-8 byte order, which is `comparePaths` order for files.
|
|
48
|
+
* A recursive listing is therefore already ordered. A non-recursive one is
|
|
49
|
+
* not quite: the directory `a` arrives as the common prefix `a/`, after keys
|
|
50
|
+
* such as `a-x` and `a.txt` that must follow it, so entries pass through a
|
|
51
|
+
* small reorder buffer (see {@link releaseBound}).
|
|
52
|
+
*/
|
|
46
53
|
list(path: string, options?: ListOptions): AsyncIterable<FileInfo>;
|
|
47
54
|
stats(path: string): Promise<FileStats | undefined>;
|
|
48
55
|
exists(path: string): Promise<boolean>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"s3-files-api.d.ts","sourceRoot":"","sources":["../src/s3-files-api.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAanD,OAAO,KAAK,EACV,QAAQ,EACR,SAAS,EACT,QAAQ,EACR,WAAW,EACX,WAAW,EACZ,MAAM,2BAA2B,CAAC;AASnC,MAAM,WAAW,iBAAiB;IAChC;;OAEG;IACH,MAAM,EAAE,QAAQ,CAAC;IAEjB;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IAEf;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB;;;;OAIG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED;;;;;GAKG;AACH,qBAAa,UAAW,YAAW,QAAQ;IACzC,OAAO,CAAC,MAAM,CAAW;IACzB,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,QAAQ,CAAS;IAEzB,YAAY,OAAO,EAAE,iBAAiB,EAKrC;IAED,OAAO,CAAC,UAAU;IAUlB,OAAO,CAAC,SAAS;IAWV,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,WAAW,GAAG,aAAa,CAAC,UAAU,CAAC,CA6C1E;IAEK,KAAK,CACT,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,QAAQ,CAAC,UAAU,CAAC,GAAG,aAAa,CAAC,UAAU,CAAC,GACxD,OAAO,CAAC,IAAI,CAAC,CAkHf;IAEK,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAYvC;
|
|
1
|
+
{"version":3,"file":"s3-files-api.d.ts","sourceRoot":"","sources":["../src/s3-files-api.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAanD,OAAO,KAAK,EACV,QAAQ,EACR,SAAS,EACT,QAAQ,EACR,WAAW,EACX,WAAW,EACZ,MAAM,2BAA2B,CAAC;AASnC,MAAM,WAAW,iBAAiB;IAChC;;OAEG;IACH,MAAM,EAAE,QAAQ,CAAC;IAEjB;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IAEf;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB;;;;OAIG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED;;;;;GAKG;AACH,qBAAa,UAAW,YAAW,QAAQ;IACzC,OAAO,CAAC,MAAM,CAAW;IACzB,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,QAAQ,CAAS;IAEzB,YAAY,OAAO,EAAE,iBAAiB,EAKrC;IAED,OAAO,CAAC,UAAU;IAUlB,OAAO,CAAC,SAAS;IAWV,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,WAAW,GAAG,aAAa,CAAC,UAAU,CAAC,CA6C1E;IAEK,KAAK,CACT,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,QAAQ,CAAC,UAAU,CAAC,GAAG,aAAa,CAAC,UAAU,CAAC,GACxD,OAAO,CAAC,IAAI,CAAC,CAkHf;IAEK,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAYvC;IAED;;;;;;OAMG;IACI,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,WAAW,GAAG,aAAa,CAAC,QAAQ,CAAC,CA0GxE;IAEK,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,GAAG,SAAS,CAAC,CA+CxD;IAEK,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAG3C;IAEK,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAkE3C;IAEK,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAI3D;IAEK,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAwD3D;CACF"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@statewalker/webrun-files-s3",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.10.2",
|
|
4
4
|
"private": false,
|
|
5
5
|
"type": "module",
|
|
6
6
|
"description": "S3-backed FilesApi implementation for webrun-files",
|
|
@@ -19,6 +19,7 @@
|
|
|
19
19
|
"types": "./dist/index.d.ts",
|
|
20
20
|
"exports": {
|
|
21
21
|
".": {
|
|
22
|
+
"source": "./src/index.ts",
|
|
22
23
|
"types": "./dist/index.d.ts",
|
|
23
24
|
"import": "./dist/esm/index.js",
|
|
24
25
|
"require": "./dist/cjs/index.cjs"
|
|
@@ -29,20 +30,20 @@
|
|
|
29
30
|
"src"
|
|
30
31
|
],
|
|
31
32
|
"dependencies": {
|
|
32
|
-
"@statewalker/webrun-files": "0.
|
|
33
|
+
"@statewalker/webrun-files": "^0.10.1"
|
|
33
34
|
},
|
|
34
35
|
"peerDependencies": {
|
|
35
36
|
"@aws-sdk/client-s3": "^3.1032.0"
|
|
36
37
|
},
|
|
37
38
|
"devDependencies": {
|
|
38
|
-
"@aws-sdk/client-s3": "^3.
|
|
39
|
-
"@types/node": "^26.
|
|
39
|
+
"@aws-sdk/client-s3": "^3.1146.0",
|
|
40
|
+
"@types/node": "^26.6.4",
|
|
40
41
|
"rimraf": "^6.1.3",
|
|
41
|
-
"rolldown": "^1.2.
|
|
42
|
-
"testcontainers": "^12.
|
|
42
|
+
"rolldown": "^1.2.12",
|
|
43
|
+
"testcontainers": "^12.2.0",
|
|
43
44
|
"typescript": "^7.0.2",
|
|
44
|
-
"vitest": "^
|
|
45
|
-
"@statewalker/webrun-files-tests": "
|
|
45
|
+
"vitest": "^5.0.3",
|
|
46
|
+
"@statewalker/webrun-files-tests": "^3.0.0"
|
|
46
47
|
},
|
|
47
48
|
"sideEffects": false,
|
|
48
49
|
"publishConfig": {
|
package/src/s3-files-api.ts
CHANGED
|
@@ -24,7 +24,7 @@ import type {
|
|
|
24
24
|
ListOptions,
|
|
25
25
|
ReadOptions,
|
|
26
26
|
} from "@statewalker/webrun-files";
|
|
27
|
-
import { basename, normalizePath } from "@statewalker/webrun-files";
|
|
27
|
+
import { basename, comparePaths, normalizePath } from "@statewalker/webrun-files";
|
|
28
28
|
|
|
29
29
|
/** Default part size for multipart uploads: 5MB (S3 minimum) */
|
|
30
30
|
const DEFAULT_PART_SIZE = 5 * 1024 * 1024;
|
|
@@ -277,11 +277,42 @@ export class S3FilesApi implements FilesApi {
|
|
|
277
277
|
await this.client.send(command);
|
|
278
278
|
}
|
|
279
279
|
|
|
280
|
+
/**
|
|
281
|
+
* Keys list in UTF-8 byte order, which is `comparePaths` order for files.
|
|
282
|
+
* A recursive listing is therefore already ordered. A non-recursive one is
|
|
283
|
+
* not quite: the directory `a` arrives as the common prefix `a/`, after keys
|
|
284
|
+
* such as `a-x` and `a.txt` that must follow it, so entries pass through a
|
|
285
|
+
* small reorder buffer (see {@link releaseBound}).
|
|
286
|
+
*/
|
|
280
287
|
async *list(path: string, options?: ListOptions): AsyncIterable<FileInfo> {
|
|
281
288
|
const recursive = options?.recursive ?? false;
|
|
289
|
+
const after = options?.after;
|
|
290
|
+
const dirPath = normalizePath(path);
|
|
282
291
|
const prefix = this.resolveKey(path);
|
|
283
292
|
const normalizedPrefix = prefix ? `${prefix}/` : this.prefix ? `${this.prefix}/` : "";
|
|
284
293
|
|
|
294
|
+
// `after` as a key, by raw prefix replacement: normalising the path could
|
|
295
|
+
// move the cursor past entries that sort after it.
|
|
296
|
+
const dirPrefix = dirPath === "/" ? "" : dirPath;
|
|
297
|
+
let startAfter: string | undefined;
|
|
298
|
+
if (after !== undefined) {
|
|
299
|
+
if (after.startsWith(`${dirPrefix}/`)) {
|
|
300
|
+
startAfter = normalizedPrefix + after.slice(dirPrefix.length + 1);
|
|
301
|
+
} else if (comparePaths(after, `${dirPrefix}/`) >= 0) {
|
|
302
|
+
return; // the whole directory lies at or before the cursor
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
const pending: FileInfo[] = [];
|
|
307
|
+
// A key `a` and keys under `a/` make a file and a directory with the same
|
|
308
|
+
// path. A listing is strictly increasing, so the first released — the file,
|
|
309
|
+
// as `stats` also reports — is kept.
|
|
310
|
+
let released: string | undefined;
|
|
311
|
+
const release = (entry: FileInfo): boolean => {
|
|
312
|
+
if (entry.path === released) return false;
|
|
313
|
+
released = entry.path;
|
|
314
|
+
return after === undefined || comparePaths(entry.path, after) > 0;
|
|
315
|
+
};
|
|
285
316
|
let continuationToken: string | undefined;
|
|
286
317
|
|
|
287
318
|
do {
|
|
@@ -290,58 +321,75 @@ export class S3FilesApi implements FilesApi {
|
|
|
290
321
|
Prefix: normalizedPrefix,
|
|
291
322
|
Delimiter: recursive ? undefined : "/",
|
|
292
323
|
ContinuationToken: continuationToken,
|
|
324
|
+
StartAfter: continuationToken ? undefined : startAfter,
|
|
293
325
|
MaxKeys: 1000,
|
|
294
326
|
});
|
|
295
327
|
|
|
296
328
|
const response = await this.client.send(command);
|
|
297
329
|
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
name: basename(dirPath),
|
|
314
|
-
path: dirPath,
|
|
315
|
-
};
|
|
316
|
-
}
|
|
330
|
+
// Common prefixes and contents each come sorted; interleave them by key.
|
|
331
|
+
const page: { key: string; entry?: FileInfo }[] = [];
|
|
332
|
+
for (const cpPrefix of recursive ? [] : (response.CommonPrefixes ?? [])) {
|
|
333
|
+
if (!cpPrefix.Prefix) continue;
|
|
334
|
+
const dirKey = cpPrefix.Prefix.endsWith("/")
|
|
335
|
+
? cpPrefix.Prefix.slice(0, -1)
|
|
336
|
+
: cpPrefix.Prefix;
|
|
337
|
+
const childPath = this.keyToPath(dirKey);
|
|
338
|
+
// A common prefix IS the directory variant: it exists only as an
|
|
339
|
+
// artefact of key naming, so there is nothing to report about it
|
|
340
|
+
// beyond its being a directory.
|
|
341
|
+
page.push({
|
|
342
|
+
key: cpPrefix.Prefix,
|
|
343
|
+
entry: { kind: "directory", name: basename(childPath), path: childPath },
|
|
344
|
+
});
|
|
317
345
|
}
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
346
|
+
for (const obj of response.Contents ?? []) {
|
|
347
|
+
if (!obj.Key) continue;
|
|
348
|
+
if (obj.Key === normalizedPrefix) continue;
|
|
349
|
+
if (`${obj.Key}/` === normalizedPrefix) continue;
|
|
350
|
+
// A key ending in "/" is the zero-byte marker `mkdir` writes to make
|
|
351
|
+
// an empty directory visible. It is NOT a file with `size: 0` — it
|
|
352
|
+
// is how this store spells a directory, and it surfaces through
|
|
353
|
+
// `CommonPrefixes` above. Every other key is a file, and a genuinely
|
|
354
|
+
// empty object is the file variant with `size: 0`. It still moves
|
|
355
|
+
// the listing's position forward.
|
|
356
|
+
if (obj.Key.endsWith("/")) {
|
|
357
|
+
page.push({ key: obj.Key });
|
|
358
|
+
continue;
|
|
359
|
+
}
|
|
360
|
+
const filePath = this.keyToPath(obj.Key);
|
|
361
|
+
page.push({
|
|
362
|
+
key: obj.Key,
|
|
363
|
+
entry: {
|
|
334
364
|
kind: "file",
|
|
335
365
|
name: basename(filePath),
|
|
336
366
|
path: filePath,
|
|
337
367
|
size: obj.Size ?? 0,
|
|
338
368
|
lastModified: obj.LastModified?.getTime() ?? 0,
|
|
339
|
-
}
|
|
369
|
+
},
|
|
370
|
+
});
|
|
371
|
+
}
|
|
372
|
+
page.sort((x, y) => comparePaths(x.key, y.key));
|
|
373
|
+
|
|
374
|
+
for (const { key, entry } of page) {
|
|
375
|
+
if (recursive) {
|
|
376
|
+
if (entry && release(entry)) yield entry;
|
|
377
|
+
continue;
|
|
378
|
+
}
|
|
379
|
+
if (entry) insertInOrder(pending, entry);
|
|
380
|
+
const bound = releaseBound(this.keyToPath(key));
|
|
381
|
+
while (pending.length > 0 && comparePaths(pending[0].path, bound) < 0) {
|
|
382
|
+
const next = pending.shift() as FileInfo;
|
|
383
|
+
if (release(next)) yield next;
|
|
340
384
|
}
|
|
341
385
|
}
|
|
342
386
|
|
|
343
387
|
continuationToken = response.NextContinuationToken;
|
|
344
388
|
} while (continuationToken);
|
|
389
|
+
|
|
390
|
+
for (const entry of pending) {
|
|
391
|
+
if (release(entry)) yield entry;
|
|
392
|
+
}
|
|
345
393
|
}
|
|
346
394
|
|
|
347
395
|
async stats(path: string): Promise<FileStats | undefined> {
|
|
@@ -530,3 +578,24 @@ export class S3FilesApi implements FilesApi {
|
|
|
530
578
|
return copied;
|
|
531
579
|
}
|
|
532
580
|
}
|
|
581
|
+
|
|
582
|
+
/**
|
|
583
|
+
* Once a delimited listing has reached key position `position` (as a path), a
|
|
584
|
+
* directory still to come — which arrives as `D + "/"` at or after that
|
|
585
|
+
* position — has a path of at least this bound: the shortest prefix of
|
|
586
|
+
* `position` followed by a character that sorts before `/`, or `position`
|
|
587
|
+
* itself if there is none. Buffered entries below the bound are final.
|
|
588
|
+
*/
|
|
589
|
+
function releaseBound(position: string): string {
|
|
590
|
+
for (let i = 0; i < position.length; i++) {
|
|
591
|
+
if (position.charCodeAt(i) < 0x2f) return position.slice(0, i);
|
|
592
|
+
}
|
|
593
|
+
return position;
|
|
594
|
+
}
|
|
595
|
+
|
|
596
|
+
function insertInOrder(entries: FileInfo[], entry: FileInfo): void {
|
|
597
|
+
let i = entries.length;
|
|
598
|
+
// Equal paths keep arrival order: a file arrives before its same-named prefix.
|
|
599
|
+
while (i > 0 && comparePaths(entries[i - 1].path, entry.path) > 0) i--;
|
|
600
|
+
entries.splice(i, 0, entry);
|
|
601
|
+
}
|