@statewalker/webrun-files-s3 0.9.0 → 0.10.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -50,7 +50,7 @@ console.log(content); // "Hello, S3!"
50
50
 
51
51
  // List directory contents
52
52
  for await (const entry of files.list('/docs')) {
53
- console.log(entry.name, entry.kind, entry.size);
53
+ console.log(entry.name, entry.kind, entry.kind === 'file' ? entry.size : '');
54
54
  }
55
55
  ```
56
56
 
@@ -142,14 +142,25 @@ S3 doesn't have real directories, but this implementation simulates them using:
142
142
  - **ListObjectsV2** with `Delimiter="/"` to get "subdirectories" via `CommonPrefixes`
143
143
  - Files are returned from `Contents`
144
144
 
145
+ With `{ recursive: true }` the delimiter is omitted and only file entries are yielded: directories,
146
+ including empty ones created by `mkdir()`, do not appear in a recursive listing.
147
+
148
+ Entries come out in path order (code-point order, as every `FilesApi` lists), and `{ after }` is
149
+ sent to S3 as `StartAfter`, so resuming a listing does not re-list what came before. Keys list in
150
+ UTF-8 byte order, which is already that order for files; but a directory `a` arrives as the common
151
+ prefix `a/` *after* keys such as `a-x` and `a.txt` that must follow it. A non-recursive listing
152
+ therefore holds back the few entries a directory still to come could precede, across page
153
+ boundaries, and releases them in order. When a key `a` and keys under `a/` both exist, the path
154
+ `/a` is listed once, as the file — the same answer `stats()` gives.
155
+
145
156
  Because `FileStats` is a discriminated union, this implementation has to commit
146
157
  to two things a store without real directories could otherwise leave vague:
147
158
 
148
159
  - A **common prefix is the directory variant** - `{ kind: "directory" }` and
149
160
  nothing more. There is no size or modification time to report for something
150
161
  that exists only as an artefact of key naming.
151
- - A **zero-byte key ending in `/` is a directory, not a file**. That is the
152
- marker `mkdir()` writes so an empty directory is visible, and it is skipped
162
+ - **Any key ending in `/` is a directory, not a file** — such as the zero-byte
163
+ marker `mkdir()` writes so an empty directory is visible — and it is skipped
153
164
  when reading `Contents`. Every other key is a file, and a genuinely empty
154
165
  object is the file variant with `size: 0`.
155
166
 
@@ -221,31 +232,26 @@ This implementation works with any S3-compatible storage:
221
232
 
222
233
  ## Testing
223
234
 
224
- Tests use [testcontainers](https://github.com/testcontainers/testcontainers-node) with MinIO:
235
+ The package has integration tests only. They need Docker and run against a real S3-compatible
236
+ server, [RustFS](https://github.com/rustfs/rustfs), started through
237
+ [testcontainers](https://github.com/testcontainers/testcontainers-node) as a `GenericContainer` of
238
+ `rustfs/rustfs:latest` (override the image with `RUSTFS_IMAGE`). They run the shared
239
+ `createFilesApiTests` and `createBigFilesApiTests` suites from `@statewalker/webrun-files-tests`.
225
240
 
226
- ```typescript
227
- import { MinioContainer } from '@testcontainers/minio';
228
- import { S3Client, CreateBucketCommand } from '@aws-sdk/client-s3';
229
- import { S3FilesApi } from '@statewalker/webrun-files-s3';
241
+ ```bash
242
+ pnpm test # unit profile: excludes *.integration.test.ts, so no Docker needed (runs nothing today)
243
+ pnpm test:integration # Docker: RustFS container, all suites including the 256 MiB big-file suite
244
+ ```
230
245
 
231
- const minioContainer = await new MinioContainer().start();
246
+ The client the tests build is the one to use against any S3-compatible server on a custom endpoint:
232
247
 
248
+ ```typescript
233
249
  const s3Client = new S3Client({
234
- endpoint: minioContainer.getConnectionUrl(),
250
+ endpoint: `http://${container.getHost()}:${container.getMappedPort(9000)}`,
235
251
  region: 'us-east-1',
236
- credentials: {
237
- accessKeyId: minioContainer.getUsername(),
238
- secretAccessKey: minioContainer.getPassword(),
239
- },
252
+ credentials: { accessKeyId: 'rustfsadmin', secretAccessKey: 'rustfsadmin' },
240
253
  forcePathStyle: true,
241
254
  });
242
-
243
- await s3Client.send(new CreateBucketCommand({ Bucket: 'test-bucket' }));
244
-
245
- const files = new S3FilesApi({
246
- client: s3Client,
247
- bucket: 'test-bucket',
248
- });
249
255
  ```
250
256
 
251
257
  ## License
@@ -154,10 +154,32 @@ var S3FilesApi = class {
154
154
  });
155
155
  await this.client.send(command);
156
156
  }
157
+ /**
158
+ * Keys list in UTF-8 byte order, which is `comparePaths` order for files.
159
+ * A recursive listing is therefore already ordered. A non-recursive one is
160
+ * not quite: the directory `a` arrives as the common prefix `a/`, after keys
161
+ * such as `a-x` and `a.txt` that must follow it, so entries pass through a
162
+ * small reorder buffer (see {@link releaseBound}).
163
+ */
157
164
  async *list(path, options) {
158
165
  const recursive = options?.recursive ?? false;
166
+ const after = options?.after;
167
+ const dirPath = (0, _statewalker_webrun_files.normalizePath)(path);
159
168
  const prefix = this.resolveKey(path);
160
169
  const normalizedPrefix = prefix ? `${prefix}/` : this.prefix ? `${this.prefix}/` : "";
170
+ const dirPrefix = dirPath === "/" ? "" : dirPath;
171
+ let startAfter;
172
+ if (after !== void 0) {
173
+ if (after.startsWith(`${dirPrefix}/`)) startAfter = normalizedPrefix + after.slice(dirPrefix.length + 1);
174
+ else if ((0, _statewalker_webrun_files.comparePaths)(after, `${dirPrefix}/`) >= 0) return;
175
+ }
176
+ const pending = [];
177
+ let released;
178
+ const release = (entry) => {
179
+ if (entry.path === released) return false;
180
+ released = entry.path;
181
+ return after === void 0 || (0, _statewalker_webrun_files.comparePaths)(entry.path, after) > 0;
182
+ };
161
183
  let continuationToken;
162
184
  do {
163
185
  const command = new _aws_sdk_client_s3.ListObjectsV2Command({
@@ -165,35 +187,60 @@ var S3FilesApi = class {
165
187
  Prefix: normalizedPrefix,
166
188
  Delimiter: recursive ? void 0 : "/",
167
189
  ContinuationToken: continuationToken,
190
+ StartAfter: continuationToken ? void 0 : startAfter,
168
191
  MaxKeys: 1e3
169
192
  });
170
193
  const response = await this.client.send(command);
171
- if (!recursive && response.CommonPrefixes) for (const cpPrefix of response.CommonPrefixes) {
194
+ const page = [];
195
+ for (const cpPrefix of recursive ? [] : response.CommonPrefixes ?? []) {
172
196
  if (!cpPrefix.Prefix) continue;
173
197
  const dirKey = cpPrefix.Prefix.endsWith("/") ? cpPrefix.Prefix.slice(0, -1) : cpPrefix.Prefix;
174
- const dirPath = this.keyToPath(dirKey);
175
- yield {
176
- kind: "directory",
177
- name: (0, _statewalker_webrun_files.basename)(dirPath),
178
- path: dirPath
179
- };
198
+ const childPath = this.keyToPath(dirKey);
199
+ page.push({
200
+ key: cpPrefix.Prefix,
201
+ entry: {
202
+ kind: "directory",
203
+ name: (0, _statewalker_webrun_files.basename)(childPath),
204
+ path: childPath
205
+ }
206
+ });
180
207
  }
181
- if (response.Contents) for (const obj of response.Contents) {
208
+ for (const obj of response.Contents ?? []) {
182
209
  if (!obj.Key) continue;
183
210
  if (obj.Key === normalizedPrefix) continue;
184
211
  if (`${obj.Key}/` === normalizedPrefix) continue;
185
- if (obj.Key.endsWith("/")) continue;
212
+ if (obj.Key.endsWith("/")) {
213
+ page.push({ key: obj.Key });
214
+ continue;
215
+ }
186
216
  const filePath = this.keyToPath(obj.Key);
187
- yield {
188
- kind: "file",
189
- name: (0, _statewalker_webrun_files.basename)(filePath),
190
- path: filePath,
191
- size: obj.Size ?? 0,
192
- lastModified: obj.LastModified?.getTime() ?? 0
193
- };
217
+ page.push({
218
+ key: obj.Key,
219
+ entry: {
220
+ kind: "file",
221
+ name: (0, _statewalker_webrun_files.basename)(filePath),
222
+ path: filePath,
223
+ size: obj.Size ?? 0,
224
+ lastModified: obj.LastModified?.getTime() ?? 0
225
+ }
226
+ });
227
+ }
228
+ page.sort((x, y) => (0, _statewalker_webrun_files.comparePaths)(x.key, y.key));
229
+ for (const { key, entry } of page) {
230
+ if (recursive) {
231
+ if (entry && release(entry)) yield entry;
232
+ continue;
233
+ }
234
+ if (entry) insertInOrder(pending, entry);
235
+ const bound = releaseBound(this.keyToPath(key));
236
+ while (pending.length > 0 && (0, _statewalker_webrun_files.comparePaths)(pending[0].path, bound) < 0) {
237
+ const next = pending.shift();
238
+ if (release(next)) yield next;
239
+ }
194
240
  }
195
241
  continuationToken = response.NextContinuationToken;
196
242
  } while (continuationToken);
243
+ for (const entry of pending) if (release(entry)) yield entry;
197
244
  }
198
245
  async stats(path) {
199
246
  const normalized = (0, _statewalker_webrun_files.normalizePath)(path);
@@ -314,5 +361,21 @@ var S3FilesApi = class {
314
361
  return copied;
315
362
  }
316
363
  };
364
+ /**
365
+ * Once a delimited listing has reached key position `position` (as a path), a
366
+ * directory still to come — which arrives as `D + "/"` at or after that
367
+ * position — has a path of at least this bound: the shortest prefix of
368
+ * `position` followed by a character that sorts before `/`, or `position`
369
+ * itself if there is none. Buffered entries below the bound are final.
370
+ */
371
+ function releaseBound(position) {
372
+ for (let i = 0; i < position.length; i++) if (position.charCodeAt(i) < 47) return position.slice(0, i);
373
+ return position;
374
+ }
375
+ function insertInOrder(entries, entry) {
376
+ let i = entries.length;
377
+ while (i > 0 && (0, _statewalker_webrun_files.comparePaths)(entries[i - 1].path, entry.path) > 0) i--;
378
+ entries.splice(i, 0, entry);
379
+ }
317
380
  //#endregion
318
381
  exports.S3FilesApi = S3FilesApi;
package/dist/esm/index.js CHANGED
@@ -1,5 +1,5 @@
1
1
  import { AbortMultipartUploadCommand, CompleteMultipartUploadCommand, CopyObjectCommand, CreateMultipartUploadCommand, DeleteObjectCommand, GetObjectCommand, HeadObjectCommand, ListObjectsV2Command, PutObjectCommand, UploadPartCommand } from "@aws-sdk/client-s3";
2
- import { basename, normalizePath } from "@statewalker/webrun-files";
2
+ import { basename, comparePaths, normalizePath } from "@statewalker/webrun-files";
3
3
  //#region src/s3-files-api.ts
4
4
  /** Default part size for multipart uploads: 5MB (S3 minimum) */
5
5
  const DEFAULT_PART_SIZE = 5242880;
@@ -153,10 +153,32 @@ var S3FilesApi = class {
153
153
  });
154
154
  await this.client.send(command);
155
155
  }
156
+ /**
157
+ * Keys list in UTF-8 byte order, which is `comparePaths` order for files.
158
+ * A recursive listing is therefore already ordered. A non-recursive one is
159
+ * not quite: the directory `a` arrives as the common prefix `a/`, after keys
160
+ * such as `a-x` and `a.txt` that must follow it, so entries pass through a
161
+ * small reorder buffer (see {@link releaseBound}).
162
+ */
156
163
  async *list(path, options) {
157
164
  const recursive = options?.recursive ?? false;
165
+ const after = options?.after;
166
+ const dirPath = normalizePath(path);
158
167
  const prefix = this.resolveKey(path);
159
168
  const normalizedPrefix = prefix ? `${prefix}/` : this.prefix ? `${this.prefix}/` : "";
169
+ const dirPrefix = dirPath === "/" ? "" : dirPath;
170
+ let startAfter;
171
+ if (after !== void 0) {
172
+ if (after.startsWith(`${dirPrefix}/`)) startAfter = normalizedPrefix + after.slice(dirPrefix.length + 1);
173
+ else if (comparePaths(after, `${dirPrefix}/`) >= 0) return;
174
+ }
175
+ const pending = [];
176
+ let released;
177
+ const release = (entry) => {
178
+ if (entry.path === released) return false;
179
+ released = entry.path;
180
+ return after === void 0 || comparePaths(entry.path, after) > 0;
181
+ };
160
182
  let continuationToken;
161
183
  do {
162
184
  const command = new ListObjectsV2Command({
@@ -164,35 +186,60 @@ var S3FilesApi = class {
164
186
  Prefix: normalizedPrefix,
165
187
  Delimiter: recursive ? void 0 : "/",
166
188
  ContinuationToken: continuationToken,
189
+ StartAfter: continuationToken ? void 0 : startAfter,
167
190
  MaxKeys: 1e3
168
191
  });
169
192
  const response = await this.client.send(command);
170
- if (!recursive && response.CommonPrefixes) for (const cpPrefix of response.CommonPrefixes) {
193
+ const page = [];
194
+ for (const cpPrefix of recursive ? [] : response.CommonPrefixes ?? []) {
171
195
  if (!cpPrefix.Prefix) continue;
172
196
  const dirKey = cpPrefix.Prefix.endsWith("/") ? cpPrefix.Prefix.slice(0, -1) : cpPrefix.Prefix;
173
- const dirPath = this.keyToPath(dirKey);
174
- yield {
175
- kind: "directory",
176
- name: basename(dirPath),
177
- path: dirPath
178
- };
197
+ const childPath = this.keyToPath(dirKey);
198
+ page.push({
199
+ key: cpPrefix.Prefix,
200
+ entry: {
201
+ kind: "directory",
202
+ name: basename(childPath),
203
+ path: childPath
204
+ }
205
+ });
179
206
  }
180
- if (response.Contents) for (const obj of response.Contents) {
207
+ for (const obj of response.Contents ?? []) {
181
208
  if (!obj.Key) continue;
182
209
  if (obj.Key === normalizedPrefix) continue;
183
210
  if (`${obj.Key}/` === normalizedPrefix) continue;
184
- if (obj.Key.endsWith("/")) continue;
211
+ if (obj.Key.endsWith("/")) {
212
+ page.push({ key: obj.Key });
213
+ continue;
214
+ }
185
215
  const filePath = this.keyToPath(obj.Key);
186
- yield {
187
- kind: "file",
188
- name: basename(filePath),
189
- path: filePath,
190
- size: obj.Size ?? 0,
191
- lastModified: obj.LastModified?.getTime() ?? 0
192
- };
216
+ page.push({
217
+ key: obj.Key,
218
+ entry: {
219
+ kind: "file",
220
+ name: basename(filePath),
221
+ path: filePath,
222
+ size: obj.Size ?? 0,
223
+ lastModified: obj.LastModified?.getTime() ?? 0
224
+ }
225
+ });
226
+ }
227
+ page.sort((x, y) => comparePaths(x.key, y.key));
228
+ for (const { key, entry } of page) {
229
+ if (recursive) {
230
+ if (entry && release(entry)) yield entry;
231
+ continue;
232
+ }
233
+ if (entry) insertInOrder(pending, entry);
234
+ const bound = releaseBound(this.keyToPath(key));
235
+ while (pending.length > 0 && comparePaths(pending[0].path, bound) < 0) {
236
+ const next = pending.shift();
237
+ if (release(next)) yield next;
238
+ }
193
239
  }
194
240
  continuationToken = response.NextContinuationToken;
195
241
  } while (continuationToken);
242
+ for (const entry of pending) if (release(entry)) yield entry;
196
243
  }
197
244
  async stats(path) {
198
245
  const normalized = normalizePath(path);
@@ -313,5 +360,21 @@ var S3FilesApi = class {
313
360
  return copied;
314
361
  }
315
362
  };
363
+ /**
364
+ * Once a delimited listing has reached key position `position` (as a path), a
365
+ * directory still to come — which arrives as `D + "/"` at or after that
366
+ * position — has a path of at least this bound: the shortest prefix of
367
+ * `position` followed by a character that sorts before `/`, or `position`
368
+ * itself if there is none. Buffered entries below the bound are final.
369
+ */
370
+ function releaseBound(position) {
371
+ for (let i = 0; i < position.length; i++) if (position.charCodeAt(i) < 47) return position.slice(0, i);
372
+ return position;
373
+ }
374
+ function insertInOrder(entries, entry) {
375
+ let i = entries.length;
376
+ while (i > 0 && comparePaths(entries[i - 1].path, entry.path) > 0) i--;
377
+ entries.splice(i, 0, entry);
378
+ }
316
379
  //#endregion
317
380
  export { S3FilesApi };
@@ -43,6 +43,13 @@ export declare class S3FilesApi implements FilesApi {
43
43
  read(path: string, options?: ReadOptions): AsyncIterable<Uint8Array>;
44
44
  write(path: string, content: Iterable<Uint8Array> | AsyncIterable<Uint8Array>): Promise<void>;
45
45
  mkdir(path: string): Promise<void>;
46
+ /**
47
+ * Keys list in UTF-8 byte order, which is `comparePaths` order for files.
48
+ * A recursive listing is therefore already ordered. A non-recursive one is
49
+ * not quite: the directory `a` arrives as the common prefix `a/`, after keys
50
+ * such as `a-x` and `a.txt` that must follow it, so entries pass through a
51
+ * small reorder buffer (see {@link releaseBound}).
52
+ */
46
53
  list(path: string, options?: ListOptions): AsyncIterable<FileInfo>;
47
54
  stats(path: string): Promise<FileStats | undefined>;
48
55
  exists(path: string): Promise<boolean>;
@@ -1 +1 @@
1
- {"version":3,"file":"s3-files-api.d.ts","sourceRoot":"","sources":["../src/s3-files-api.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAanD,OAAO,KAAK,EACV,QAAQ,EACR,SAAS,EACT,QAAQ,EACR,WAAW,EACX,WAAW,EACZ,MAAM,2BAA2B,CAAC;AASnC,MAAM,WAAW,iBAAiB;IAChC;;OAEG;IACH,MAAM,EAAE,QAAQ,CAAC;IAEjB;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IAEf;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB;;;;OAIG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED;;;;;GAKG;AACH,qBAAa,UAAW,YAAW,QAAQ;IACzC,OAAO,CAAC,MAAM,CAAW;IACzB,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,QAAQ,CAAS;IAEzB,YAAY,OAAO,EAAE,iBAAiB,EAKrC;IAED,OAAO,CAAC,UAAU;IAUlB,OAAO,CAAC,SAAS;IAWV,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,WAAW,GAAG,aAAa,CAAC,UAAU,CAAC,CA6C1E;IAEK,KAAK,CACT,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,QAAQ,CAAC,UAAU,CAAC,GAAG,aAAa,CAAC,UAAU,CAAC,GACxD,OAAO,CAAC,IAAI,CAAC,CAkHf;IAEK,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAYvC;IAEM,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,WAAW,GAAG,aAAa,CAAC,QAAQ,CAAC,CAiExE;IAEK,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,GAAG,SAAS,CAAC,CA+CxD;IAEK,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAG3C;IAEK,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAkE3C;IAEK,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAI3D;IAEK,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAwD3D;CACF"}
1
+ {"version":3,"file":"s3-files-api.d.ts","sourceRoot":"","sources":["../src/s3-files-api.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAanD,OAAO,KAAK,EACV,QAAQ,EACR,SAAS,EACT,QAAQ,EACR,WAAW,EACX,WAAW,EACZ,MAAM,2BAA2B,CAAC;AASnC,MAAM,WAAW,iBAAiB;IAChC;;OAEG;IACH,MAAM,EAAE,QAAQ,CAAC;IAEjB;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IAEf;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB;;;;OAIG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED;;;;;GAKG;AACH,qBAAa,UAAW,YAAW,QAAQ;IACzC,OAAO,CAAC,MAAM,CAAW;IACzB,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,QAAQ,CAAS;IAEzB,YAAY,OAAO,EAAE,iBAAiB,EAKrC;IAED,OAAO,CAAC,UAAU;IAUlB,OAAO,CAAC,SAAS;IAWV,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,WAAW,GAAG,aAAa,CAAC,UAAU,CAAC,CA6C1E;IAEK,KAAK,CACT,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,QAAQ,CAAC,UAAU,CAAC,GAAG,aAAa,CAAC,UAAU,CAAC,GACxD,OAAO,CAAC,IAAI,CAAC,CAkHf;IAEK,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAYvC;IAED;;;;;;OAMG;IACI,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,WAAW,GAAG,aAAa,CAAC,QAAQ,CAAC,CA0GxE;IAEK,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,GAAG,SAAS,CAAC,CA+CxD;IAEK,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAG3C;IAEK,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAkE3C;IAEK,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAI3D;IAEK,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAwD3D;CACF"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@statewalker/webrun-files-s3",
3
- "version": "0.9.0",
3
+ "version": "0.10.2",
4
4
  "private": false,
5
5
  "type": "module",
6
6
  "description": "S3-backed FilesApi implementation for webrun-files",
@@ -19,6 +19,7 @@
19
19
  "types": "./dist/index.d.ts",
20
20
  "exports": {
21
21
  ".": {
22
+ "source": "./src/index.ts",
22
23
  "types": "./dist/index.d.ts",
23
24
  "import": "./dist/esm/index.js",
24
25
  "require": "./dist/cjs/index.cjs"
@@ -29,20 +30,20 @@
29
30
  "src"
30
31
  ],
31
32
  "dependencies": {
32
- "@statewalker/webrun-files": "0.9.0"
33
+ "@statewalker/webrun-files": "^0.10.1"
33
34
  },
34
35
  "peerDependencies": {
35
36
  "@aws-sdk/client-s3": "^3.1032.0"
36
37
  },
37
38
  "devDependencies": {
38
- "@aws-sdk/client-s3": "^3.1110.0",
39
- "@types/node": "^26.2.0",
39
+ "@aws-sdk/client-s3": "^3.1146.0",
40
+ "@types/node": "^26.6.4",
40
41
  "rimraf": "^6.1.3",
41
- "rolldown": "^1.2.4",
42
- "testcontainers": "^12.1.0",
42
+ "rolldown": "^1.2.12",
43
+ "testcontainers": "^12.2.0",
43
44
  "typescript": "^7.0.2",
44
- "vitest": "^4.1.10",
45
- "@statewalker/webrun-files-tests": "2.0.0"
45
+ "vitest": "^5.0.3",
46
+ "@statewalker/webrun-files-tests": "^3.0.0"
46
47
  },
47
48
  "sideEffects": false,
48
49
  "publishConfig": {
@@ -24,7 +24,7 @@ import type {
24
24
  ListOptions,
25
25
  ReadOptions,
26
26
  } from "@statewalker/webrun-files";
27
- import { basename, normalizePath } from "@statewalker/webrun-files";
27
+ import { basename, comparePaths, normalizePath } from "@statewalker/webrun-files";
28
28
 
29
29
  /** Default part size for multipart uploads: 5MB (S3 minimum) */
30
30
  const DEFAULT_PART_SIZE = 5 * 1024 * 1024;
@@ -277,11 +277,42 @@ export class S3FilesApi implements FilesApi {
277
277
  await this.client.send(command);
278
278
  }
279
279
 
280
+ /**
281
+ * Keys list in UTF-8 byte order, which is `comparePaths` order for files.
282
+ * A recursive listing is therefore already ordered. A non-recursive one is
283
+ * not quite: the directory `a` arrives as the common prefix `a/`, after keys
284
+ * such as `a-x` and `a.txt` that must follow it, so entries pass through a
285
+ * small reorder buffer (see {@link releaseBound}).
286
+ */
280
287
  async *list(path: string, options?: ListOptions): AsyncIterable<FileInfo> {
281
288
  const recursive = options?.recursive ?? false;
289
+ const after = options?.after;
290
+ const dirPath = normalizePath(path);
282
291
  const prefix = this.resolveKey(path);
283
292
  const normalizedPrefix = prefix ? `${prefix}/` : this.prefix ? `${this.prefix}/` : "";
284
293
 
294
+ // `after` as a key, by raw prefix replacement: normalising the path could
295
+ // move the cursor past entries that sort after it.
296
+ const dirPrefix = dirPath === "/" ? "" : dirPath;
297
+ let startAfter: string | undefined;
298
+ if (after !== undefined) {
299
+ if (after.startsWith(`${dirPrefix}/`)) {
300
+ startAfter = normalizedPrefix + after.slice(dirPrefix.length + 1);
301
+ } else if (comparePaths(after, `${dirPrefix}/`) >= 0) {
302
+ return; // the whole directory lies at or before the cursor
303
+ }
304
+ }
305
+
306
+ const pending: FileInfo[] = [];
307
+ // A key `a` and keys under `a/` make a file and a directory with the same
308
+ // path. A listing is strictly increasing, so the first released — the file,
309
+ // as `stats` also reports — is kept.
310
+ let released: string | undefined;
311
+ const release = (entry: FileInfo): boolean => {
312
+ if (entry.path === released) return false;
313
+ released = entry.path;
314
+ return after === undefined || comparePaths(entry.path, after) > 0;
315
+ };
285
316
  let continuationToken: string | undefined;
286
317
 
287
318
  do {
@@ -290,58 +321,75 @@ export class S3FilesApi implements FilesApi {
290
321
  Prefix: normalizedPrefix,
291
322
  Delimiter: recursive ? undefined : "/",
292
323
  ContinuationToken: continuationToken,
324
+ StartAfter: continuationToken ? undefined : startAfter,
293
325
  MaxKeys: 1000,
294
326
  });
295
327
 
296
328
  const response = await this.client.send(command);
297
329
 
298
- if (!recursive && response.CommonPrefixes) {
299
- for (const cpPrefix of response.CommonPrefixes) {
300
- if (!cpPrefix.Prefix) continue;
301
-
302
- const dirKey = cpPrefix.Prefix.endsWith("/")
303
- ? cpPrefix.Prefix.slice(0, -1)
304
- : cpPrefix.Prefix;
305
-
306
- const dirPath = this.keyToPath(dirKey);
307
-
308
- // A common prefix IS the directory variant: it exists only as an
309
- // artefact of key naming, so there is nothing to report about it
310
- // beyond its being a directory.
311
- yield {
312
- kind: "directory",
313
- name: basename(dirPath),
314
- path: dirPath,
315
- };
316
- }
330
+ // Common prefixes and contents each come sorted; interleave them by key.
331
+ const page: { key: string; entry?: FileInfo }[] = [];
332
+ for (const cpPrefix of recursive ? [] : (response.CommonPrefixes ?? [])) {
333
+ if (!cpPrefix.Prefix) continue;
334
+ const dirKey = cpPrefix.Prefix.endsWith("/")
335
+ ? cpPrefix.Prefix.slice(0, -1)
336
+ : cpPrefix.Prefix;
337
+ const childPath = this.keyToPath(dirKey);
338
+ // A common prefix IS the directory variant: it exists only as an
339
+ // artefact of key naming, so there is nothing to report about it
340
+ // beyond its being a directory.
341
+ page.push({
342
+ key: cpPrefix.Prefix,
343
+ entry: { kind: "directory", name: basename(childPath), path: childPath },
344
+ });
317
345
  }
318
-
319
- if (response.Contents) {
320
- for (const obj of response.Contents) {
321
- if (!obj.Key) continue;
322
- if (obj.Key === normalizedPrefix) continue;
323
- if (`${obj.Key}/` === normalizedPrefix) continue;
324
- // A key ending in "/" is the zero-byte marker `mkdir` writes to make
325
- // an empty directory visible. It is NOT a file with `size: 0` — it
326
- // is how this store spells a directory, and it surfaces through
327
- // `CommonPrefixes` above. Every other key is a file, and a genuinely
328
- // empty object is the file variant with `size: 0`.
329
- if (obj.Key.endsWith("/")) continue;
330
-
331
- const filePath = this.keyToPath(obj.Key);
332
-
333
- yield {
346
+ for (const obj of response.Contents ?? []) {
347
+ if (!obj.Key) continue;
348
+ if (obj.Key === normalizedPrefix) continue;
349
+ if (`${obj.Key}/` === normalizedPrefix) continue;
350
+ // A key ending in "/" is the zero-byte marker `mkdir` writes to make
351
+ // an empty directory visible. It is NOT a file with `size: 0` — it
352
+ // is how this store spells a directory, and it surfaces through
353
+ // `CommonPrefixes` above. Every other key is a file, and a genuinely
354
+ // empty object is the file variant with `size: 0`. It still moves
355
+ // the listing's position forward.
356
+ if (obj.Key.endsWith("/")) {
357
+ page.push({ key: obj.Key });
358
+ continue;
359
+ }
360
+ const filePath = this.keyToPath(obj.Key);
361
+ page.push({
362
+ key: obj.Key,
363
+ entry: {
334
364
  kind: "file",
335
365
  name: basename(filePath),
336
366
  path: filePath,
337
367
  size: obj.Size ?? 0,
338
368
  lastModified: obj.LastModified?.getTime() ?? 0,
339
- };
369
+ },
370
+ });
371
+ }
372
+ page.sort((x, y) => comparePaths(x.key, y.key));
373
+
374
+ for (const { key, entry } of page) {
375
+ if (recursive) {
376
+ if (entry && release(entry)) yield entry;
377
+ continue;
378
+ }
379
+ if (entry) insertInOrder(pending, entry);
380
+ const bound = releaseBound(this.keyToPath(key));
381
+ while (pending.length > 0 && comparePaths(pending[0].path, bound) < 0) {
382
+ const next = pending.shift() as FileInfo;
383
+ if (release(next)) yield next;
340
384
  }
341
385
  }
342
386
 
343
387
  continuationToken = response.NextContinuationToken;
344
388
  } while (continuationToken);
389
+
390
+ for (const entry of pending) {
391
+ if (release(entry)) yield entry;
392
+ }
345
393
  }
346
394
 
347
395
  async stats(path: string): Promise<FileStats | undefined> {
@@ -530,3 +578,24 @@ export class S3FilesApi implements FilesApi {
530
578
  return copied;
531
579
  }
532
580
  }
581
+
582
+ /**
583
+ * Once a delimited listing has reached key position `position` (as a path), a
584
+ * directory still to come — which arrives as `D + "/"` at or after that
585
+ * position — has a path of at least this bound: the shortest prefix of
586
+ * `position` followed by a character that sorts before `/`, or `position`
587
+ * itself if there is none. Buffered entries below the bound are final.
588
+ */
589
+ function releaseBound(position: string): string {
590
+ for (let i = 0; i < position.length; i++) {
591
+ if (position.charCodeAt(i) < 0x2f) return position.slice(0, i);
592
+ }
593
+ return position;
594
+ }
595
+
596
+ function insertInOrder(entries: FileInfo[], entry: FileInfo): void {
597
+ let i = entries.length;
598
+ // Equal paths keep arrival order: a file arrives before its same-named prefix.
599
+ while (i > 0 && comparePaths(entries[i - 1].path, entry.path) > 0) i--;
600
+ entries.splice(i, 0, entry);
601
+ }