lean-s3 0.1.3 → 0.1.4

package/README.md

@@ -1,4 +1,4 @@
-# lean-s3
+# lean-s3 [![npm badge](https://img.shields.io/npm/v/lean-s3)](https://www.npmjs.com/package/lean-s3)
 
 A server-side S3 API for the regular user. lean-s3 tries to provide the 80% of S3 that most people use. It is heavily inspired by [Bun's S3 API](https://bun.sh/docs/api/s3). Requires a Node.js version that supports `fetch`.
 
@@ -61,7 +61,7 @@ pnpm add lean-s3
 ```
 
 ## Why?
-[@aws-sdk/client-s3](https://github.com/aws/aws-sdk-js-v3) is cumbersome to use and doesn't align well with the current web standards. It is focused on providing a great experienced when used in conjunction with other AWS services. This comes at the cost of performance and package size:
+[@aws-sdk/client-s3](https://github.com/aws/aws-sdk-js-v3) is cumbersome to use and doesn't align well with the current web standards. It is focused on providing a great experience when used in conjunction with other AWS services. This comes at the cost of performance and package size:
 
 ```sh
 $ npm i @aws-sdk/client-s3 @aws-sdk/s3-request-presigner
@@ -83,27 +83,27 @@ lean-s3 is currently about 20x faster than AWS SDK when it comes to pre-signing
 ```
 benchmark                    avg (min … max) p75 / p99
 -------------------------------------------- ---------
-@aws-sdk/s3-request-presigner 184.32 µs/iter 183.38 µs
-                       (141.15 µs … 1.19 ms) 579.17 µs
-                     (312.00  b …   5.07 mb) 233.20 kb
+@aws-sdk/s3-request-presigner 130.73 µs/iter 128.99 µs
+                     (102.27 µs … 938.72 µs) 325.96 µs
+                     (712.00  b …   5.85 mb) 228.48 kb
 
-lean-s3                         8.48 µs/iter   8.21 µs
-                         (7.85 µs … 1.06 ms)  11.23 µs
-                     (128.00  b … 614.83 kb)   5.26 kb
+lean-s3                         4.22 µs/iter   4.20 µs
+                         (4.02 µs … 5.96 µs)   4.52 µs
+                     (  3.54 kb …   3.54 kb)   3.54 kb
 
-aws4fetch                      65.49 µs/iter  62.83 µs
-                        (52.43 µs … 1.01 ms) 158.99 µs
-                     ( 24.00  b …   1.42 mb)  53.38 kb
+aws4fetch                      52.41 µs/iter  50.71 µs
+                        (36.06 µs … 1.79 ms) 173.15 µs
+                     ( 24.00  b …   1.66 mb)  51.60 kb
 
-minio client                   19.82 µs/iter  18.35 µs
-                        (17.28 µs … 1.61 ms)  34.41 µs
-                     (768.00  b … 721.07 kb)  16.18 kb
+minio client                   16.21 µs/iter  15.13 µs
+                        (13.14 µs … 1.25 ms)  27.08 µs
+                     (192.00  b …   1.43 mb)  16.02 kb
 
 summary
   lean-s3
-   2.34x faster than minio client
-   7.72x faster than aws4fetch
-   21.74x faster than @aws-sdk/s3-request-presigner
+   3.84x faster than minio client
+   12.42x faster than aws4fetch
+   30.99x faster than @aws-sdk/s3-request-presigner
 ```
 
 Don't trust this benchmark and run it yourself[^2]. I am just some random internet guy trying to tell you [how much better this s3 client is](https://xkcd.com/927/). For `PUT` operations, it is ~1.45x faster than `@aws-sdk/client-s3`. We still work on improving these numbers.

package/package.json

@@ -2,7 +2,7 @@
   "name": "lean-s3",
   "author": "Niklas Mollenhauer",
   "license": "MIT",
-  "version": "0.1.3",
+  "version": "0.1.4",
   "description": "A server-side S3 API for the regular user.",
   "keywords": [
     "s3",
@@ -27,19 +27,20 @@
     "format": "biome format --write ./src && biome lint --write ./src && biome check --write ./src"
   },
   "devDependencies": {
-    "@aws-sdk/client-s3": "^3.787.0",
+    "@aws-sdk/client-s3": "^3.802.0",
     "@biomejs/biome": "^1.9.4",
-    "@testcontainers/localstack": "^10.24.2",
-    "@testcontainers/minio": "^10.24.2",
-    "@types/node": "^22.14.1",
+    "@testcontainers/localstack": "^10.25.0",
+    "@testcontainers/minio": "^10.25.0",
+    "@types/node": "^22.15.3",
     "expect": "^29.7.0",
-    "lefthook": "^1.11.10",
-    "typedoc": "^0.28.2"
+    "lefthook": "^1.11.12",
+    "typedoc": "^0.28.4"
   },
   "engines": {
-    "node": "^20.19.0 || ^22.14.0"
+    "node": "^20.19.0 || ^22.14.0 || ^24.0.0"
   },
   "dependencies": {
+    "fast-xml-parser": "^5.2.1",
     "undici": "^7.8.0"
   }
 }

package/src/AmzDate.js

@@ -1,5 +1,3 @@
-// @ts-check
-
 const ONE_DAY = 1000 * 60 * 60 * 24;
 
 /**

package/src/KeyCache.js

@@ -1,5 +1,3 @@
-// @ts-check
-
 import * as sign from "./sign.js";
 
 /**@typedef {import("./AmzDate.js").AmzDate} AmzDate */

package/src/S3BucketEntry.js

@@ -0,0 +1,91 @@
+/**
+ * @typedef {import("./index.js").StorageClass} StorageClass
+ * @typedef {import("./index.js").ChecksumAlgorithm} ChecksumAlgorithm
+ * @typedef {import("./index.js").ChecksumType} ChecksumType
+ */
+
+/**
+ * @internal Normally, we'd use an interface for that, but having a class with pre-defined fields makes it easier for V8 top optimize hidden classes.
+ */
+export default class S3BucketEntry {
+	/**
+	 * @readonly
+	 * @type {string}
+	 */
+	key;
+	/**
+	 * @readonly
+	 * @type {number}
+	 */
+	size;
+	/**
+	 * @readonly
+	 * @type {Date}
+	 */
+	lastModified;
+	/**
+	 * @readonly
+	 * @type {string}
+	 */
+	etag;
+	/**
+	 * @readonly
+	 * @type {StorageClass}
+	 */
+	storageClass;
+	/**
+	 * @readonly
+	 * @type {ChecksumAlgorithm | undefined}
+	 */
+	checksumAlgorithm;
+	/**
+	 * @readonly
+	 * @type {ChecksumType | undefined}
+	 */
+	checksumType;
+
+	/**
+	 * @param {string} key
+	 * @param {number} size
+	 * @param {Date} lastModified
+	 * @param {string} etag
+	 * @param {StorageClass} storageClass
+	 * @param {ChecksumAlgorithm | undefined} checksumAlgorithm
+	 * @param {ChecksumType | undefined} checksumType
+	 */
+	constructor(
+		key,
+		size,
+		lastModified,
+		etag,
+		storageClass,
+		checksumAlgorithm,
+		checksumType,
+	) {
+		this.key = key;
+		this.size = size;
+		this.lastModified = lastModified;
+		this.etag = etag;
+		this.storageClass = storageClass;
+		this.checksumAlgorithm = checksumAlgorithm;
+		this.checksumType = checksumType;
+	}
+
+	/**
+	 * @internal
+	 * @param {any} source
+	 * @returns {S3BucketEntry}
+	 */
+	static parse(source) {
+		// TODO: check values and throw exceptions
+		return new S3BucketEntry(
+			source.Key,
+			source.Size,
+			new Date(source.LastModified),
+			source.ETag,
+			source.StorageClass,
+			source.ChecksumAlgorithm,
+			source.ChecksumType,
+		);
+	}
+}

package/src/S3Client.js

@@ -1,9 +1,9 @@
-// @ts-check
-
 import { request, Dispatcher, Agent } from "undici";
+import { XMLParser, XMLBuilder } from "fast-xml-parser";
 
 import S3File from "./S3File.js";
 import S3Error from "./S3Error.js";
+import S3BucketEntry from "./S3BucketEntry.js";
 import KeyCache from "./KeyCache.js";
 import * as amzDate from "./AmzDate.js";
 import * as sign from "./sign.js";
@@ -16,6 +16,9 @@ import {
 export const write = Symbol("write");
 export const stream = Symbol("stream");
 
+const xmlParser = new XMLParser();
+const xmlBuilder = new XMLBuilder();
+
 /**
  * @typedef {import("./index.d.ts").S3ClientOptions} S3ClientOptions
  * @typedef {import("./index.d.ts").PresignableHttpMethod} PresignableHttpMethod
@@ -24,6 +27,7 @@ export const stream = Symbol("stream");
  * @typedef {import("./index.d.ts").S3FilePresignOptions} S3FilePresignOptions
  * @typedef {import("./index.d.ts").OverridableS3ClientOptions} OverridableS3ClientOptions
  * @typedef {import("./index.d.ts").CreateFileInstanceOptions} CreateFileInstanceOptions
+ * @typedef {import("./index.d.ts").ListObjectsResponse} ListObjectsResponse
  */
 
 /**
@@ -167,7 +171,7 @@ export default class S3Client {
 		const dataDigest = sign.createCanonicalDataDigestHostOnly(
 			method,
 			res.pathname,
-			query.toString(),
+			query,
 			res.host,
 		);
 
@@ -186,12 +190,245 @@ export default class S3Client {
 		);
 
 		// See `buildSearchParams` for casing on this parameter
-		query.set("X-Amz-Signature", signature);
-
-		res.search = query.toString();
+		res.search = `${query}&X-Amz-Signature=${signature}`;
 		return res.toString();
 	}
 
+	/**
+	 *
+	 * @param {readonly S3BucketEntry[] | readonly string[]} objects
+	 * @param {*} options
+	 */
+	async deleteObjects(objects, options) {
+		throw new Error("Not implemented");
+	}
+
+	//#region list
+
+	/**
+	 * Uses `ListObjectsV2` to iterate over all keys. Pagination and continuation is handled internally.
+
+	* @param {{
+	 *   prefix?: string;
+	 *   startAfter?: string;
+	 *   signal?: AbortSignal;
+	 *   internalPageSize?: number;
+	 * }} [options]
+	 * @returns {AsyncGenerator<S3BucketEntry>}
+	 */
+	async *listIterating(options) {
+		// only used to get smaller pages, so we can test this properly
+		const maxKeys = options?.internalPageSize ?? undefined;
+
+		let res = undefined;
+		let continuationToken = undefined;
+		do {
+			res = await this.list({
+				...options,
+				maxKeys,
+				continuationToken,
+			});
+
+			if (!res || res.contents.length === 0) {
+				break;
+			}
+
+			yield* res.contents;
+
+			continuationToken = res.nextContinuationToken;
+		} while (continuationToken);
+	}
+
+	/**
+	 *
+	 * @param {{
+	 *   prefix?: string;
+	 *   maxKeys?: number;
+	 *   startAfter?: string;
+	 *   continuationToken?: string;
+	 *   signal?: AbortSignal;
+	 * }} [options]
+	 * @returns {Promise<ListObjectsResponse>}
+	 */
+	async list(options = {}) {
+		// See `benchmark-operations.js` on why we don't use URLSearchParams but string concat
+		// tldr: This is faster and we know the params exactly, so we can focus our encoding
+
+		// ! minio requires these params to be in alphabetical order
+
+		let query = "";
+
+		if (typeof options.continuationToken !== "undefined") {
+			if (typeof options.continuationToken !== "string") {
+				throw new TypeError("`continuationToken` should be a `string`.");
+			}
+
+			query += `continuation-token=${encodeURIComponent(options.continuationToken)}&`;
+		}
+
+		query += "list-type=2";
+
+		if (typeof options.maxKeys !== "undefined") {
+			if (typeof options.maxKeys !== "number") {
+				throw new TypeError("`maxKeys` should be a `number`.");
+			}
+
+			query += `&max-keys=${options.maxKeys}`; // no encoding needed, it's a number
+		}
+
+		// TODO: delimiter?
+
+		// plan `if(a)` check, so empty strings will also not go into this branch, omitting the parameter
+		if (options.prefix) {
+			if (typeof options.prefix !== "string") {
+				throw new TypeError("`prefix` should be a `string`.");
+			}
+
+			query += `&prefix=${encodeURIComponent(options.prefix)}`;
+		}
+
+		if (typeof options.startAfter !== "undefined") {
+			if (typeof options.startAfter !== "string") {
+				throw new TypeError("`startAfter` should be a `string`.");
+			}
+
+			query += `&start-after=${encodeURIComponent(options.startAfter)}`;
+		}
+
+		const response = await this.#signedRequest(
+			"GET",
+			"",
+			query,
+			undefined,
+			undefined,
+			undefined,
+			undefined,
+			options.signal,
+		);
+
+		if (response.statusCode === 200) {
+			const text = await response.body.text();
+
+			let res = undefined;
+			try {
+				res = xmlParser.parse(text)?.ListBucketResult;
+			} catch (cause) {
+				// Possible according to AWS docs
+				throw new S3Error("Unknown", "", {
+					message: "S3 service responded with invalid XML.",
+					cause,
+				});
+			}
+
+			if (!res) {
+				throw new S3Error("Unknown", "", {
+					message: "Could not read bucket contents.",
+				});
+			}
+
+			// S3 is weird and doesn't return an array if there is only one item
+			const contents = Array.isArray(res.Contents)
+				? (res.Contents?.map(S3BucketEntry.parse) ?? [])
+				: res.Contents
+					? [res.Contents]
+					: [];
+
+			return {
+				name: res.Name,
+				prefix: res.Prefix,
+				startAfter: res.StartAfter,
+				isTruncated: res.IsTruncated,
+				continuationToken: res.ContinuationToken,
+				maxKeys: res.MaxKeys,
+				keyCount: res.KeyCount,
+				nextContinuationToken: res.NextContinuationToken,
+				contents,
+			};
+		}
+
+		// undici docs state that we shoul dump the body if not used
+		response.body.dump();
+		throw new Error(
+			`Response code not implemented yet: ${response.statusCode}`,
+		);
+	}
+
+	//#endregion
+
+	/**
+	 * @param {import("./index.js").HttpMethod} method
+	 * @param {string} pathWithoutBucket
+	 * @param {string | undefined} query
+	 * @param {import("./index.d.ts").UndiciBodyInit | undefined} body
+	 * @param {Record<string, string>| undefined} additionalSignedHeaders
+	 * @param {Record<string, string> | undefined} additionalUnsignedHeaders
+	 * @param {Buffer | undefined} contentHash
+	 * @param {AbortSignal | undefined} signal
+	 */
+	async #signedRequest(
+		method,
+		pathWithoutBucket,
+		query,
+		body,
+		additionalSignedHeaders,
+		additionalUnsignedHeaders,
+		contentHash,
+		signal,
+	) {
+		const bucket = this.#options.bucket;
+		const endpoint = this.#options.endpoint;
+		const region = this.#options.region;
+
+		const url = buildRequestUrl(endpoint, bucket, region, pathWithoutBucket);
+		if (query) {
+			url.search = query;
+		}
+
+		const now = amzDate.now();
+
+		const contentHashStr = contentHash?.toString("hex") ?? sign.unsignedPayload;
+
+		// Signed headers have to be sorted
+		// To enhance sorting, we're adding all possible values somehow pre-ordered
+		const headersToBeSigned = prepareHeadersForSigning({
+			host: url.host,
+			"x-amz-date": now.dateTime,
+			"x-amz-content-sha256": contentHashStr,
+			...additionalSignedHeaders,
+		});
+
+		try {
+			return await request(url, {
+				method,
+				signal,
+				dispatcher: this.#dispatcher,
+				headers: {
+					...headersToBeSigned,
+					authorization: this.#getAuthorizationHeader(
+						method,
+						url.pathname,
+						query ?? "",
+						now,
+						headersToBeSigned,
+						region,
+						contentHashStr,
+						this.#options.accessKeyId,
+						this.#options.secretAccessKey,
+					),
+					...additionalUnsignedHeaders,
+					"user-agent": "lean-s3",
+				},
+				body,
+			});
+		} catch (cause) {
+			signal?.throwIfAborted();
+			throw new S3Error("Unknown", pathWithoutBucket, {
+				message: "Unknown error during S3 request.",
+				cause,
+			});
+		}
+	}
+
 	/**
 	 * @internal
 	 * @param {string} path
@@ -222,15 +459,16 @@ export default class S3Client {
 
 		const now = amzDate.now();
 
+		const contentHashStr = contentHash?.toString("hex") ?? sign.unsignedPayload;
+
 		// Signed headers have to be sorted
 		// To enhance sorting, we're adding all possible values somehow pre-ordered
 		const headersToBeSigned = prepareHeadersForSigning({
-			"amz-sdk-invocation-id": crypto.randomUUID(),
 			"content-length": contentLength?.toString() ?? undefined,
 			"content-type": contentType,
 			host: url.host,
 			range: getRangeHeader(rageStart, rangeEndExclusive),
-			"x-amz-content-sha256": contentHash?.toString("hex") ?? undefined,
+			"x-amz-content-sha256": contentHashStr,
 			"x-amz-date": now.dateTime,
 		});
 
@@ -250,11 +488,10 @@ export default class S3Client {
 						now,
 						headersToBeSigned,
 						region,
-						contentHash,
+						contentHashStr,
 						this.#options.accessKeyId,
 						this.#options.secretAccessKey,
 					),
-					accept: "application/json", // So that we can parse errors as JSON instead of XML, if the server supports that
 					"user-agent": "lean-s3",
 				},
 				body: data,
@@ -273,25 +510,6 @@ export default class S3Client {
 			return;
 		}
 
-		const ct = response.headers["content-type"];
-		if (ct === "application/json") {
-			/** @type {any} */
-			let error = undefined;
-			try {
-				error = await response.body.json();
-			} catch (cause) {
-				throw new S3Error("Unknown", path, {
-					message: "Could not read response body.",
-					cause,
-				});
-			}
-			throw new S3Error(error?.Code ?? "Unknown", path, {
-				message: error?.Message || undefined, // Message might be "",
-				requestId: error?.RequestId || undefined, // RequestId might be ""
-				hostId: error?.HostId || undefined, // HostId might be ""
-			});
-		}
-
 		let body = undefined;
 		try {
 			body = await response.body.text();
@@ -301,12 +519,23 @@ export default class S3Client {
 				cause,
 			});
 		}
-		if (ct === "application/xml") {
-			const error = tryProcessXMLError(body);
-			throw new S3Error(error.code ?? "Unknown", path, {
-				message: error.message || undefined, // Message might be "",
+
+		if (response.headers["content-type"] === "application/xml") {
+			let error = undefined;
+			try {
+				error = xmlParser.parse(body);
+			} catch (cause) {
+				throw new S3Error("Unknown", path, {
+					message: "Could not parse XML error response.",
+					cause,
+				});
+			}
+
+			throw new S3Error(error.Code || "Unknown", path, {
+				message: error.Message || undefined, // Message might be "",
 			});
 		}
+
 		throw new S3Error("Unknown", path, {
 			message: "Unknown error during S3 request.",
 		});
@@ -331,14 +560,15 @@ export default class S3Client {
 
 		const range = getRangeHeader(rageStart, rangeEndExclusive);
 
+		const contentHashStr = contentHash?.toString("hex") ?? sign.unsignedPayload;
+
 		const headersToBeSigned = prepareHeadersForSigning({
 			"amz-sdk-invocation-id": crypto.randomUUID(),
 			// TODO: Maybe support retries and do "amz-sdk-request": attempt=1; max=3
 			host: url.host,
 			range,
 			// Hetzner doesnt care if the x-amz-content-sha256 header is missing, R2 requires it to be present
-			"x-amz-content-sha256":
-				contentHash?.toString("hex") ?? "UNSIGNED-PAYLOAD",
+			"x-amz-content-sha256": contentHashStr,
 			"x-amz-date": now.dateTime,
 		});
 
@@ -369,7 +599,7 @@ export default class S3Client {
 							now,
 							headersToBeSigned,
 							region,
-							contentHash,
+							contentHashStr,
 							this.#options.accessKeyId,
 							this.#options.secretAccessKey,
 						),
@@ -417,26 +647,22 @@ export default class S3Client {
 						const responseText = undefined;
 						const ct = response.headers["content-type"];
 
-						if (ct === "application/xml") {
+						if (response.headers["content-type"] === "application/xml") {
 							return response.body.text().then(body => {
-								const error = tryProcessXMLError(body);
-								const code = error?.code || "Unknown"; // || instead of ??, so we coerce empty strings
-								return controller.error(
-									new S3Error(code, path, {
-										message: error?.message || undefined, // || instead of ??, so we coerce empty strings
-										cause: responseText,
-									}),
-								);
-							}, onNetworkError);
-						}
-
-						if (typeof ct === "string" && ct.startsWith("application/json")) {
-							return response.body.json().then((/** @type {any} */ error) => {
-								const code = error?.code || "Unknown"; // || instead of ??, so we coerce empty strings
+								let error = undefined;
+								try {
+									error = xmlParser.parse(body);
+								} catch (cause) {
+									return controller.error(
+										new S3Error("Unknown", path, {
+											message: "Could not parse XML error response.",
+											cause,
+										}),
+									);
+								}
 								return controller.error(
-									new S3Error(code, path, {
-										message: error?.message || undefined, // || instead of ??, so we coerce empty strings
-										cause: responseText,
+									new S3Error(error.Code || "Unknown", path, {
+										message: error.Message || undefined, // Message might be "",
 									}),
 								);
 							}, onNetworkError);
@@ -465,13 +691,13 @@ export default class S3Client {
 	}
 
 	/**
-	 * @param {PresignableHttpMethod} method
+	 * @param {import("./index.js").HttpMethod} method
 	 * @param {string} path
 	 * @param {string} query
 	 * @param {amzDate.AmzDate} date
 	 * @param {Record<string, string>} sortedSignedHeaders
 	 * @param {string} region
-	 * @param {Buffer | undefined} contentHash
+	 * @param {string} contentHashStr
 	 * @param {string} accessKeyId
 	 * @param {string} secretAccessKey
 	 */
@@ -482,7 +708,7 @@ export default class S3Client {
 		date,
 		sortedSignedHeaders,
 		region,
-		contentHash,
+		contentHashStr,
 		accessKeyId,
 		secretAccessKey,
 	) {
@@ -491,7 +717,7 @@ export default class S3Client {
 			path,
 			query,
 			sortedSignedHeaders,
-			contentHash?.toString("hex") ?? sign.unsignedPayload,
+			contentHashStr,
 		);
 
 		const signingKey = this.#keyCache.computeIfAbsent(
@@ -524,7 +750,7 @@ export default class S3Client {
  * @param {string | null | undefined} sessionToken
  * @param {Acl | null | undefined} acl
  * @param {string | null | undefined} contentHashStr
- * @returns
+ * @returns {string}
  */
 export function buildSearchParams(
 	amzCredential,
@@ -539,38 +765,36 @@ export function buildSearchParams(
 	// We tried to make these query params entirely lower-cased, just like the headers
 	// but Cloudflare R2 requires them to have this exact casing
 
-	const q = new URLSearchParams();
-	q.set("X-Amz-Algorithm", "AWS4-HMAC-SHA256");
-	q.set("X-Amz-Credential", amzCredential);
-	q.set("X-Amz-Date", date.dateTime);
-	q.set("X-Amz-Expires", expiresIn.toString());
-	q.set("X-Amz-SignedHeaders", headerList);
+	// We didn't have any issues with them being in non-alphaetical order, but as some implementations decide to require sorting
+	// in non-pre-signed cases, we do it here as well
 
-	if (contentHashStr) {
-		q.set("X-Amz-Content-Sha256", contentHashStr);
+	// See `benchmark-operations.js` on why we don't use URLSearchParams but string concat
+
+	let res = "";
+
+	if (acl) {
+		res += `X-Amz-Acl=${encodeURIComponent(acl)}&`;
 	}
-	if (storageClass) {
-		q.set("X-Amz-Storage-Class", storageClass);
+
+	res += "X-Amz-Algorithm=AWS4-HMAC-SHA256";
+
+	if (contentHashStr) {
+		// We assume that this is always hex-encoded, so no encoding needed
+		res += `&X-Amz-Content-Sha256=${contentHashStr}`;
 	}
+
+	res += `&X-Amz-Credential=${encodeURIComponent(amzCredential)}`;
+	res += `&X-Amz-Date=${date.dateTime}`; // internal dateTimes don't need encoding
+	res += `&X-Amz-Expires=${expiresIn}`; // number -> no encoding
+
 	if (sessionToken) {
-		q.set("X-Amz-Security-Token", sessionToken);
+		res += `&X-Amz-Security-Token=${encodeURIComponent(sessionToken)}`;
 	}
-	if (acl) {
-		q.set("X-Amz-Acl", acl);
-	}
-	return q;
-}
 
-const codePattern = /<Code>([a-zA-Z0-9\s-]+?)<\/Code>/g;
-const messagePattern = /<Message>([a-zA-Z0-9\s-\.]+?)<\/Message>/g;
-/**
- * @param {string} responseText May or may not be XML
- */
-function tryProcessXMLError(responseText) {
-	// We don't have an XML parser in Node.js' std lib and we don't want to reference one for an optional diagnostic
-	// So... :hide-the-pain-harold:
-	return {
-		code: codePattern.exec(responseText)?.[1] ?? undefined,
-		message: messagePattern.exec(responseText)?.[1] ?? undefined,
-	};
+	res += `&X-Amz-SignedHeaders=${encodeURIComponent(headerList)}`;
+
+	if (storageClass) {
+		res += `&X-Amz-Storage-Class=${storageClass}`;
+	}
+	return res;
 }

package/src/S3Error.js

@@ -1,5 +1,3 @@
-// @ts-check
-
 export default class S3Error extends Error {
 	/**
 	 * @type {string}

package/src/S3File.js

@@ -1,5 +1,3 @@
-// @ts-check
-
 import S3Error from "./S3Error.js";
 import S3Stat from "./S3Stat.js";
 import { write, stream } from "./S3Client.js";

package/src/S3Stat.js

@@ -1,7 +1,3 @@
-// @ts-check
-
-import { inspect } from "node:util";
-
 export default class S3Stat {
 	/**
 	 * @type {string}
@@ -69,8 +65,4 @@ export default class S3Stat {
 
 		return new S3Stat(etag, new Date(lm), size, ct);
 	}
-
-	toString() {
-		return `S3Stats {\n\tlastModified: ${inspect(this.lastModified)},\n\tsize: ${inspect(this.size)},\n\ttype: ${inspect(this.type)},\n\tetag: ${inspect(this.etag)}\n}`;
-	}
 }

package/src/index.d.ts

@@ -1,9 +1,12 @@
 import type { Readable } from "node:stream";
 
+import type S3BucketEntry from "./S3BucketEntry.js";
+
 export { default as S3File } from "./S3File.js";
 export { default as S3Client } from "./S3Client.js";
 export { default as S3Error } from "./S3Error.js";
 export { default as S3Stat } from "./S3Stat.js";
+export { default as S3BucketEntry } from "./S3BucketEntry.js";
 
 export interface S3ClientOptions {
 	bucket: string;
@@ -37,7 +40,17 @@ export type StorageClass =
 	| "SNOW"
 	| "STANDARD_IA";
 
+export type ChecksumAlgorithm =
+	| "CRC32"
+	| "CRC32C"
+	| "CRC64NVME"
+	| "SHA1"
+	| "SHA256";
+
+export type ChecksumType = "COMPOSITE" | "FULL_OBJECT";
+
 export type PresignableHttpMethod = "GET" | "DELETE" | "PUT" | "HEAD";
+export type HttpMethod = PresignableHttpMethod | "POST"; // There are also others, but we don't want to support them yet
 
 export interface S3FilePresignOptions {
 	contentHash: Buffer;
@@ -78,3 +91,15 @@ export type ByteSource = UndiciBodyInit | Blob;
 // | Response
 // | S3File
 // | ReadableStream<Uint8Array>
+
+export type ListObjectsResponse = {
+	name: string;
+	prefix: string | undefined;
+	startAfter: string | undefined;
+	isTruncated: boolean;
+	continuationToken: string | undefined;
+	maxKeys: number;
+	keyCount: number;
+	nextContinuationToken: string | undefined;
+	contents: readonly S3BucketEntry[];
+};

package/src/sign.js

@@ -1,4 +1,3 @@
-// @ts-check
 import { createHmac, createHash } from "node:crypto";
 
 // Spec:
@@ -41,16 +40,12 @@ export function signCanonicalDataHash(
 	date,
 	region,
 ) {
-	// TODO: Investigate if its actually faster than just concatenating the parts and do a single update()
+	// it is actually faster to pass a single large string instead of doing multiple .update() chains with the parameters
+	// see `benchmark-operations.js`
 	return createHmac("sha256", signinKey)
-		.update("AWS4-HMAC-SHA256\n")
-		.update(date.dateTime)
-		.update("\n")
-		.update(date.date)
-		.update("/")
-		.update(region)
-		.update("/s3/aws4_request\n")
-		.update(canonicalDataHash)
+		.update(
+			`AWS4-HMAC-SHA256\n${date.dateTime}\n${date.date}/${region}/s3/aws4_request\n${canonicalDataHash}`,
+		)
 		.digest("hex");
 }
 
@@ -63,8 +58,6 @@ export const unsignedPayload = "UNSIGNED-PAYLOAD";
  * Used for pre-signing only. Pre-signed URLs [cannot contain content hashes](https://github.com/aws/aws-sdk-js/blob/966fa6c316dbb11ca9277564ff7120e6b16467f4/lib/signers/v4.js#L182-L183)
  * and the only header that is signed is `host`. So we can use an optimized version for that.
  *
- * TODO: Maybe passing a contentHash is supported on GET in order to restrict access to a specific file
- *
  * @param {import("./index.js").PresignableHttpMethod} method
  * @param {string} path
  * @param {string} query
@@ -72,21 +65,18 @@ export const unsignedPayload = "UNSIGNED-PAYLOAD";
  * @returns
  */
 export function createCanonicalDataDigestHostOnly(method, path, query, host) {
-	// TODO: Investigate if its actually faster than just concatenating the parts and do a single update()
+	// it is actually faster to pass a single large string instead of doing multiple .update() chains with the parameters
+	// see `benchmark-operations.js`
+
 	return createHash("sha256")
-		.update(method)
-		.update("\n")
-		.update(path)
-		.update("\n")
-		.update(query)
-		.update("\nhost:")
-		.update(host)
-		.update("\n\nhost\nUNSIGNED-PAYLOAD")
+		.update(
+			`${method}\n${path}\n${query}\nhost:${host}\n\nhost\nUNSIGNED-PAYLOAD`,
+		)
 		.digest("hex");
 }
 
 /**
- * @param {import("./index.js").PresignableHttpMethod} method
+ * @param {import("./index.js").HttpMethod} method
  * @param {string} path
  * @param {string} query
  * @param {Record<string, string>} sortedHeaders
@@ -100,31 +90,42 @@ export function createCanonicalDataDigest(
 	sortedHeaders,
 	contentHashStr,
 ) {
+	// Use this for debugging
+	/*
+	const xHash = {
+		h: createHash("sha256"),
+		m: "",
+		update(v) {
+			this.m += v;
+			this.h.update(v);
+			return this;
+		},
+		digest(v) {
+			if (this.m.includes("continuation-token")) console.log(this.m);
+			return this.h.digest(v);
+		},
+	};
+	*/
+
 	const sortedHeaderNames = Object.keys(sortedHeaders);
-	// TODO: Investigate if its actually faster than just concatenating the parts and do a single update()
-	const hash = createHash("sha256")
-		.update(method)
-		.update("\n")
-		.update(path)
-		.update("\n")
-		.update(query)
-		.update("\n");
+	// it is actually faster to pass a single large string instead of doing multiple .update() chains with the parameters
+	// see `benchmark-operations.js`
 
+	let canonData = `${method}\n${path}\n${query}\n`;
 	for (const header of sortedHeaderNames) {
-		hash.update(header).update(":").update(sortedHeaders[header]);
-		hash.update("\n");
+		canonData += `${header}:${sortedHeaders[header]}\n`;
 	}
+	canonData += "\n";
 
-	hash.update("\n");
-
-	for (let i = 0; i < sortedHeaderNames.length; ++i) {
-		hash.update(sortedHeaderNames[i]);
-		if (i < sortedHeaderNames.length - 1) {
-			hash.update(";");
-		}
+	// emit the first header without ";", so we can avoid branching inside the loop for the other headers
+	// this is just a version of `sortedHeaderList.join(";")` that seems about 2x faster (see `benchmark-operations.js`)
+	canonData += sortedHeaderNames.length > 0 ? sortedHeaderNames[0] : "";
+	for (let i = 1; i < sortedHeaderNames.length; ++i) {
+		canonData += `;${sortedHeaderNames[i]}`;
 	}
+	canonData += `\n${contentHashStr}`;
 
-	return hash.update("\n").update(contentHashStr).digest("hex");
+	return createHash("sha256").update(canonData).digest("hex");
 }
 
 /**
@@ -134,3 +135,11 @@ export function createCanonicalDataDigest(
 export function sha256(data) {
 	return createHash("sha256").update(data).digest();
 }
+
+/**
+ * @param {import("node:crypto").BinaryLike} data
+ * @returns {string}
+ */
+export function md5Hex(data) {
+	return createHash("md5").update(data).digest("hex");
+}

package/src/url.js

@@ -1,5 +1,3 @@
-// @ts-check
-
 /**
  * @param {string} endpoint
  * @param {string} bucket

package/src/test-common.js

@@ -1,94 +0,0 @@
-/**
- * @module Used by integration tests and unit tests.
- */
-
-// @ts-check
-import { test } from "node:test";
-import { expect } from "expect";
-
-import { S3Client } from "./index.js";
-
-/**
- * @param {number} runId
- * @param {string} endpoint
- * @param {boolean} forcePathStyle
- * @param {string} accessKeyId
- * @param {string} secretAccessKey
- * @param {string} region
- * @param {string} bucket
- */
-export function runTests(
-	runId,
-	endpoint,
-	forcePathStyle,
-	accessKeyId,
-	secretAccessKey,
-	region,
-	bucket,
-) {
-	const client = new S3Client({
-		endpoint,
-		accessKeyId,
-		secretAccessKey,
-		region,
-		bucket,
-	});
-
-	test("presign-put", async () => {
-		const testId = crypto.randomUUID();
-		const expected = {
-			hello: testId,
-		};
-
-		const url = client.presign(`${runId}/presign-test.json`, { method: "PUT" });
-		const res = await fetch(url, {
-			method: "PUT",
-			body: JSON.stringify(expected),
-			headers: {
-				accept: "application/json",
-			},
-		});
-		expect(res.ok).toBe(true);
-
-		const f = client.file(`${runId}/presign-test.json`);
-		try {
-			const actual = await f.json();
-			expect(actual).toStrictEqual(expected);
-		} finally {
-			await f.delete();
-		}
-	});
-
-	test("roundtrip", async () => {
-		const testId = crypto.randomUUID();
-		const f = client.file(`${runId}/roundtrip.txt`);
-		await f.write(testId);
-		try {
-			const stat = await f.stat();
-			expect(stat).toEqual(
-				expect.objectContaining({
-					size: testId.length,
-					type: "application/octet-stream",
-				}),
-			);
-
-			const actual = await f.text();
-			expect(actual).toStrictEqual(testId);
-		} finally {
-			await f.delete();
-		}
-	});
-
-	test("slicing", async () => {
-		const testId = crypto.randomUUID();
-		const f = client.file(`${runId}/slicing.txt`);
-		await f.write(testId);
-		try {
-			const slicedFile = f.slice(10, 20);
-			const s = await slicedFile.text();
-			expect(s).toEqual(testId.substring(10, 20));
-		} finally {
-			await f.delete();
-		}
-	});
-}