lean-s3 0.1.1

package/src/S3File.js

@@ -0,0 +1,293 @@
+// @ts-check
+
+import S3Error from "./S3Error.js";
+import S3Stat from "./S3Stat.js";
+import { write, stream } from "./S3Client.js";
+import { sha256 } from "./sign.js";
+import { Readable } from "node:stream";
+
+/** @typedef {import("./S3Client.js").default} S3Client */
+/** @typedef {import("./index.d.ts").S3ClientOptions} S3ClientOptions */
+/** @typedef {import("./index.d.ts").PresignableHttpMethod} PresignableHttpMethod */
+/** @typedef {import("./index.d.ts").StorageClass} StorageClass */
+/** @typedef {import("./index.d.ts").Acl} Acl */
+/** @typedef {import("./index.d.ts").S3FilePresignOptions} S3FilePresignOptions */
+/** @typedef {import("./index.d.ts").S3StatOptions} S3StatOptions */
+/** @typedef {import("./index.d.ts").S3FileExistsOptions} S3FileExistsOptions */
+/** @typedef {import("./index.d.ts").S3FileDeleteOptions} S3FileDeleteOptions */
+/** @typedef {import("./index.d.ts").ByteSource} ByteSource */
+
+// TODO: If we want to hack around, we can use this to access the private implementation of the "get stream" algorithm used by Node.js's blob internally
+// We probably have to do this some day if the fetch implementation is moved to internals.
+// If this happens, fetch will probably use `[kHandle].getReader()` instead of .stream() to read the Blob
+// This would break our use-case of passing an S3File as a body
+// Using this hack would also make `.text()`, `.bytes()` etc. "just work" in every case, since these use `[kHandle]` internally as well.
+// We now resort back into overriding text/bytes/etc. But as soon as another internal Node.js API uses this functionality, this would probably also use `[kHandle]` and bypass our data.
+// const kHandle = Object.getOwnPropertySymbols(new Blob).find(s => s.toString() === 'Symbol(kHandle)');
+export default class S3File {
+	/** @type {S3Client} */
+	#client;
+	/** @type {string} */
+	#path;
+	/** @type {number | undefined} */
+	#start;
+	/** @type {number | undefined} */
+	#end;
+	/** @type {string} */
+	#contentType;
+
+	/**
+	 * @internal
+	 * @param {S3Client} client
+	 * @param {string} path
+	 * @param {number | undefined} start
+	 * @param {number | undefined} end
+	 * @param {string | undefined} contentType
+	 */
+	constructor(client, path, start, end, contentType) {
+		if (typeof start === "number" && start < 0) {
+			throw new Error("Invalid slice `start`.");
+		}
+		if (
+			typeof end === "number" &&
+			(end < 0 || (typeof start === "number" && end < start))
+		) {
+			throw new Error("Invalid slice `end`.");
+		}
+
+		this.#client = client;
+		this.#path = path;
+		this.#start = start;
+		this.#end = end;
+		this.#contentType = contentType ?? "application/octet-stream";
+	}
+
+	// TODO: slice overloads
+	/**
+	 * @param {number | undefined} [start]
+	 * @param {number | undefined} [end]
+	 * @param {string | undefined} [contentType]
+	 * @returns {S3File}
+	 */
+	slice(start, end, contentType) {
+		return new S3File(
+			this.#client,
+			this.#path,
+			start ?? undefined,
+			end ?? undefined,
+			contentType ?? this.#contentType,
+		);
+	}
+
+	/**
+	 *  Get the stat of a file in the bucket. Uses `HEAD` request to check existence.
+	 *
+	 * @throws {Error} If the file does not exist.
+	 * @param {Partial<S3StatOptions>} [options]
+	 * @returns {Promise<S3Stat>}
+	 */
+	async stat({ signal } = {}) {
+		// TODO: Support all options
+
+		// TODO: Don't use presign here
+		const url = this.#client.presign(this.#path, { method: "HEAD" });
+		const response = await fetch(url, { method: "HEAD", signal }); // TODO: Use undici
+
+		if (!response.ok) {
+			switch (response.status) {
+				case 404:
+					// TODO: Process response body
+					throw new S3Error("NoSuchKey", this.#path);
+				default:
+					// TODO: Process response body
+					throw new S3Error("Unknown", this.#path);
+			}
+		}
+
+		const result = S3Stat.tryParseFromHeaders(response.headers);
+		if (!result) {
+			throw new Error("S3 server returned an invalid response for HEAD");
+		}
+		return result;
+	}
+	/**
+	 * Check if a file exists in the bucket. Uses `HEAD` request to check existence.
+	 * @param {Partial<S3FileExistsOptions>} [options]
+	 * @returns {Promise<boolean>}
+	 */
+	async exists({ signal } = {}) {
+		// TODO: Support all options
+
+		// TODO: Don't use presign here
+		const url = this.#client.presign(this.#path, { method: "HEAD" });
+		const res = await fetch(url, { method: "HEAD", signal }); // TODO: Use undici
+		return res.ok;
+	}
+
+	/**
+	 * Delete a file from the bucket.
+	 * @param {Partial<S3FileDeleteOptions>} [options]
+	 * @returns {Promise<void>}
+	 *
+	 * @example
+	 * ```js
+	 * // Simple delete
+	 * await client.unlink("old-file.txt");
+	 *
+	 * // With error handling
+	 * try {
+	 *   await client.unlink("file.dat");
+	 *   console.log("File deleted");
+	 * } catch (err) {
+	 *   console.error("Delete failed:", err);
+	 * }
+	 * ```
+	 */
+	async delete({ signal } = {}) {
+		// TODO: Support all options
+
+		// TODO: Don't use presign here
+		const url = this.#client.presign(this.#path, { method: "DELETE" });
+		const response = await fetch(url, { method: "DELETE", signal }); // TODO: Use undici
+		if (!response.ok) {
+			switch (response.status) {
+				case 404:
+					// TODO: Process response body
+					throw new S3Error("NoSuchKey", this.#path);
+				default:
+					// TODO: Process response body
+					throw new S3Error("Unknown", this.#path);
+			}
+		}
+	}
+
+	toString() {
+		return `S3File { path: "${this.#path}" }`;
+	}
+
+	/** @returns {Promise<unknown>} */
+	json() {
+		// Not using JSON.parse(await this.text()), so the env can parse json while loading
+		// Also, see TODO note above this class
+		return new Response(this.stream()).json();
+	}
+	// TODO
+	// /** @returns {Promise<Uint8Array>} */
+	// bytes() {
+	// 	return new Response(this.stream()).bytes(); // TODO: Does this exist?
+	// }
+	/** @returns {Promise<ArrayBuffer>} */
+	arrayBuffer() {
+		return new Response(this.stream()).arrayBuffer();
+	}
+	/** @returns {Promise<string>} */
+	text() {
+		return new Response(this.stream()).text();
+	}
+	/** @returns {Promise<Blob>} */
+	blob() {
+		return new Response(this.stream()).blob();
+	}
+
+	/** @returns {ReadableStream<Uint8Array>} */
+	stream() {
+		// This function is called for every operation on the blob
+		return this.#client[stream](this.#path, undefined, this.#start, this.#end);
+	}
+
+	/**
+	 * @param {ByteSource} data
+	 * @returns {Promise<[
+	 *   buffer: import("./index.d.ts").UndiciBodyInit,
+	 *   size: number | undefined,
+	 *   hash: Buffer | undefined,
+	 * ]>}
+	 */
+	async #transformData(data) {
+		if (typeof data === "string") {
+			const binary = new TextEncoder();
+			const bytes = binary.encode(data);
+			return [
+				bytes,
+				bytes.byteLength,
+				sha256(bytes), // TODO: Maybe use some streaming to compute hash while encoding?
+			];
+		}
+
+		if (data instanceof Blob) {
+			const bytes = await data.bytes();
+			return [
+				bytes,
+				bytes.byteLength,
+				sha256(bytes), // TODO: Maybe use some streaming to compute hash while encoding?
+			];
+		}
+
+		if (data instanceof Readable) {
+			return [data, undefined, undefined];
+		}
+
+		if (
+			data instanceof ArrayBuffer ||
+			data instanceof SharedArrayBuffer ||
+			ArrayBuffer.isView(data)
+		) {
+			// TODO: Support hashing
+			return [
+				data,
+				data.byteLength,
+				undefined, // TODO: Compute hash some day
+			];
+		}
+
+		assertNever(data);
+	}
+
+	/**
+	 * @param {ByteSource} data
+	 * @returns {Promise<void>}
+	 */
+	async write(data) {
+		/** @type {AbortSignal | undefined} */
+		const signal = undefined; // TODO: Take this as param
+
+		// TODO: Support S3File as input and maybe use CopyObject
+		// TODO: Support Request and Response as input?
+		const [bytes, length, hash] = await this.#transformData(data);
+		return await this.#client[write](
+			this.#path,
+			bytes,
+			this.#contentType,
+			length,
+			hash,
+			this.#start,
+			this.#end,
+			signal,
+		);
+	}
+
+	/*
+	// Future API?
+	/** @returns {WritableStream<ArrayBufferLike | ArrayBufferView>} *
+	writer() {
+		throw new Error("Not implemented");
+	}
+	// Future API?
+	/** @returns {Promise<void>} *
+	setTags() {
+		throw new Error("Not implemented");
+	}
+	/** @returns {Promise<unknown>} *
+	getTags() {
+		throw new Error("Not implemented");
+	}
+	*/
+}
+
+/**
+ * @param {never} v
+ * @returns {never}
+ */
+function assertNever(v) {
+	throw new TypeError(`Expected value not to have type ${typeof v}`);
+}

package/src/S3Stat.js

@@ -0,0 +1,76 @@
+// @ts-check
+
+import { inspect } from "node:util";
+
+export default class S3Stat {
+	/**
+	 * @type {string}
+	 * @readonly
+	 */
+	etag;
+	/**
+	 * @type {Date}
+	 * @readonly
+	 */
+	lastModified;
+	/**
+	 * @type {number}
+	 * @readonly
+	 */
+	size;
+	/**
+	 * @type {string}
+	 * @readonly
+	 */
+	type;
+
+	/**
+	 * @param {string} etag
+	 * @param {Date} lastModified
+	 * @param {number} size
+	 * @param {string} type
+	 */
+	constructor(etag, lastModified, size, type) {
+		this.etag = etag;
+		this.lastModified = lastModified;
+		this.size = size;
+		this.type = type;
+	}
+
+	/**
+	 * @param {Headers} headers
+	 * @returns {S3Stat | undefined}
+	 */
+	static tryParseFromHeaders(headers) {
+		const lm = headers.get("last-modified");
+		if (lm === null) {
+			return undefined;
+		}
+
+		const etag = headers.get("etag");
+		if (etag === null) {
+			return undefined;
+		}
+
+		const cl = headers.get("content-length");
+		if (cl === null) {
+			return undefined;
+		}
+
+		const size = Number(cl);
+		if (!Number.isSafeInteger(size)) {
+			return undefined;
+		}
+
+		const ct = headers.get("content-type");
+		if (ct === null) {
+			return undefined;
+		}
+
+		return new S3Stat(etag, new Date(lm), size, ct);
+	}
+
+	toString() {
+		return `S3Stats {\n\tlastModified: ${inspect(this.lastModified)},\n\tsize: ${inspect(this.size)},\n\ttype: ${inspect(this.type)},\n\tetag: ${inspect(this.etag)}\n}`;
+	}
+}

package/src/index.d.ts

@@ -0,0 +1,80 @@
+import type { Readable } from "node:stream";
+
+export { default as S3File } from "./S3File.js";
+export { default as S3Client } from "./S3Client.js";
+export { default as S3Error } from "./S3Error.js";
+export { default as S3Stat } from "./S3Stat.js";
+
+export interface S3ClientOptions {
+	bucket: string;
+	region: string;
+	endpoint: string;
+	accessKeyId: string;
+	secretAccessKey: string;
+	sessionToken?: string;
+}
+
+export type Acl =
+	| "private"
+	| "public-read"
+	| "public-read-write"
+	| "aws-exec-read"
+	| "authenticated-read"
+	| "bucket-owner-read"
+	| "bucket-owner-full-control"
+	| "log-delivery-write";
+
+export type StorageClass =
+	| "STANDARD"
+	| "DEEP_ARCHIVE"
+	| "EXPRESS_ONEZONE"
+	| "GLACIER"
+	| "GLACIER_IR"
+	| "INTELLIGENT_TIERING"
+	| "ONEZONE_IA"
+	| "OUTPOSTS"
+	| "REDUCED_REDUNDANCY"
+	| "SNOW"
+	| "STANDARD_IA";
+
+export type PresignableHttpMethod = "GET" | "DELETE" | "PUT" | "HEAD";
+
+export interface S3FilePresignOptions {
+	contentHash: Buffer;
+	/** Seconds. */
+	expiresIn: number; // TODO: Maybe support Temporal.Duration once major support arrives
+	method: PresignableHttpMethod;
+	storageClass: StorageClass;
+	acl: Acl;
+}
+
+export type OverridableS3ClientOptions = Pick<
+	S3ClientOptions,
+	"region" | "bucket" | "endpoint"
+>;
+
+export interface S3StatOptions extends OverridableS3ClientOptions {
+	signal: AbortSignal;
+}
+export interface S3FileExistsOptions extends OverridableS3ClientOptions {
+	signal: AbortSignal;
+}
+export interface S3FileDeleteOptions extends OverridableS3ClientOptions {
+	signal: AbortSignal;
+}
+
+// biome-ignore lint/complexity/noBannedTypes: TODO
+export type CreateFileInstanceOptions = {}; // TODO
+
+/** Body values supported by undici. */
+export type UndiciBodyInit = string | Buffer | Uint8Array | Readable;
+
+export type ByteSource = UndiciBodyInit | Blob;
+// TODO
+// | ArrayBufferView
+// | ArrayBuffer
+// | SharedArrayBuffer
+// | Request
+// | Response
+// | S3File
+// | ReadableStream<Uint8Array>

package/src/index.js

@@ -0,0 +1,4 @@
+export { default as S3Client } from "./S3Client.js";
+export { default as S3File } from "./S3File.js";
+export { default as S3Error } from "./S3Error.js";
+export { default as S3Stat } from "./S3Stat.js";

package/src/sign.js

@@ -0,0 +1,136 @@
+// @ts-check
+import { createHmac, createHash } from "node:crypto";
+
+// Spec:
+// https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html
+
+/**
+ * @param {string} date
+ * @param {string} region
+ * @param {string} secretAccessKey
+ * @returns {Buffer}
+ */
+export function deriveSigningKey(date, region, secretAccessKey) {
+	const key = `AWS4${secretAccessKey}`;
+
+	const signedDate = createHmac("sha256", key).update(date).digest();
+
+	const signedDateRegion = createHmac("sha256", signedDate)
+		.update(region)
+		.digest();
+
+	const signedDateRegionService = createHmac("sha256", signedDateRegion)
+		.update("s3")
+		.digest();
+
+	return createHmac("sha256", signedDateRegionService)
+		.update("aws4_request")
+		.digest();
+}
+
+/**
+ * @param {Buffer} signinKey
+ * @param {string} canonicalDataHash
+ * @param {import("./AmzDate.js").AmzDate} date
+ * @param {string} region
+ * @returns {string}
+ */
+export function signCanonicalDataHash(
+	signinKey,
+	canonicalDataHash,
+	date,
+	region,
+) {
+	// TODO: Investigate if its actually faster than just concatenating the parts and do a single update()
+	return createHmac("sha256", signinKey)
+		.update("AWS4-HMAC-SHA256\n")
+		.update(date.dateTime)
+		.update("\n")
+		.update(date.date)
+		.update("/")
+		.update(region)
+		.update("/s3/aws4_request\n")
+		.update(canonicalDataHash)
+		.digest("hex");
+}
+
+export const unsignedPayload = "UNSIGNED-PAYLOAD";
+
+/**
+ *
+ * Same as {@see createCanonicalDataDigest}, but only sets the `host` header and the content hash to `UNSIGNED-PAYLOAD`.
+ *
+ * Used for pre-signing only. Pre-signed URLs [cannot contain content hashes](https://github.com/aws/aws-sdk-js/blob/966fa6c316dbb11ca9277564ff7120e6b16467f4/lib/signers/v4.js#L182-L183)
+ * and the only header that is signed is `host`. So we can use an optimized version for that.
+ *
+ * TODO: Maybe passing a contentHash is supported on GET in order to restrict access to a specific file
+ *
+ * @param {import("./index.js").PresignableHttpMethod} method
+ * @param {string} path
+ * @param {string} query
+ * @param {string} host
+ * @returns
+ */
+export function createCanonicalDataDigestHostOnly(method, path, query, host) {
+	// TODO: Investigate if its actually faster than just concatenating the parts and do a single update()
+	return createHash("sha256")
+		.update(method)
+		.update("\n")
+		.update(path)
+		.update("\n")
+		.update(query)
+		.update("\nhost:")
+		.update(host)
+		.update("\n\nhost\nUNSIGNED-PAYLOAD")
+		.digest("hex");
+}
+
+/**
+ * @param {import("./index.js").PresignableHttpMethod} method
+ * @param {string} path
+ * @param {string} query
+ * @param {Record<string, string>} sortedHeaders
+ * @param {string} contentHashStr
+ * @returns
+ */
+export function createCanonicalDataDigest(
+	method,
+	path,
+	query,
+	sortedHeaders,
+	contentHashStr,
+) {
+	const sortedHeaderNames = Object.keys(sortedHeaders);
+	// TODO: Investigate if its actually faster than just concatenating the parts and do a single update()
+	const hash = createHash("sha256")
+		.update(method)
+		.update("\n")
+		.update(path)
+		.update("\n")
+		.update(query)
+		.update("\n");
+
+	for (const header of sortedHeaderNames) {
+		hash.update(header).update(":").update(sortedHeaders[header]);
+		hash.update("\n");
+	}
+
+	hash.update("\n");
+
+	for (let i = 0; i < sortedHeaderNames.length; ++i) {
+		hash.update(sortedHeaderNames[i]);
+		if (i < sortedHeaderNames.length - 1) {
+			hash.update(";");
+		}
+	}
+
+	return hash.update("\n").update(contentHashStr).digest("hex");
+}
+
+/**
+ * @param {import("node:crypto").BinaryLike} data
+ * @returns {Buffer}
+ */
+export function sha256(data) {
+	return createHash("sha256").update(data).digest();
+}

package/src/test-common.js

@@ -0,0 +1,94 @@
+/**
+ * @module Used by integration tests and unit tests.
+ */
+
+// @ts-check
+import { test } from "node:test";
+import { expect } from "expect";
+
+import { S3Client } from "./index.js";
+
+/**
+ * @param {number} runId
+ * @param {string} endpoint
+ * @param {boolean} forcePathStyle
+ * @param {string} accessKeyId
+ * @param {string} secretAccessKey
+ * @param {string} region
+ * @param {string} bucket
+ */
+export function runTests(
+	runId,
+	endpoint,
+	forcePathStyle,
+	accessKeyId,
+	secretAccessKey,
+	region,
+	bucket,
+) {
+	const client = new S3Client({
+		endpoint,
+		accessKeyId,
+		secretAccessKey,
+		region,
+		bucket,
+	});
+
+	test("presign-put", async () => {
+		const testId = crypto.randomUUID();
+		const expected = {
+			hello: testId,
+		};
+
+		const url = client.presign(`${runId}/presign-test.json`, { method: "PUT" });
+		const res = await fetch(url, {
+			method: "PUT",
+			body: JSON.stringify(expected),
+			headers: {
+				accept: "application/json",
+			},
+		});
+		expect(res.ok).toBe(true);
+
+		const f = client.file(`${runId}/presign-test.json`);
+		try {
+			const actual = await f.json();
+			expect(actual).toStrictEqual(expected);
+		} finally {
+			await f.delete();
+		}
+	});
+
+	test("roundtrip", async () => {
+		const testId = crypto.randomUUID();
+		const f = client.file(`${runId}/roundtrip.txt`);
+		await f.write(testId);
+		try {
+			const stat = await f.stat();
+			expect(stat).toEqual(
+				expect.objectContaining({
+					size: testId.length,
+					type: "application/octet-stream",
+				}),
+			);
+
+			const actual = await f.text();
+			expect(actual).toStrictEqual(testId);
+		} finally {
+			await f.delete();
+		}
+	});
+
+	test("slicing", async () => {
+		const testId = crypto.randomUUID();
+		const f = client.file(`${runId}/slicing.txt`);
+		await f.write(testId);
+		try {
+			const slicedFile = f.slice(10, 20);
+			const s = await slicedFile.text();
+			expect(s).toEqual(testId.substring(10, 20));
+		} finally {
+			await f.delete();
+		}
+	});
+}