@appsemble/node-utils 0.38.1 → 0.39.0

package/README.md

@@ -1,9 +1,9 @@
-# ![](https://gitlab.com/appsemble/appsemble/-/raw/0.38.1/config/assets/logo.svg) Appsemble Node Utilities
+# ![](https://gitlab.com/appsemble/appsemble/-/raw/0.39.0/config/assets/logo.svg) Appsemble Node Utilities
 
 > NodeJS utilities used by Appsemble internally.
 
 [![npm](https://img.shields.io/npm/v/@appsemble/node-utils)](https://www.npmjs.com/package/@appsemble/node-utils)
-[![GitLab CI](https://gitlab.com/appsemble/appsemble/badges/0.38.1/pipeline.svg)](https://gitlab.com/appsemble/appsemble/-/releases/0.38.1)
+[![GitLab CI](https://gitlab.com/appsemble/appsemble/badges/0.39.0/pipeline.svg)](https://gitlab.com/appsemble/appsemble/-/releases/0.39.0)
 [![Prettier](https://img.shields.io/badge/code_style-prettier-ff69b4.svg)](https://prettier.io)
 
 ## Table of Contents
@@ -26,5 +26,5 @@ compatibility is not guaranteed.
 
 ## License
 
-[LGPL-3.0-only](https://gitlab.com/appsemble/appsemble/-/blob/0.38.1/LICENSE.md) ©
+[LGPL-3.0-only](https://gitlab.com/appsemble/appsemble/-/blob/0.39.0/LICENSE.md) ©
 [Appsemble](https://appsemble.com)

package/assetCssURL.d.ts

@@ -1,5 +1,9 @@
 /**
- * Resolve `asset()` functions and app asset URLs in CSS to absolute app asset URLs.
+ * Resolve `asset()` utilities and app asset URLs in CSS to absolute app asset URLs.
+ *
+ * An `asset()` utility takes a single string, holding either an asset reference or an app asset
+ * endpoint path. Both it and `url()` values addressing an app asset endpoint are replaced with a
+ * `url()` value addressing the given app on the given host. All other values are left untouched.
  *
  * @param css The CSS to process.
  * @param appId The id of the app the assets belong to.

package/assetCssURL.js

@@ -1,96 +1,194 @@
-const assetUrlMatcher = /url\(\s*asset\(\s*(?:("|')(.*?)\1|([^\s"'()][^)]*))\s*\)\s*\)/g;
-const standaloneAssetMatcher = /asset\(\s*(?:("|')(.*?)\1|([^\s"'()][^)]*))\s*\)/g;
-const urlMatcher = /url\(\s*(?:("|')(.*?)\1|([^\s"'()][^)]*))\s*\)/g;
-const appAssetPathMatcher = /^\/api\/apps\/\d+\/assets\/(.+)$/;
-function hasRejectedTokens(value) {
-    let currentValue = value;
-    for (let i = 0; i < 3; i += 1) {
-        if (currentValue.includes('..') || /%2f|%5c/i.test(currentValue)) {
-            return true;
-        }
-        if (!currentValue.includes('%')) {
-            return false;
-        }
-        try {
-            const decodedValue = decodeURIComponent(currentValue);
-            if (decodedValue === currentValue) {
-                return false;
-            }
-            if (decodedValue.includes('/') || decodedValue.includes('\\')) {
-                return true;
-            }
-            currentValue = decodedValue;
-        }
-        catch {
-            return true;
-        }
-    }
-    return false;
+import { normalized, StyleValidationError, uuid4Pattern } from '@appsemble/utils';
+import { parse, walk } from 'css-tree';
+const appIdPattern = /^\d+$/;
+/**
+ * Create a replacement of the source a node was parsed from.
+ *
+ * @param node The node whose source to replace. It must have been parsed with positions.
+ * @param offset The offset of the source the node was parsed from within the stylesheet.
+ * @param value The value to replace the source of the node with.
+ * @returns The replacement.
+ */
+function replaceNode(node, offset, value) {
+    return {
+        start: offset + node.loc.start.offset,
+        end: offset + node.loc.end.offset,
+        value,
+    };
 }
-function rewriteAppAssetURL(url, appId, host) {
-    const trimmedUrl = url.trim();
-    if (trimmedUrl.startsWith('/')) {
-        const appAssetPathMatch = appAssetPathMatcher.exec(trimmedUrl);
-        if (!appAssetPathMatch) {
-            return trimmedUrl;
-        }
-        const appAssetPath = appAssetPathMatch[1];
-        if (hasRejectedTokens(appAssetPath)) {
-            return trimmedUrl;
-        }
-        return String(new URL(`/api/apps/${appId}/assets/${appAssetPath}`, host));
-    }
+/**
+ * Parse the value of a declaration whose value the stylesheet parser left raw.
+ *
+ * Custom properties hold an arbitrary token stream, so their value is never parsed as part of the
+ * stylesheet.
+ *
+ * @param value The raw value to parse.
+ * @returns The parsed value, or `null` if it is not a valid declaration value.
+ */
+function parseRawValue(value) {
     try {
-        const parsedUrl = new URL(trimmedUrl, host);
-        const appAssetPathMatch = appAssetPathMatcher.exec(parsedUrl.pathname);
-        if (!appAssetPathMatch) {
-            return trimmedUrl;
-        }
-        const appAssetPath = appAssetPathMatch[1];
-        if (hasRejectedTokens(appAssetPath)) {
-            return trimmedUrl;
-        }
-        const rewrittenUrl = new URL(`/api/apps/${appId}/assets/${appAssetPath}`, host);
-        rewrittenUrl.search = parsedUrl.search;
-        rewrittenUrl.hash = parsedUrl.hash;
-        return String(rewrittenUrl);
+        return parse(value, { context: 'value', positions: true });
     }
     catch {
-        return trimmedUrl;
+        return null;
     }
 }
-function resolveAssetURL(appId, assetId, host) {
-    if (assetId.startsWith('data:')) {
-        return assetId;
+function isValidAssetReference(reference) {
+    return normalized.test(reference) || uuid4Pattern.test(reference);
+}
+/**
+ * Split a URL value into the part that addresses a resource and the query and fragment that follow.
+ *
+ * @param value The URL value to split.
+ * @returns The path and the query and fragment suffix.
+ */
+function splitURLParts(value) {
+    const suffixIndex = Math.min(...['?', '#'].map((separator) => {
+        const index = value.indexOf(separator);
+        return index === -1 ? value.length : index;
+    }));
+    return { path: value.slice(0, suffixIndex), suffix: value.slice(suffixIndex) };
+}
+/**
+ * Extract the asset reference addressed by an app asset endpoint path.
+ *
+ * @param path The path to inspect.
+ * @returns The asset reference, or `null` if the path is not an app asset endpoint path.
+ */
+function getAppAssetPathReference(path) {
+    const [empty, api, apps, appIdSegment, assets, reference, ...rest] = path.split('/');
+    if (rest.length || empty !== '' || api !== 'api' || apps !== 'apps' || assets !== 'assets') {
+        return null;
     }
-    if (assetId.startsWith('/')) {
-        const appAssetPathMatch = appAssetPathMatcher.exec(assetId.trim());
-        if (appAssetPathMatch && hasRejectedTokens(appAssetPathMatch[1])) {
-            return null;
+    if (!appIdPattern.test(appIdSegment) || !isValidAssetReference(reference)) {
+        return null;
+    }
+    return reference;
+}
+/**
+ * Create the app asset URL addressing an asset, to be embedded in a single quoted CSS string.
+ *
+ * The query and fragment are taken from the reference the asset was addressed with, so they may
+ * hold characters that end the CSS string. `URL` strips or encodes all of those except the quote
+ * and the backslash, which are percent encoded here.
+ *
+ * @param reference The reference of the asset to address.
+ * @param suffix The query and fragment to append to the URL.
+ * @param appId The id of the app the asset belongs to.
+ * @param host The host on which the app asset endpoints are available.
+ * @returns The app asset URL.
+ */
+function createAppAssetURL(reference, suffix, appId, host) {
+    const path = `/api/apps/${appId}/assets/${encodeURIComponent(reference)}${suffix}`;
+    return String(new URL(path, host)).replaceAll('\\', '%5C').replaceAll("'", '%27');
+}
+/**
+ * Resolve the contents of an `asset()` utility to an app asset URL.
+ *
+ * @param value The asset reference or app asset endpoint path the utility was called with.
+ * @param appId The id of the app the asset belongs to.
+ * @param host The host on which the app asset endpoints are available.
+ * @returns The app asset URL the utility resolves to.
+ */
+function resolveAssetReference(value, appId, host) {
+    if (value.startsWith('/')) {
+        const { path, suffix } = splitURLParts(value);
+        const reference = getAppAssetPathReference(path);
+        if (reference) {
+            return createAppAssetURL(reference, suffix, appId, host);
         }
-        return rewriteAppAssetURL(assetId, appId, host);
     }
-    if (/^https?:\/\//.test(assetId)) {
-        const trimmedAssetId = assetId.trim();
-        try {
-            const parsedUrl = new URL(trimmedAssetId, host);
-            const appAssetPathMatch = appAssetPathMatcher.exec(parsedUrl.pathname);
-            if (appAssetPathMatch && hasRejectedTokens(appAssetPathMatch[1])) {
-                return null;
-            }
+    else if (isValidAssetReference(value)) {
+        return createAppAssetURL(value, '', appId, host);
+    }
+    throw new StyleValidationError(`Invalid asset reference: ${value}`);
+}
+/**
+ * Rewrite an app asset URL so it addresses the given app on the given host.
+ *
+ * Stylesheets are stored with their asset references already resolved to absolute URLs, so a
+ * stylesheet copied from another app addresses that app on this host. Such URLs are rewritten as
+ * well, unlike URLs addressing another host.
+ *
+ * @param value The URL to rewrite.
+ * @param appId The id of the app the asset belongs to.
+ * @param host The host on which the app asset endpoints are available.
+ * @returns The rewritten URL, or `null` if the value does not address an app asset.
+ */
+function rewriteAppAssetURL(value, appId, host) {
+    let path;
+    let suffix;
+    if (value.startsWith('/')) {
+        ({ path, suffix } = splitURLParts(value));
+    }
+    else {
+        if (!URL.canParse(value)) {
+            return null;
         }
-        catch {
+        const url = new URL(value);
+        if (url.origin !== new URL(host).origin) {
             return null;
         }
-        return rewriteAppAssetURL(assetId, appId, host);
+        path = url.pathname;
+        suffix = `${url.search}${url.hash}`;
     }
-    if (hasRejectedTokens(assetId)) {
-        return null;
+    const reference = getAppAssetPathReference(path);
+    return reference ? createAppAssetURL(reference, suffix, appId, host) : null;
+}
+/**
+ * Resolve all app asset references in CSS to absolute app asset URLs.
+ *
+ * @param css The CSS to process.
+ * @param appId The id of the app the assets belong to.
+ * @param host The host on which the app asset endpoints are available.
+ * @returns The CSS with all app asset references resolved.
+ */
+function resolveAssetReferences(css, appId, host) {
+    const replacements = [];
+    /**
+     * Collect the replacements of all app asset references in a tree.
+     *
+     * @param node The tree to collect replacements from.
+     * @param offset The offset of the source the tree was parsed from within the stylesheet.
+     */
+    function collectReplacements(node, offset) {
+        walk(node, (child) => {
+            if (child.type === 'Function' && child.name.toLowerCase() === 'asset') {
+                const [argument, ...rest] = child.children;
+                if (rest.length || argument?.type !== 'String') {
+                    throw new StyleValidationError('The asset utility takes a single string');
+                }
+                replacements.push(replaceNode(child, offset, `url('${resolveAssetReference(argument.value, appId, host)}')`));
+                return;
+            }
+            if (child.type === 'Url') {
+                const rewrittenURL = rewriteAppAssetURL(child.value, appId, host);
+                if (rewrittenURL) {
+                    replacements.push(replaceNode(child, offset, `url('${rewrittenURL}')`));
+                }
+                return;
+            }
+            if (child.type === 'Raw') {
+                const value = parseRawValue(child.value);
+                if (value) {
+                    collectReplacements(value, offset + child.loc.start.offset);
+                }
+            }
+        });
+    }
+    collectReplacements(parse(css, { positions: true }), 0);
+    let result = css;
+    for (const replacement of replacements.reverse()) {
+        result = result.slice(0, replacement.start) + replacement.value + result.slice(replacement.end);
     }
-    return String(new URL(`/api/apps/${appId}/assets/${assetId}`, host));
+    return result;
 }
 /**
- * Resolve `asset()` functions and app asset URLs in CSS to absolute app asset URLs.
+ * Resolve `asset()` utilities and app asset URLs in CSS to absolute app asset URLs.
+ *
+ * An `asset()` utility takes a single string, holding either an asset reference or an app asset
+ * endpoint path. Both it and `url()` values addressing an app asset endpoint are replaced with a
+ * `url()` value addressing the given app on the given host. All other values are left untouched.
  *
  * @param css The CSS to process.
  * @param appId The id of the app the assets belong to.
@@ -98,37 +196,6 @@ function resolveAssetURL(appId, assetId, host) {
  * @returns The CSS with all app asset references resolved.
  */
 export function replaceAssetFunctions(css, appId, host) {
-    if (!appId) {
-        return css;
-    }
-    const replaceAssetFunction = (match, ...args) => {
-        const quotedAssetId = args[1];
-        const unquotedAssetId = args[2];
-        const assetId = (quotedAssetId || unquotedAssetId)?.trim();
-        if (!assetId) {
-            return match;
-        }
-        const resolvedAssetUrl = resolveAssetURL(appId, assetId, host);
-        if (!resolvedAssetUrl) {
-            return match;
-        }
-        return `url('${resolvedAssetUrl}')`;
-    };
-    const cssWithResolvedAssetFunctionURLs = css
-        .replaceAll(assetUrlMatcher, replaceAssetFunction)
-        .replaceAll(standaloneAssetMatcher, replaceAssetFunction);
-    return cssWithResolvedAssetFunctionURLs.replaceAll(urlMatcher, (match, ...args) => {
-        const quotedUrl = args[1];
-        const unquotedUrl = args[2];
-        const url = (quotedUrl || unquotedUrl)?.trim();
-        if (!url) {
-            return match;
-        }
-        const rewrittenUrl = rewriteAppAssetURL(url, appId, host);
-        if (rewrittenUrl === url) {
-            return match;
-        }
-        return `url('${rewrittenUrl}')`;
-    });
+    return appId ? resolveAssetReferences(css, appId, host) : css;
 }
 //# sourceMappingURL=assetCssURL.js.map

package/assets.d.ts

@@ -1,6 +1,6 @@
 import { type Context } from 'koa';
-import { type BucketItemStat } from 'minio';
-export declare function setAssetHeaders(ctx: Context, mime: string, filename: string | null, stats?: BucketItemStat): void;
+import { type S3FileStats } from './s3.js';
+export declare function setAssetHeaders(ctx: Context, mime: string, filename: string | null, stats?: S3FileStats): void;
 interface FileMeta {
     filename?: string;
     mime: string;

package/assets.js

@@ -1,7 +1,7 @@
 import { createReadStream } from 'node:fs';
 import { stat } from 'node:fs/promises';
 import { logger } from './logger.js';
-import { uploadS3File } from './s3.js';
+import { getAppAssetLocation, uploadS3File } from './s3.js';
 import { removeUploads } from './uploads.js';
 export function setAssetHeaders(ctx, mime, filename, stats) {
     ctx.set('content-type', mime || 'application/octet-stream');
@@ -24,7 +24,8 @@ export async function uploadAsset(appId, asset) {
     try {
         const stats = await stat(path);
         const stream = createReadStream(path);
-        await uploadS3File(`app-${appId}`, id, stream, stats.size);
+        const { bucket, key } = getAppAssetLocation(appId, id);
+        await uploadS3File(bucket, key, stream, stats.size);
     }
     catch (error) {
         logger.error(error);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@appsemble/node-utils",
-  "version": "0.38.1",
+  "version": "0.39.0",
   "description": "NodeJS utilities used by Appsemble internally.",
   "keywords": [
     "app",
@@ -40,9 +40,11 @@
     "test": "vitest"
   },
   "dependencies": {
-    "@appsemble/lang-sdk": "0.38.1",
-    "@appsemble/types": "0.38.1",
-    "@appsemble/utils": "0.38.1",
+    "@appsemble/lang-sdk": "0.39.0",
+    "@appsemble/types": "0.39.0",
+    "@appsemble/utils": "0.39.0",
+    "@aws-sdk/client-s3": "3.1130.0",
+    "@aws-sdk/lib-storage": "3.1130.0",
     "@formatjs/fast-memoize": "^2.0.0",
     "@fortawesome/fontawesome-free": "^6.0.0",
     "@inquirer/prompts": "^8.0.0",
@@ -85,7 +87,6 @@
     "logform": "^2.0.0",
     "memfs": "4.68.2",
     "mime-types": "^2.0.0",
-    "minio": "^8.0.7",
     "mustache": "^4.0.0",
     "openapi-types": "^12.1.3",
     "parse-duration": "^1.0.0",

package/readAsset.js

@@ -1,6 +1,6 @@
 import { readFile } from 'node:fs/promises';
 export const assetDir = new URL('assets/', import.meta.url);
 export function readAsset(filename, encoding) {
-    return readFile(new URL(filename, assetDir), encoding);
+    return readFile(new URL(filename, assetDir), encoding ?? null);
 }
 //# sourceMappingURL=readAsset.js.map

package/s3.d.ts

@@ -1,27 +1,58 @@
 import { Readable } from 'node:stream';
-import { type BucketItemStat } from 'minio';
-export interface S3FileReference {
-    etag: string;
+export declare const blockAssetsBucketName = "appsemble-block-assets";
+export interface S3Location {
+    bucket: string;
     key: string;
+}
+export interface S3FileStats {
+    etag: string;
     lastModified: Date;
-    metadata: BucketItemStat['metaData'];
+    metadata: Record<string, string>;
     size: number;
 }
+export interface S3FileReference extends S3FileStats {
+    key: string;
+}
 export interface InitS3ClientParams {
     endPoint: string;
     port?: number;
     useSSL?: boolean;
     accessKey: string;
     secretKey: string;
+    region?: string;
+    pathStyle?: boolean;
+    /**
+     * The single, pre-provisioned bucket that holds all objects.
+     *
+     * When set, app assets live under `apps/<appId>/` and block assets under `blocks/` in this
+     * bucket, and buckets are never created or listed. When unset, every app gets its own
+     * `app-<appId>` bucket and block assets live in the `appsemble-block-assets` bucket, both created
+     * on demand.
+     */
+    bucket?: string;
 }
-export declare function initS3Client({ accessKey, endPoint, port, secretKey, useSSL, }: InitS3ClientParams): void;
+export declare function initS3Client({ accessKey, bucket, endPoint, pathStyle, port, region, secretKey, useSSL, }: InitS3ClientParams): void;
+/**
+ * @returns The configured single bucket, or `undefined` in the bucket-per-app layout.
+ */
+export declare function getS3Bucket(): string | undefined;
+export declare function getAppAssetLocation(appId: number, assetId: string): S3Location;
+export declare function getBlockAssetLocation(storageKey: string): S3Location;
+export declare function isS3ErrorCode(error: unknown, code: string): boolean;
 export declare function uploadS3File(bucket: string, key: string, content: Buffer | Readable | string, size?: number, metadata?: Record<string, string>): Promise<void>;
 export declare function uploadS3FileFromPath(bucket: string, key: string, path: string): Promise<void>;
 export declare function getS3File(bucket: string, key: string): Promise<Readable>;
 export declare function getS3FileBuffer(bucket: string, key: string): Promise<Buffer>;
-export declare function getS3FileStats(bucket: string, key: string): Promise<BucketItemStat>;
-export declare function listS3Files(bucket: string): Promise<S3FileReference[]>;
+export declare function getS3FileStats(bucket: string, key: string): Promise<S3FileStats>;
+export declare function listS3Files(bucket: string, prefix?: string): Promise<S3FileReference[]>;
 export declare function setS3BucketPolicy(bucket: string, policy: string): Promise<void>;
 export declare function deleteS3Files(bucket: string, keys: string[]): Promise<void>;
 export declare function deleteS3File(bucket: string, key: string): Promise<void>;
+export declare function deleteAppAssetObjects(appId: number, assetIds: string[]): Promise<void>;
+/**
+ * Remove every object this client can reach.
+ *
+ * In the single-bucket layout this empties the configured bucket. In the bucket-per-app layout
+ * this empties and removes every bucket.
+ */
 export declare function clearAllS3Buckets(): Promise<void>;

package/s3.js

@@ -1,85 +1,149 @@
+import { createReadStream } from 'node:fs';
+import { stat } from 'node:fs/promises';
 import { Readable } from 'node:stream';
 import { buffer as streamToBuffer } from 'node:stream/consumers';
-import { Client, S3Error } from 'minio';
+import { CreateBucketCommand, DeleteBucketCommand, DeleteObjectsCommand, GetObjectCommand, HeadBucketCommand, HeadObjectCommand, ListBucketsCommand, ListObjectsV2Command, PutBucketPolicyCommand, PutObjectCommand, S3Client, S3ServiceException, } from '@aws-sdk/client-s3';
+import { Upload } from '@aws-sdk/lib-storage';
 import { logger } from './logger.js';
 let s3Client;
-export function initS3Client({ accessKey, endPoint, port = 9000, secretKey, useSSL = true, }) {
+let s3Bucket;
+let s3Region;
+export const blockAssetsBucketName = 'appsemble-block-assets';
+// S3 limits the number of keys in a single DeleteObjects request.
+const deleteBatchSize = 1000;
+export function initS3Client({ accessKey, bucket, endPoint, pathStyle = true, port = 9000, region = 'us-east-1', secretKey, useSSL = true, }) {
     try {
-        s3Client = new Client({
-            endPoint,
-            port,
-            useSSL,
-            accessKey,
-            secretKey,
+        s3Client = new S3Client({
+            endpoint: `${useSSL ? 'https' : 'http'}://${endPoint}:${port}`,
+            region,
+            forcePathStyle: pathStyle,
+            credentials: { accessKeyId: accessKey, secretAccessKey: secretKey },
+            // Only add checksums where S3 requires them, so requests stay compatible with
+            // S3-compatible stores that do not implement flexible checksums.
+            requestChecksumCalculation: 'WHEN_REQUIRED',
+            responseChecksumValidation: 'WHEN_REQUIRED',
         });
+        s3Bucket = bucket || undefined;
+        s3Region = region;
     }
     catch (error) {
         logger.error(error);
         throw error;
     }
 }
+/**
+ * @returns The configured single bucket, or `undefined` in the bucket-per-app layout.
+ */
+export function getS3Bucket() {
+    return s3Bucket;
+}
+function getAppAssetsBucket(appId) {
+    return s3Bucket ?? `app-${appId}`;
+}
+export function getAppAssetLocation(appId, assetId) {
+    return {
+        bucket: getAppAssetsBucket(appId),
+        key: s3Bucket ? `apps/${appId}/${assetId}` : assetId,
+    };
+}
+export function getBlockAssetLocation(storageKey) {
+    return s3Bucket
+        ? { bucket: s3Bucket, key: `blocks/${storageKey}` }
+        : { bucket: blockAssetsBucketName, key: storageKey };
+}
+export function isS3ErrorCode(error, code) {
+    return error instanceof S3ServiceException && error.name === code;
+}
 async function ensureBucket(name) {
+    if (s3Bucket) {
+        return;
+    }
     try {
-        const bucketExists = await s3Client.bucketExists(name);
-        if (!bucketExists) {
-            try {
-                await s3Client.makeBucket(name);
-            }
-            catch (makeBucketError) {
-                if (makeBucketError instanceof S3Error &&
-                    makeBucketError.code === 'BucketAlreadyOwnedByYou') {
-                    logger.warn(makeBucketError);
-                    logger.info('This was probably called in an asynchronous batch upload.');
-                }
-                else {
-                    throw makeBucketError;
-                }
-            }
-        }
+        await s3Client.send(new HeadBucketCommand({ Bucket: name }));
+        return;
     }
     catch (error) {
-        logger.error(error);
-        throw error;
+        if (!isS3ErrorCode(error, 'NotFound')) {
+            logger.error(error);
+            throw error;
+        }
     }
-}
-function isS3ErrorCode(error, code) {
-    return error instanceof S3Error && error.code === code;
-}
-export async function uploadS3File(bucket, key, content, size, metadata) {
     try {
-        await ensureBucket(bucket);
-        await s3Client.putObject(bucket, key, content, size, metadata);
+        await s3Client.send(new CreateBucketCommand({
+            Bucket: name,
+            // S3 rejects a location constraint for its default region.
+            ...(s3Region === 'us-east-1'
+                ? {}
+                : {
+                    CreateBucketConfiguration: {
+                        LocationConstraint: s3Region,
+                    },
+                }),
+        }));
     }
     catch (error) {
-        if (isS3ErrorCode(error, 'NoSuchBucket') && !(content instanceof Readable)) {
+        if (isS3ErrorCode(error, 'BucketAlreadyOwnedByYou')) {
             logger.warn(error);
-            await ensureBucket(bucket);
-            await s3Client.putObject(bucket, key, content, size, metadata);
+            logger.info('This was probably called in an asynchronous batch upload.');
             return;
         }
         logger.error(error);
         throw error;
     }
 }
-export async function uploadS3FileFromPath(bucket, key, path) {
+function splitMetadata(metadata = {}) {
+    const result = { Metadata: {} };
+    for (const [name, value] of Object.entries(metadata)) {
+        switch (name.toLowerCase()) {
+            case 'cache-control':
+                result.CacheControl = value;
+                break;
+            case 'content-type':
+                result.ContentType = value;
+                break;
+            default:
+                result.Metadata[name] = value;
+        }
+    }
+    return result;
+}
+async function putObject(bucket, key, content, size, metadata) {
+    const params = { Bucket: bucket, Key: key, Body: content, ...splitMetadata(metadata) };
+    if (content instanceof Readable && size == null) {
+        // S3 needs the object size up front, so a stream of unknown length goes through a multipart
+        // upload.
+        await new Upload({ client: s3Client, params }).done();
+        return;
+    }
+    await s3Client.send(new PutObjectCommand({
+        ...params,
+        ContentLength: size ?? Buffer.byteLength(content),
+    }));
+}
+export async function uploadS3File(bucket, key, content, size, metadata) {
     try {
         await ensureBucket(bucket);
-        await s3Client.fPutObject(bucket, key, path);
+        await putObject(bucket, key, content, size, metadata);
     }
     catch (error) {
-        if (isS3ErrorCode(error, 'NoSuchBucket')) {
+        if (isS3ErrorCode(error, 'NoSuchBucket') && !s3Bucket && !(content instanceof Readable)) {
             logger.warn(error);
             await ensureBucket(bucket);
-            await s3Client.fPutObject(bucket, key, path);
+            await putObject(bucket, key, content, size, metadata);
             return;
         }
         logger.error(error);
         throw error;
     }
 }
+export async function uploadS3FileFromPath(bucket, key, path) {
+    const { size } = await stat(path);
+    await uploadS3File(bucket, key, createReadStream(path), size);
+}
 export async function getS3File(bucket, key) {
     try {
-        return await s3Client.getObject(bucket, key);
+        const { Body } = await s3Client.send(new GetObjectCommand({ Bucket: bucket, Key: key }));
+        return Body;
     }
     catch (error) {
         logger.error(error);
@@ -101,52 +165,72 @@ export async function getS3FileBuffer(bucket, key) {
 }
 export async function getS3FileStats(bucket, key) {
     try {
-        return await s3Client.statObject(bucket, key);
+        const { CacheControl, ContentLength, ContentType, ETag, LastModified, Metadata } = await s3Client.send(new HeadObjectCommand({ Bucket: bucket, Key: key }));
+        return {
+            etag: ETag,
+            lastModified: LastModified,
+            metadata: {
+                ...(CacheControl && { 'cache-control': CacheControl }),
+                ...(ContentType && { 'content-type': ContentType }),
+                ...Metadata,
+            },
+            size: ContentLength,
+        };
     }
     catch (error) {
         logger.error(error);
         throw error;
     }
 }
-export async function listS3Files(bucket) {
-    const keys = await new Promise((resolve, reject) => {
-        const objects = [];
-        const stream = s3Client.listObjectsV2(bucket, '', true);
-        stream.on('data', (item) => {
-            if (item.name) {
-                objects.push(item.name);
+async function listS3Keys(bucket, prefix) {
+    const keys = [];
+    let continuationToken;
+    do {
+        const { Contents, NextContinuationToken } = await s3Client.send(new ListObjectsV2Command({
+            Bucket: bucket,
+            Prefix: prefix,
+            ContinuationToken: continuationToken,
+        }));
+        for (const { Key } of Contents ?? []) {
+            if (Key) {
+                keys.push(Key);
             }
-        });
-        stream.on('error', reject);
-        stream.on('end', () => resolve(objects));
-    });
-    return Promise.all(keys.map(async (key) => {
-        const stats = await getS3FileStats(bucket, key);
-        return {
-            etag: stats.etag,
-            key,
-            lastModified: stats.lastModified,
-            metadata: stats.metaData,
-            size: stats.size,
-        };
-    }));
+        }
+        continuationToken = NextContinuationToken;
+    } while (continuationToken);
+    return keys;
+}
+export async function listS3Files(bucket, prefix) {
+    const keys = await listS3Keys(bucket, prefix);
+    return Promise.all(keys.map(async (key) => ({ key, ...(await getS3FileStats(bucket, key)) })));
 }
 export async function setS3BucketPolicy(bucket, policy) {
     try {
         await ensureBucket(bucket);
-        await s3Client.setBucketPolicy(bucket, policy);
+        await s3Client.send(new PutBucketPolicyCommand({ Bucket: bucket, Policy: policy }));
     }
     catch (error) {
         logger.error(error);
         throw error;
     }
 }
+async function deleteObjects(bucket, keys) {
+    for (let index = 0; index < keys.length; index += deleteBatchSize) {
+        await s3Client.send(new DeleteObjectsCommand({
+            Bucket: bucket,
+            Delete: {
+                Objects: keys.slice(index, index + deleteBatchSize).map((Key) => ({ Key })),
+                Quiet: true,
+            },
+        }));
+    }
+}
 export async function deleteS3Files(bucket, keys) {
     try {
-        await s3Client.removeObjects(bucket, keys);
+        await deleteObjects(bucket, keys);
     }
     catch (error) {
-        if (error instanceof S3Error && error.code === 'NoSuchBucket') {
+        if (isS3ErrorCode(error, 'NoSuchBucket')) {
             logger.warn(`S3 bucket "${bucket}" does not exist; skipping deletion`);
             return;
         }
@@ -157,34 +241,35 @@ export async function deleteS3Files(bucket, keys) {
 export async function deleteS3File(bucket, key) {
     await deleteS3Files(bucket, [key]);
 }
+export async function deleteAppAssetObjects(appId, assetIds) {
+    await deleteS3Files(getAppAssetsBucket(appId), assetIds.map((assetId) => getAppAssetLocation(appId, assetId).key));
+}
+/**
+ * Remove every object this client can reach.
+ *
+ * In the single-bucket layout this empties the configured bucket. In the bucket-per-app layout
+ * this empties and removes every bucket.
+ */
 export async function clearAllS3Buckets() {
     try {
-        const buckets = await s3Client.listBuckets();
-        for (const bucket of buckets) {
+        if (s3Bucket) {
+            await deleteObjects(s3Bucket, await listS3Keys(s3Bucket));
+            return;
+        }
+        const { Buckets } = await s3Client.send(new ListBucketsCommand({}));
+        for (const { Name: bucket } of Buckets ?? []) {
             try {
-                const objectsStream = s3Client.listObjectsV2(bucket.name, '', true);
-                const objects = [];
-                for await (const o of objectsStream) {
-                    objects.push(o.name);
-                }
-                await s3Client.removeObjects(bucket.name, objects);
+                await deleteObjects(bucket, await listS3Keys(bucket));
                 try {
-                    await s3Client.removeBucket(bucket.name);
+                    await s3Client.send(new DeleteBucketCommand({ Bucket: bucket }));
                 }
                 catch (error) {
-                    if (isS3ErrorCode(error, 'NoSuchBucket')) {
-                        continue;
-                    }
                     if (!isS3ErrorCode(error, 'BucketNotEmpty')) {
                         throw error;
                     }
-                    const remainingObjectsStream = s3Client.listObjectsV2(bucket.name, '', true);
-                    const remainingObjects = [];
-                    for await (const o of remainingObjectsStream) {
-                        remainingObjects.push(o.name);
-                    }
-                    await s3Client.removeObjects(bucket.name, remainingObjects);
-                    await s3Client.removeBucket(bucket.name);
+                    // Objects uploaded while the bucket was being emptied.
+                    await deleteObjects(bucket, await listS3Keys(bucket));
+                    await s3Client.send(new DeleteBucketCommand({ Bucket: bucket }));
                 }
             }
             catch (error) {

package/testFixtures.js

@@ -27,7 +27,7 @@ export function resolveFixture(path) {
     return fileURLToPath(new URL(`__fixtures__/${path}`, baseDir));
 }
 export function readFixture(path, encoding) {
-    return readFile(resolveFixture(path), encoding);
+    return readFile(resolveFixture(path), encoding ?? null);
 }
 /**
  * Create a read stream for a fixture.