@appsemble/node-utils 0.37.4 → 0.37.5

package/README.md

@@ -1,9 +1,9 @@
-# ![](https://gitlab.com/appsemble/appsemble/-/raw/0.37.4/config/assets/logo.svg) Appsemble Node Utilities
+# ![](https://gitlab.com/appsemble/appsemble/-/raw/0.37.5/config/assets/logo.svg) Appsemble Node Utilities
 
 > NodeJS utilities used by Appsemble internally.
 
 [![npm](https://img.shields.io/npm/v/@appsemble/node-utils)](https://www.npmjs.com/package/@appsemble/node-utils)
-[![GitLab CI](https://gitlab.com/appsemble/appsemble/badges/0.37.4/pipeline.svg)](https://gitlab.com/appsemble/appsemble/-/releases/0.37.4)
+[![GitLab CI](https://gitlab.com/appsemble/appsemble/badges/0.37.5/pipeline.svg)](https://gitlab.com/appsemble/appsemble/-/releases/0.37.5)
 [![Prettier](https://img.shields.io/badge/code_style-prettier-ff69b4.svg)](https://prettier.io)
 
 ## Table of Contents
@@ -26,5 +26,5 @@ compatibility is not guaranteed.
 
 ## License
 
-[LGPL-3.0-only](https://gitlab.com/appsemble/appsemble/-/blob/0.37.4/LICENSE.md) ©
+[LGPL-3.0-only](https://gitlab.com/appsemble/appsemble/-/blob/0.37.5/LICENSE.md) ©
 [Appsemble](https://appsemble.com)

package/assetCssURL.d.ts

@@ -0,0 +1,9 @@
+/**
+ * Resolve `asset()` functions and app asset URLs in CSS to absolute app asset URLs.
+ *
+ * @param css The CSS to process.
+ * @param appId The id of the app the assets belong to.
+ * @param host The host on which the app asset endpoints are available.
+ * @returns The CSS with all app asset references resolved.
+ */
+export declare function replaceAssetFunctions(css: string, appId: number | undefined, host: string): string;

package/assetCssURL.js

@@ -0,0 +1,134 @@
+const assetUrlMatcher = /url\(\s*asset\(\s*(?:("|')(.*?)\1|([^\s"'()][^)]*))\s*\)\s*\)/g;
+const standaloneAssetMatcher = /asset\(\s*(?:("|')(.*?)\1|([^\s"'()][^)]*))\s*\)/g;
+const urlMatcher = /url\(\s*(?:("|')(.*?)\1|([^\s"'()][^)]*))\s*\)/g;
+const appAssetPathMatcher = /^\/api\/apps\/\d+\/assets\/(.+)$/;
+function hasRejectedTokens(value) {
+    let currentValue = value;
+    for (let i = 0; i < 3; i += 1) {
+        if (currentValue.includes('..') || /%2f|%5c/i.test(currentValue)) {
+            return true;
+        }
+        if (!currentValue.includes('%')) {
+            return false;
+        }
+        try {
+            const decodedValue = decodeURIComponent(currentValue);
+            if (decodedValue === currentValue) {
+                return false;
+            }
+            if (decodedValue.includes('/') || decodedValue.includes('\\')) {
+                return true;
+            }
+            currentValue = decodedValue;
+        }
+        catch {
+            return true;
+        }
+    }
+    return false;
+}
+function rewriteAppAssetURL(url, appId, host) {
+    const trimmedUrl = url.trim();
+    if (trimmedUrl.startsWith('/')) {
+        const appAssetPathMatch = appAssetPathMatcher.exec(trimmedUrl);
+        if (!appAssetPathMatch) {
+            return trimmedUrl;
+        }
+        const appAssetPath = appAssetPathMatch[1];
+        if (hasRejectedTokens(appAssetPath)) {
+            return trimmedUrl;
+        }
+        return String(new URL(`/api/apps/${appId}/assets/${appAssetPath}`, host));
+    }
+    try {
+        const parsedUrl = new URL(trimmedUrl, host);
+        const appAssetPathMatch = appAssetPathMatcher.exec(parsedUrl.pathname);
+        if (!appAssetPathMatch) {
+            return trimmedUrl;
+        }
+        const appAssetPath = appAssetPathMatch[1];
+        if (hasRejectedTokens(appAssetPath)) {
+            return trimmedUrl;
+        }
+        const rewrittenUrl = new URL(`/api/apps/${appId}/assets/${appAssetPath}`, host);
+        rewrittenUrl.search = parsedUrl.search;
+        rewrittenUrl.hash = parsedUrl.hash;
+        return String(rewrittenUrl);
+    }
+    catch {
+        return trimmedUrl;
+    }
+}
+function resolveAssetURL(appId, assetId, host) {
+    if (assetId.startsWith('data:')) {
+        return assetId;
+    }
+    if (assetId.startsWith('/')) {
+        const appAssetPathMatch = appAssetPathMatcher.exec(assetId.trim());
+        if (appAssetPathMatch && hasRejectedTokens(appAssetPathMatch[1])) {
+            return null;
+        }
+        return rewriteAppAssetURL(assetId, appId, host);
+    }
+    if (/^https?:\/\//.test(assetId)) {
+        const trimmedAssetId = assetId.trim();
+        try {
+            const parsedUrl = new URL(trimmedAssetId, host);
+            const appAssetPathMatch = appAssetPathMatcher.exec(parsedUrl.pathname);
+            if (appAssetPathMatch && hasRejectedTokens(appAssetPathMatch[1])) {
+                return null;
+            }
+        }
+        catch {
+            return null;
+        }
+        return rewriteAppAssetURL(assetId, appId, host);
+    }
+    if (hasRejectedTokens(assetId)) {
+        return null;
+    }
+    return String(new URL(`/api/apps/${appId}/assets/${assetId}`, host));
+}
+/**
+ * Resolve `asset()` functions and app asset URLs in CSS to absolute app asset URLs.
+ *
+ * @param css The CSS to process.
+ * @param appId The id of the app the assets belong to.
+ * @param host The host on which the app asset endpoints are available.
+ * @returns The CSS with all app asset references resolved.
+ */
+export function replaceAssetFunctions(css, appId, host) {
+    if (!appId) {
+        return css;
+    }
+    const replaceAssetFunction = (match, ...args) => {
+        const quotedAssetId = args[1];
+        const unquotedAssetId = args[2];
+        const assetId = (quotedAssetId || unquotedAssetId)?.trim();
+        if (!assetId) {
+            return match;
+        }
+        const resolvedAssetUrl = resolveAssetURL(appId, assetId, host);
+        if (!resolvedAssetUrl) {
+            return match;
+        }
+        return `url('${resolvedAssetUrl}')`;
+    };
+    const cssWithResolvedAssetFunctionURLs = css
+        .replaceAll(assetUrlMatcher, replaceAssetFunction)
+        .replaceAll(standaloneAssetMatcher, replaceAssetFunction);
+    return cssWithResolvedAssetFunctionURLs.replaceAll(urlMatcher, (match, ...args) => {
+        const quotedUrl = args[1];
+        const unquotedUrl = args[2];
+        const url = (quotedUrl || unquotedUrl)?.trim();
+        if (!url) {
+            return match;
+        }
+        const rewrittenUrl = rewriteAppAssetURL(url, appId, host);
+        if (rewrittenUrl === url) {
+            return match;
+        }
+        return `url('${rewrittenUrl}')`;
+    });
+}
+//# sourceMappingURL=assetCssURL.js.map

package/getBlockAssetDownloadUrl.d.ts

@@ -0,0 +1,3 @@
+import { type BlockManifest } from '@appsemble/types';
+export declare function isValidBlockAssetFilename(filename: string): boolean;
+export declare function getBlockAssetDownloadUrl(blockUrl: string, fileUrls: BlockManifest['fileUrls'], filename: string): string;

package/getBlockAssetDownloadUrl.js

@@ -0,0 +1,22 @@
+const blockAssetUnsafePathPattern = /(^|\/)\.{1,2}(\/|$)|^\/|\\/;
+export function isValidBlockAssetFilename(filename) {
+    if (!filename || blockAssetUnsafePathPattern.test(filename)) {
+        return false;
+    }
+    for (let i = 0; i < filename.length; i += 1) {
+        if (filename.charCodeAt(i) < 0x20) {
+            return false;
+        }
+    }
+    return true;
+}
+export function getBlockAssetDownloadUrl(blockUrl, fileUrls, filename) {
+    const fileUrl = fileUrls && Object.hasOwn(fileUrls, filename) ? fileUrls[filename] : undefined;
+    if (fileUrl) {
+        return fileUrl;
+    }
+    const fallbackUrl = new URL(`${blockUrl.replace(/\/+$/, '')}/asset`);
+    fallbackUrl.searchParams.set('filename', filename);
+    return String(fallbackUrl);
+}
+//# sourceMappingURL=getBlockAssetDownloadUrl.js.map

package/index.d.ts

@@ -19,6 +19,7 @@ export * from './icon.js';
 export * from './readAsset.js';
 export * from './render.js';
 export * from './getAppsembleMessages.js';
+export * from './getBlockAssetDownloadUrl.js';
 export * from './odata.js';
 export * from './resource.js';
 export * from './resourceEtag.js';
@@ -43,6 +44,7 @@ export * from './container/index.js';
 export * from './s3.js';
 export * from './uploads.js';
 export * from './assets.js';
+export * from './assetCssURL.js';
 export * from './getValidTrainings.js';
 export * from './PhoneNumberValidationError.js';
 export * from './createUser.js';

package/index.js

@@ -19,6 +19,7 @@ export * from './icon.js';
 export * from './readAsset.js';
 export * from './render.js';
 export * from './getAppsembleMessages.js';
+export * from './getBlockAssetDownloadUrl.js';
 export * from './odata.js';
 export * from './resource.js';
 export * from './resourceEtag.js';
@@ -43,6 +44,7 @@ export * from './container/index.js';
 export * from './s3.js';
 export * from './uploads.js';
 export * from './assets.js';
+export * from './assetCssURL.js';
 export * from './getValidTrainings.js';
 export * from './PhoneNumberValidationError.js';
 export * from './createUser.js';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@appsemble/node-utils",
-  "version": "0.37.4",
+  "version": "0.37.5",
   "description": "NodeJS utilities used by Appsemble internally.",
   "keywords": [
     "app",
@@ -40,9 +40,9 @@
     "test": "vitest"
   },
   "dependencies": {
-    "@appsemble/lang-sdk": "0.37.4",
-    "@appsemble/types": "0.37.4",
-    "@appsemble/utils": "0.37.4",
+    "@appsemble/lang-sdk": "0.37.5",
+    "@appsemble/types": "0.37.5",
+    "@appsemble/utils": "0.37.5",
     "@formatjs/fast-memoize": "^2.0.0",
     "@fortawesome/fontawesome-free": "^6.0.0",
     "@inquirer/prompts": "^8.0.0",
@@ -85,7 +85,7 @@
     "logform": "^2.0.0",
     "memfs": "4.57.8",
     "mime-types": "^2.0.0",
-    "minio": "^8.0.3",
+    "minio": "^8.0.7",
     "mustache": "^4.0.0",
     "openapi-types": "^12.1.3",
     "parse-duration": "^1.0.0",

package/s3.d.ts

@@ -1,5 +1,12 @@
-import { type Readable } from 'node:stream';
+import { Readable } from 'node:stream';
 import { type BucketItemStat } from 'minio';
+export interface S3FileReference {
+    etag: string;
+    key: string;
+    lastModified: Date;
+    metadata: BucketItemStat['metaData'];
+    size: number;
+}
 export interface InitS3ClientParams {
     endPoint: string;
     port?: number;
@@ -8,11 +15,13 @@ export interface InitS3ClientParams {
     secretKey: string;
 }
 export declare function initS3Client({ accessKey, endPoint, port, secretKey, useSSL, }: InitS3ClientParams): void;
-export declare function uploadS3File(bucket: string, key: string, content: Buffer | Readable | string, size?: number): Promise<void>;
+export declare function uploadS3File(bucket: string, key: string, content: Buffer | Readable | string, size?: number, metadata?: Record<string, string>): Promise<void>;
 export declare function uploadS3FileFromPath(bucket: string, key: string, path: string): Promise<void>;
 export declare function getS3File(bucket: string, key: string): Promise<Readable>;
 export declare function getS3FileBuffer(bucket: string, key: string): Promise<Buffer>;
 export declare function getS3FileStats(bucket: string, key: string): Promise<BucketItemStat>;
+export declare function listS3Files(bucket: string): Promise<S3FileReference[]>;
+export declare function setS3BucketPolicy(bucket: string, policy: string): Promise<void>;
 export declare function deleteS3Files(bucket: string, keys: string[]): Promise<void>;
 export declare function deleteS3File(bucket: string, key: string): Promise<void>;
 export declare function clearAllS3Buckets(): Promise<void>;

package/s3.js

@@ -1,3 +1,4 @@
+import { Readable } from 'node:stream';
 import { buffer as streamToBuffer } from 'node:stream/consumers';
 import { Client, S3Error } from 'minio';
 import { logger } from './logger.js';
@@ -41,12 +42,21 @@ async function ensureBucket(name) {
         throw error;
     }
 }
-export async function uploadS3File(bucket, key, content, size) {
+function isS3ErrorCode(error, code) {
+    return error instanceof S3Error && error.code === code;
+}
+export async function uploadS3File(bucket, key, content, size, metadata) {
     try {
         await ensureBucket(bucket);
-        await s3Client.putObject(bucket, key, content, size);
+        await s3Client.putObject(bucket, key, content, size, metadata);
     }
     catch (error) {
+        if (isS3ErrorCode(error, 'NoSuchBucket') && !(content instanceof Readable)) {
+            logger.warn(error);
+            await ensureBucket(bucket);
+            await s3Client.putObject(bucket, key, content, size, metadata);
+            return;
+        }
         logger.error(error);
         throw error;
     }
@@ -57,6 +67,12 @@ export async function uploadS3FileFromPath(bucket, key, path) {
         await s3Client.fPutObject(bucket, key, path);
     }
     catch (error) {
+        if (isS3ErrorCode(error, 'NoSuchBucket')) {
+            logger.warn(error);
+            await ensureBucket(bucket);
+            await s3Client.fPutObject(bucket, key, path);
+            return;
+        }
         logger.error(error);
         throw error;
     }
@@ -92,6 +108,39 @@ export async function getS3FileStats(bucket, key) {
         throw error;
     }
 }
+export async function listS3Files(bucket) {
+    const keys = await new Promise((resolve, reject) => {
+        const objects = [];
+        const stream = s3Client.listObjectsV2(bucket, '', true);
+        stream.on('data', (item) => {
+            if (item.name) {
+                objects.push(item.name);
+            }
+        });
+        stream.on('error', reject);
+        stream.on('end', () => resolve(objects));
+    });
+    return Promise.all(keys.map(async (key) => {
+        const stats = await getS3FileStats(bucket, key);
+        return {
+            etag: stats.etag,
+            key,
+            lastModified: stats.lastModified,
+            metadata: stats.metaData,
+            size: stats.size,
+        };
+    }));
+}
+export async function setS3BucketPolicy(bucket, policy) {
+    try {
+        await ensureBucket(bucket);
+        await s3Client.setBucketPolicy(bucket, policy);
+    }
+    catch (error) {
+        logger.error(error);
+        throw error;
+    }
+}
 export async function deleteS3Files(bucket, keys) {
     try {
         await s3Client.removeObjects(bucket, keys);
@@ -112,13 +161,37 @@ export async function clearAllS3Buckets() {
     try {
         const buckets = await s3Client.listBuckets();
         for (const bucket of buckets) {
-            const objectsStream = s3Client.listObjectsV2(bucket.name, '', true);
-            const objects = [];
-            for await (const o of objectsStream) {
-                objects.push(o.name);
+            try {
+                const objectsStream = s3Client.listObjectsV2(bucket.name, '', true);
+                const objects = [];
+                for await (const o of objectsStream) {
+                    objects.push(o.name);
+                }
+                await s3Client.removeObjects(bucket.name, objects);
+                try {
+                    await s3Client.removeBucket(bucket.name);
+                }
+                catch (error) {
+                    if (isS3ErrorCode(error, 'NoSuchBucket')) {
+                        continue;
+                    }
+                    if (!isS3ErrorCode(error, 'BucketNotEmpty')) {
+                        throw error;
+                    }
+                    const remainingObjectsStream = s3Client.listObjectsV2(bucket.name, '', true);
+                    const remainingObjects = [];
+                    for await (const o of remainingObjectsStream) {
+                        remainingObjects.push(o.name);
+                    }
+                    await s3Client.removeObjects(bucket.name, remainingObjects);
+                    await s3Client.removeBucket(bucket.name);
+                }
+            }
+            catch (error) {
+                if (!isS3ErrorCode(error, 'NoSuchBucket')) {
+                    throw error;
+                }
             }
-            await s3Client.removeObjects(bucket.name, objects);
-            await s3Client.removeBucket(bucket.name);
         }
     }
     catch (error) {

package/server/routes/appRouter/blockAssetHandler.js

@@ -6,8 +6,17 @@ export function createBlockAssetHandler({ getBlockAsset }) {
         params: { filename, name, version }, } = ctx;
         const blockAsset = await getBlockAsset({ filename, name, version, context: ctx });
         assertKoaCondition(blockAsset != null, ctx, 404, 'Block asset not found');
-        ctx.set('Cache-Control', 'max-age=31536000,immutable');
-        ctx.body = blockAsset.content;
+        ctx.set('Cache-Control', 'public,max-age=31536000,immutable');
+        if (blockAsset.size != null) {
+            ctx.set('Content-Length', String(blockAsset.size));
+        }
+        if (blockAsset.etag) {
+            ctx.set('ETag', blockAsset.etag);
+        }
+        if (blockAsset.lastModified) {
+            ctx.set('Last-Modified', blockAsset.lastModified.toUTCString());
+        }
+        ctx.body = blockAsset.stream ?? blockAsset.content;
         ctx.type = blockAsset.mime;
     };
 }

package/server/types.d.ts

@@ -391,7 +391,11 @@ export interface AppAsset extends Asset {
 export interface ProjectAsset {
     filename: string;
     mime: string;
-    content: Buffer;
+    content?: Buffer;
+    stream?: Readable;
+    size?: number;
+    etag?: string;
+    lastModified?: Date;
 }
 export interface Block extends BlockDefinition {
     OrganizationId: string;