@appsemble/node-utils 0.36.7-test.0 → 0.36.7

package/README.md

@@ -1,9 +1,9 @@
-# ![](https://gitlab.com/appsemble/appsemble/-/raw/0.36.7-test.0/config/assets/logo.svg) Appsemble Node Utilities
+# ![](https://gitlab.com/appsemble/appsemble/-/raw/0.36.7/config/assets/logo.svg) Appsemble Node Utilities
 
 > NodeJS utilities used by Appsemble internally.
 
 [![npm](https://img.shields.io/npm/v/@appsemble/node-utils)](https://www.npmjs.com/package/@appsemble/node-utils)
-[![GitLab CI](https://gitlab.com/appsemble/appsemble/badges/0.36.7-test.0/pipeline.svg)](https://gitlab.com/appsemble/appsemble/-/releases/0.36.7-test.0)
+[![GitLab CI](https://gitlab.com/appsemble/appsemble/badges/0.36.7/pipeline.svg)](https://gitlab.com/appsemble/appsemble/-/releases/0.36.7)
 [![Prettier](https://img.shields.io/badge/code_style-prettier-ff69b4.svg)](https://prettier.io)
 
 ## Table of Contents
@@ -26,5 +26,5 @@ compatibility is not guaranteed.
 
 ## License
 
-[LGPL-3.0-only](https://gitlab.com/appsemble/appsemble/-/blob/0.36.7-test.0/LICENSE.md) ©
+[LGPL-3.0-only](https://gitlab.com/appsemble/appsemble/-/blob/0.36.7/LICENSE.md) ©
 [Appsemble](https://appsemble.com)

package/assets.d.ts

@@ -11,6 +11,6 @@ export interface AssetToUpload {
     mime: string;
     path: string;
 }
-export declare function uploadAsset(appId: number, asset: AssetToUpload): Promise<string[]>;
+export declare function uploadAsset(appId: number, asset: AssetToUpload): Promise<void>;
 export declare function uploadAssets(appId: number, assets: AssetToUpload[]): Promise<void>;
 export {};

package/assets.js

@@ -1,6 +1,5 @@
 import { createReadStream } from 'node:fs';
 import { stat } from 'node:fs/promises';
-import sharp from 'sharp';
 import { logger } from './logger.js';
 import { uploadS3File } from './s3.js';
 import { removeUploads } from './uploads.js';
@@ -17,49 +16,30 @@ export function setAssetHeaders(ctx, mime, filename, stats) {
     ctx.set('Access-Control-Expose-Headers', 'Content-Disposition');
     ctx.set('Cache-Control', 'max-age=31536000,immutable');
 }
-const ignoredMimes = new Set(['image/avif']);
 export function getCompressedFileMeta({ filename, mime }) {
-    if (mime?.startsWith('image') && !ignoredMimes.has(mime)) {
-        return {
-            filename: filename
-                ? filename.includes('.')
-                    ? `${filename.slice(0, filename.lastIndexOf('.'))}.avif`
-                    : `${filename}.avif`
-                : undefined,
-            mime: 'image/avif',
-        };
-    }
     return { filename, mime };
 }
 export async function uploadAsset(appId, asset) {
-    const { id, mime, path } = asset;
-    const filesToUnlink = [path];
-    let uploadFrom = path;
-    if (mime?.startsWith('image') && !ignoredMimes.has(mime)) {
-        uploadFrom = `${path}_compressed`;
-        await sharp(path).rotate().toFormat('avif').toFile(uploadFrom);
-        filesToUnlink.push(uploadFrom);
-    }
+    const { id, path } = asset;
     try {
-        const stats = await stat(uploadFrom);
-        const stream = createReadStream(uploadFrom);
+        const stats = await stat(path);
+        const stream = createReadStream(path);
         await uploadS3File(`app-${appId}`, id, stream, stats.size);
     }
     catch (error) {
         logger.error(error);
+        throw error;
     }
-    return filesToUnlink;
 }
 export async function uploadAssets(appId, assets) {
-    const filesToUnlink = [];
-    for (const asset of assets) {
-        const toUnlink = await uploadAsset(appId, asset);
-        for (const path of toUnlink) {
-            if (!filesToUnlink.includes(path)) {
-                filesToUnlink.push(path);
-            }
+    const filesToUnlink = [...new Set(assets.map(({ path }) => path))];
+    try {
+        for (const asset of assets) {
+            await uploadAsset(appId, asset);
         }
     }
-    await removeUploads(filesToUnlink);
+    finally {
+        await removeUploads(filesToUnlink);
+    }
 }
 //# sourceMappingURL=assets.js.map

package/index.d.ts

@@ -22,6 +22,7 @@ export * from './getAppsembleMessages.js';
 export * from './odata.js';
 export * from './resource.js';
 export * from './app.js';
+export * from './uploadValidation.js';
 export * from './mergeMessages.js';
 export * from './jsonschema.js';
 export * from './organizationBlocklist.js';

package/index.js

@@ -22,6 +22,7 @@ export * from './getAppsembleMessages.js';
 export * from './odata.js';
 export * from './resource.js';
 export * from './app.js';
+export * from './uploadValidation.js';
 export * from './mergeMessages.js';
 export * from './jsonschema.js';
 export * from './organizationBlocklist.js';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@appsemble/node-utils",
-  "version": "0.36.7-test.0",
+  "version": "0.36.7",
   "description": "NodeJS utilities used by Appsemble internally.",
   "keywords": [
     "app",
@@ -40,11 +40,12 @@
     "test": "vitest"
   },
   "dependencies": {
-    "@appsemble/lang-sdk": "0.36.7-test.0",
-    "@appsemble/types": "0.36.7-test.0",
-    "@appsemble/utils": "0.36.7-test.0",
+    "@appsemble/lang-sdk": "0.36.7",
+    "@appsemble/types": "0.36.7",
+    "@appsemble/utils": "0.36.7",
     "@formatjs/fast-memoize": "^2.0.0",
     "@fortawesome/fontawesome-free": "^6.0.0",
+    "@inquirer/prompts": "^8.0.0",
     "@kubernetes/client-node": "1.4.0",
     "@odata/parser": "^0.2.0",
     "@types/koa": "^2.0.0",
@@ -62,9 +63,9 @@
     "date-fns": "^2.0.0",
     "express-to-koa": "^2.0.0",
     "fast-glob": "^3.0.0",
+    "file-type": "^21.3.3",
     "form-data": "^4.0.4",
     "fs-extra": "^11.0.0",
-    "@inquirer/prompts": "^8.0.0",
     "intl-messageformat": "^11.0.0",
     "jsonschema": "~1.4.1",
     "keytar": "^7.0.0",

package/resource.d.ts

@@ -67,4 +67,4 @@ export declare function extractResourceBody(ctx: Context | ParameterizedContext<
 export declare function processResourceBody(ctx: Context | ParameterizedContext<DefaultState, DefaultContext, any>, definition: ResourceDefinition, knownAssetIds?: string[], knownExpires?: Date, knownAssetNameIds?: {
     id: string;
     name?: string;
-}[], isPatch?: boolean, resourceBody?: SerializedServerResourceBody): [Record<string, unknown> | Record<string, unknown>[], PreparedAsset[], string[]];
+}[], isPatch?: boolean, resourceBody?: SerializedServerResourceBody): Promise<[Record<string, unknown> | Record<string, unknown>[], PreparedAsset[], string[]]>;

package/resource.js

@@ -7,6 +7,7 @@ import parseDuration from 'parse-duration';
 import { preProcessCSV } from './csv.js';
 import { handleValidatorResult, TempFile } from './index.js';
 import { throwKoaError } from './koa.js';
+import { AssetUploadValidationError, validateUploadedFile } from './uploadValidation.js';
 export function stripResource({ $author, $created, $editor, $ephemeral, $group, $seed, $updated, ...data }) {
     return data;
 }
@@ -161,11 +162,24 @@ export function extractResourceBody(ctx, suppliedBody) {
  *   2. A list of newly uploaded assets which should be linked to the resources.
  *   3. Asset IDs from the `knownAssetIds` array which are no longer used.
  */
-export function processResourceBody(ctx, definition, knownAssetIds = [], knownExpires, knownAssetNameIds = [], isPatch = false, resourceBody) {
+export async function processResourceBody(ctx, definition, knownAssetIds = [], knownExpires, knownAssetNameIds = [], isPatch = false, resourceBody) {
     const [resource, assets, preValidateProperty] = extractResourceBody(ctx, resourceBody);
     const validator = new Validator();
     const reusedAssets = new Set();
     const usedAssetIndices = new Set();
+    const validatedAssetMimes = new Map();
+    const customErrors = [];
+    for (const [index, asset] of assets.entries()) {
+        try {
+            validatedAssetMimes.set(index, await validateUploadedFile(asset));
+        }
+        catch (error) {
+            if (!(error instanceof AssetUploadValidationError)) {
+                throw error;
+            }
+            customErrors.push(new ValidationError(error.message, index, undefined, ['assets', index], 'binary', 'content'));
+        }
+    }
     const thumbnailAssetSuffix = '-thumbnail.png';
     const [thumbnailAssets, regularAssets] = partition(
     // Preserve original index to handle asset references in data
@@ -175,7 +189,7 @@ export function processResourceBody(ctx, definition, knownAssetIds = [], knownEx
         asset: {
             id: randomUUID(),
             filename,
-            mime,
+            mime: validatedAssetMimes.get(index) ?? mime,
             path,
         },
     }));
@@ -247,7 +261,6 @@ export function processResourceBody(ctx, definition, knownAssetIds = [], knownEx
             ])),
         },
     };
-    const customErrors = [];
     const expiresDuration = definition.expires ? parseDuration(definition.expires) : undefined;
     const result = validator.validate(resource, Array.isArray(resource) ? { type: 'array', items: patchedSchema } : patchedSchema, {
         base: '#',

package/server/controllers/common/apps/resources/createCreateAppResourceController.js

@@ -1,6 +1,16 @@
 import { randomUUID } from 'node:crypto';
+import { copyFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
 import { isDeepStrictEqual } from 'node:util';
-import { getResourceDefinition, processResourceBody, uploadAssets, } from '../../../../../index.js';
+import { getResourceDefinition, processResourceBody, } from '../../../../../index.js';
+function clonePreparedAssets(preparedAssets) {
+    return Promise.all(preparedAssets.map(async (asset) => {
+        const path = join(tmpdir(), `${Date.now()}-${randomUUID()}`);
+        await copyFile(asset.path, path);
+        return { ...asset, path };
+    }));
+}
 /**
  * Create a controller for resource creation.
  *
@@ -29,15 +39,16 @@ export function createCreateAppResourceController(options) {
         });
         const resourceDefinition = getResourceDefinition(app.definition, resourceType, ctx);
         const appAssets = await getAppAssets({ app, context: ctx });
-        const [processedBody, preparedAssets] = processResourceBody(ctx, resourceDefinition, appAssets.map(({ id }) => id), undefined, appAssets.map((appAsset) => ({ id: appAsset.id, name: appAsset.name })));
+        const [processedBody, preparedAssets] = await processResourceBody(ctx, resourceDefinition, appAssets.map(({ id }) => id), undefined, appAssets.map((appAsset) => ({ id: appAsset.id, name: appAsset.name })));
         if (Array.isArray(processedBody) && !processedBody.length) {
             ctx.body = [];
             return;
         }
         const resources = Array.isArray(processedBody) ? processedBody : [processedBody];
-        const assetsToUpload = [];
         if (!(ctx.client && 'app' in ctx.client) && query?.seed === 'true') {
-            const preparedSeedAssets = structuredClone(preparedAssets);
+            const preparedSeedAssets = app.demoMode
+                ? await clonePreparedAssets(structuredClone(preparedAssets))
+                : structuredClone(preparedAssets);
             const preparedSeedResources = resources.map((resource) => {
                 const cleanResource = { ...resource };
                 if (app.demoMode) {
@@ -75,12 +86,8 @@ export function createCreateAppResourceController(options) {
             });
             if (!app.demoMode) {
                 ctx.body = Array.isArray(processedBody) ? createdSeedResources : createdSeedResources[0];
-                // @ts-expect-error 2345 argument of type is not assignable to parameter of type
-                // (strictNullChecks)
-                await uploadAssets(app.id, preparedSeedAssets);
                 return;
             }
-            assetsToUpload.push(...preparedSeedAssets);
         }
         const createdResources = await createAppResourcesWithAssets({
             app,
@@ -96,10 +103,6 @@ export function createCreateAppResourceController(options) {
             resourceType,
             options,
         });
-        assetsToUpload.push(...preparedAssets);
-        // @ts-expect-error 2345 argument of type is not assignable to parameter of type
-        // (strictNullChecks)
-        await uploadAssets(app.id, assetsToUpload);
         ctx.body = Array.isArray(processedBody) ? createdResources : createdResources[0];
     };
 }

package/server/controllers/common/apps/resources/createUpdateAppResourceController.js

@@ -36,7 +36,7 @@ export function createUpdateAppResourceController(options) {
         });
         const appAssets = await getAppAssets({ context: ctx, app });
         const resourceDefinition = getResourceDefinition(app.definition, resourceType, ctx);
-        const [processedBody, preparedAssets, deletedAssetIds] = processResourceBody(ctx, resourceDefinition, appAssets.filter((asset) => asset.resourceId === resourceId).map((asset) => asset.id), oldResource.expires, appAssets.map((asset) => ({ id: asset.id, name: asset.name })));
+        const [processedBody, preparedAssets, deletedAssetIds] = await processResourceBody(ctx, resourceDefinition, appAssets.filter((asset) => asset.resourceId === resourceId).map((asset) => asset.id), oldResource.expires, appAssets.map((asset) => ({ id: asset.id, name: asset.name })));
         const resources = Array.isArray(processedBody) ? processedBody : [processedBody];
         ctx.body = await updateAppResource({
             app,

package/server/controllers/main/apps/assets/createCreateAppAssetController.js

@@ -1,16 +1,27 @@
-import { assertKoaCondition } from '../../../../../koa.js';
+import { assertKoaCondition, throwKoaError } from '../../../../../koa.js';
+import { AssetUploadValidationError, validateUploadedFile, } from '../../../../../uploadValidation.js';
 export function createCreateAppAssetController({ createAppAsset, getApp }) {
     return async (ctx) => {
         const { pathParams: { appId }, request: { body: { file: { filename, mime, path }, name, }, }, } = ctx;
         const app = await getApp({ context: ctx, query: { attributes: ['id'], where: { id: appId } } });
         assertKoaCondition(app != null, ctx, 404, 'App not found');
+        let validatedMime;
+        try {
+            validatedMime = await validateUploadedFile({ filename, mime, path });
+        }
+        catch (error) {
+            if (error instanceof AssetUploadValidationError) {
+                throwKoaError(ctx, 400, error.message);
+            }
+            throw error;
+        }
         const asset = await createAppAsset({
             app,
             context: ctx,
-            payload: { filename, mime, name, path },
+            payload: { filename, mime: validatedMime, name, path },
         });
         ctx.status = 201;
-        ctx.body = { id: asset.id, mime, filename, name };
+        ctx.body = { id: asset.id, mime: asset.mime, filename, name };
     };
 }
 //# sourceMappingURL=createCreateAppAssetController.js.map

package/uploadValidation.d.ts

@@ -0,0 +1,12 @@
+import { MimeTypeCategory } from '@appsemble/utils';
+interface UploadedFileLike {
+    filename?: string | null;
+    mime: string;
+    path: string;
+}
+export declare class AssetUploadValidationError extends Error {
+    readonly Category: MimeTypeCategory.Image | MimeTypeCategory.Video;
+    constructor(message: string, category: MimeTypeCategory.Image | MimeTypeCategory.Video);
+}
+export declare function validateUploadedFile({ mime, path }: UploadedFileLike): Promise<string>;
+export {};

package/uploadValidation.js

@@ -0,0 +1,121 @@
+// CSpell:words theora
+import { open, stat } from 'node:fs/promises';
+import { MimeTypeCategory, getMimeTypeCategory } from '@appsemble/utils';
+import { fileTypeFromBuffer } from 'file-type';
+import { lookup } from 'mime-types';
+import sharp from 'sharp';
+const headerLength = 4096;
+export class AssetUploadValidationError extends Error {
+    constructor(message, category) {
+        super(message);
+        this.name = 'AssetUploadValidationError';
+        this.Category = category;
+    }
+}
+function detectSvgMime(buffer) {
+    const content = buffer
+        .toString('utf8')
+        .replace(/^\uFEFF/u, '')
+        .trimStart();
+    if (!content.startsWith('<')) {
+        return null;
+    }
+    return /<svg[\s>]/i.test(content) ? 'image/svg+xml' : null;
+}
+function detectOggVideoMime(buffer, declaredMime) {
+    if (buffer.length < 4 || buffer.toString('ascii', 0, 4) !== 'OggS') {
+        return null;
+    }
+    const content = buffer.toString('ascii').toLowerCase();
+    if (content.includes('theora') || declaredMime === 'video/ogg') {
+        return 'video/ogg';
+    }
+    return null;
+}
+function normalizeImageMime(format) {
+    const mime = lookup(format);
+    if (mime) {
+        return mime;
+    }
+    if (format === 'heif') {
+        return 'image/heif';
+    }
+    return null;
+}
+function normalizeDetectedMime(mime) {
+    if (mime === 'video/vnd.avi') {
+        return 'video/x-msvideo';
+    }
+    return mime;
+}
+async function readHeader(path) {
+    const file = await open(path, 'r');
+    try {
+        const { size } = await file.stat();
+        const buffer = Buffer.alloc(Math.min(size, headerLength));
+        if (buffer.length) {
+            await file.read(buffer, 0, buffer.length, 0);
+        }
+        return buffer;
+    }
+    finally {
+        await file.close();
+    }
+}
+async function detectMime(buffer, declaredMime) {
+    const svgMime = detectSvgMime(buffer);
+    if (svgMime) {
+        return svgMime;
+    }
+    const detected = await fileTypeFromBuffer(buffer);
+    if (detected) {
+        return normalizeDetectedMime(detected.mime);
+    }
+    return detectOggVideoMime(buffer, declaredMime);
+}
+export async function validateUploadedFile({ mime, path }) {
+    const declaredCategory = getMimeTypeCategory(mime);
+    const { size } = await stat(path);
+    const buffer = await readHeader(path);
+    const detectedMime = await detectMime(buffer, mime);
+    const detectedCategory = detectedMime ? getMimeTypeCategory(detectedMime) : null;
+    const category = detectedCategory ?? declaredCategory;
+    if (category !== MimeTypeCategory.Image && category !== MimeTypeCategory.Video) {
+        return mime;
+    }
+    if (size === 0) {
+        throw new AssetUploadValidationError(`${category === MimeTypeCategory.Image ? 'Image' : 'Video'} uploads cannot be empty`, category);
+    }
+    if (declaredCategory &&
+        detectedCategory &&
+        declaredCategory !== detectedCategory &&
+        [MimeTypeCategory.Image, MimeTypeCategory.Video].includes(declaredCategory)) {
+        throw new AssetUploadValidationError(declaredCategory === MimeTypeCategory.Image
+            ? 'Image uploads must contain a valid image'
+            : 'Video uploads must contain a supported video container', declaredCategory);
+    }
+    if (declaredCategory === MimeTypeCategory.Image || detectedCategory === MimeTypeCategory.Image) {
+        try {
+            const metadata = await sharp(path).metadata();
+            if (!metadata.format) {
+                throw new Error('Missing image format');
+            }
+            const normalizedMime = normalizeImageMime(metadata.format);
+            if (!normalizedMime) {
+                throw new Error(`Unsupported image format: ${metadata.format}`);
+            }
+            return normalizedMime;
+        }
+        catch {
+            throw new AssetUploadValidationError('Image uploads must contain a valid image', MimeTypeCategory.Image);
+        }
+    }
+    if (declaredCategory === MimeTypeCategory.Video || detectedCategory === MimeTypeCategory.Video) {
+        if (detectedCategory !== MimeTypeCategory.Video || !detectedMime) {
+            throw new AssetUploadValidationError('Video uploads must contain a supported video container', MimeTypeCategory.Video);
+        }
+        return detectedMime;
+    }
+    return mime;
+}
+//# sourceMappingURL=uploadValidation.js.map