@saltcorn/data 1.5.0-beta.8 → 1.5.0-rc.1

package/dist/models/file.js

@@ -11,15 +11,18 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 const db_1 = __importDefault(require("../db"));
 const uuid_1 = require("uuid");
 const path_1 = require("path");
+const s3_helpers_1 = require("./internal/s3_helpers");
 const { asyncMap } = require("../utils");
 const promises_1 = require("fs/promises");
 const axios_1 = __importDefault(require("axios"));
 const form_data_1 = __importDefault(require("form-data"));
 const mime_types_1 = require("mime-types");
 const path = require("path");
+const posix = path.posix;
 const fsp = require("fs").promises;
 const fs = require("fs");
 const fsx = require("fs-extended-attributes");
+const ABSOLUTE_URL_RE = /^[a-zA-Z][a-zA-Z\d+\-.]*:\/\//;
 function xattr_set(fp, attrName, value) {
     return new Promise((resolve) => fsx.set(fp, attrName, value, resolve));
 }
@@ -33,6 +36,7 @@ function xattr_get(fp, attrName) {
 }
 const dirCache = {};
 const enableDirCache = {};
+const DEFAULT_MIN_ROLE_READ = 100;
 /**
  * File Descriptor class
  *
@@ -73,14 +77,12 @@ class File {
      * @returns {Promise<*>}
      */
     static async find(where, selectopts = {}) {
-        const { getState } = require("../db/state");
-        const state = getState();
-        const useS3 = state?.getConfig("storage_s3_enabled");
-        if (useS3 || where?.inDB) {
-            if (where?.inDB)
-                delete where.inDB;
-            const db_flds = await db_1.default.select("_sc_files", where, selectopts);
-            return db_flds.map((dbf) => new File(dbf));
+        if (where?.inDB) {
+            return [];
+        }
+        const useS3 = (0, s3_helpers_1.isS3Enabled)();
+        if (useS3) {
+            return await File.findInS3(where, selectopts);
         }
         else {
             const relativeSearchFolder = where?.folder || "/";
@@ -144,11 +146,81 @@ class File {
         else
             return null;
     }
+    static isAbsoluteURL(value) {
+        return typeof value === "string" && ABSOLUTE_URL_RE.test(value);
+    }
+    static fieldValueFromRelative(relPath) {
+        if (!relPath)
+            return relPath || "";
+        const cleaned = relPath.replace(/^[\/]+/, "").replace(/\\/g, "/");
+        return cleaned;
+    }
+    static relativePathFromFieldValue(value) {
+        if (typeof value !== "string")
+            return null;
+        const trimmed = value.trim();
+        if (!trimmed)
+            return "";
+        if (File.isAbsoluteURL(trimmed)) {
+            if ((0, s3_helpers_1.isS3Enabled)()) {
+                const rel = (0, s3_helpers_1.publicUrlToRelativePath)(trimmed);
+                if (rel)
+                    return rel;
+            }
+            return null;
+        }
+        const normalised = File.normalise(trimmed).replace(/\\/g, "/");
+        return normalised.startsWith("/") ? normalised.slice(1) : normalised;
+    }
+    static normalizeFieldValueInput(value) {
+        if (typeof value !== "string")
+            return value || "";
+        const relative = File.relativePathFromFieldValue(value);
+        if (relative === null)
+            return value.trim();
+        return File.fieldValueFromRelative(relative);
+    }
+    static pathToServeUrl(value, opts = {}) {
+        if (!value)
+            return "";
+        const trimmed = value.trim();
+        if (File.isAbsoluteURL(trimmed))
+            return trimmed;
+        const relative = File.relativePathFromFieldValue(trimmed);
+        if (relative === null)
+            return trimmed;
+        if ((0, s3_helpers_1.isS3Enabled)()) {
+            try {
+                const { getState } = require("../db/state");
+                const direct = !!getState()?.getConfig("files_direct_s3_links");
+                if (direct && opts.preferDirect !== false) {
+                    return (0, s3_helpers_1.getPublicFileUrl)(relative, {
+                        download: opts.download,
+                        filename: opts.filename || posix.basename(relative),
+                    });
+                }
+            }
+            catch (e) {
+                // ignore and fall back to proxied serving
+            }
+        }
+        const cleaned = File.fieldValueFromRelative(relative);
+        const safePath = cleaned.replace(/^[\/]+/, "");
+        const route = opts.download ? "download" : "serve";
+        const prefix = opts.targetPrefix || "";
+        return `${prefix}/files/${route}/${safePath}`;
+    }
     static async rootFolder() {
+        if ((0, s3_helpers_1.isS3Enabled)())
+            return File.directoryFromS3Path("/");
         const tenant = db_1.default.getTenantSchema();
         return await File.from_file_on_disk("/", path.join(db_1.default.connectObj.file_store, tenant));
     }
     static absPathToServePath(absPath) {
+        if ((0, s3_helpers_1.isS3Enabled)()) {
+            const rel = File.normalise(`${absPath}`);
+            return rel.startsWith("/") ? rel.slice(1) : rel;
+        }
         if (typeof absPath === "number")
             return `${absPath}`;
         const tenant = db_1.default.getTenantSchema();
@@ -166,6 +238,21 @@ class File {
      * @returns
      */
     static async allDirectories(ignoreCache) {
+        if ((0, s3_helpers_1.isS3Enabled)()) {
+            const { files } = await (0, s3_helpers_1.listS3Folder)("/", { recursive: true });
+            const dirs = new Set(["/"]);
+            for (const obj of files) {
+                const relPath = obj.relativePath || "";
+                const dirName = relPath.includes("/") ? posix.dirname(relPath) : "";
+                if (!dirName || dirName === ".")
+                    continue;
+                const parts = dirName.split("/").filter(Boolean);
+                for (let i = 1; i <= parts.length; i++) {
+                    dirs.add(parts.slice(0, i).join("/"));
+                }
+            }
+            return Array.from(dirs).map((dir) => File.directoryFromS3Path(dir === "" ? "/" : dir));
+        }
         const tenant = db_1.default.getTenantSchema();
         if (!ignoreCache) {
             const cache = File.getDirCache();
@@ -233,6 +320,91 @@ class File {
         enableDirCache[db_1.default.getTenantSchema()] = false;
         dirCache[db_1.default.getTenantSchema()] = null;
     }
+    static async findInS3(where, selectopts = {}) {
+        const folder = typeof where?.folder === "string" ? where.folder : "/";
+        const recursive = !!selectopts.recursive || !!where?.search;
+        const { files, directories } = await (0, s3_helpers_1.listS3Folder)(folder, { recursive });
+        const results = [];
+        for (const dir of directories) {
+            const dirFile = File.directoryFromS3Path(dir || "/");
+            if (dirFile.filename?.startsWith(".") && !where?.hiddenFiles)
+                continue;
+            results.push(dirFile);
+        }
+        const headResults = await Promise.all(files.map((obj) => (0, s3_helpers_1.headObject)(obj.relativePath)));
+        for (const head of headResults) {
+            if (!head)
+                continue;
+            const file = File.fileFromS3Head(head);
+            if (file.filename?.startsWith(".") && !where?.hiddenFiles)
+                continue;
+            results.push(file);
+        }
+        let filtered = results;
+        if (where?.filename) {
+            filtered = filtered.filter((f) => f.filename === where.filename);
+        }
+        if (where?.mime_super) {
+            filtered = filtered.filter((f) => f.mime_super === where.mime_super);
+        }
+        if (where?.mime_sub) {
+            filtered = filtered.filter((f) => f.mime_sub === where.mime_sub);
+        }
+        if (where?.isDirectory !== undefined) {
+            filtered = filtered.filter((f) => !!f.isDirectory === !!where.isDirectory);
+        }
+        if (where?.search) {
+            filtered = filtered.filter((f) => f.filename.includes(where.search));
+        }
+        return filtered.sort((a, b) => a.filename.localeCompare(b.filename));
+    }
+    static async findOneS3(where) {
+        if (typeof where === "string") {
+            const safePath = File.normalise(where);
+            const head = await (0, s3_helpers_1.headObject)(safePath);
+            return head ? File.fileFromS3Head(head) : null;
+        }
+        const results = await File.findInS3(where);
+        return results[0] || null;
+    }
+    static fileFromS3Head(info) {
+        const metadata = info.metadata || {};
+        const mime = metadata.mime_super && metadata.mime_sub
+            ? `${metadata.mime_super}/${metadata.mime_sub}`
+            : File.nameToMimeType(metadata.filename || info.relativePath) || "";
+        const [mime_super, mime_sub] = mime ? mime.split("/") : ["", ""];
+        return new File({
+            filename: metadata.filename || posix.basename(info.relativePath),
+            location: info.relativePath,
+            uploaded_at: info.lastModified || new Date(),
+            size_kb: Math.max(1, Math.round((info.size || 0) / 1024)),
+            mime_super,
+            mime_sub,
+            min_role_read: metadata.min_role_read || DEFAULT_MIN_ROLE_READ,
+            user_id: metadata.user_id,
+            s3_store: true,
+        });
+    }
+    static directoryFromS3Path(dirPath) {
+        const normalised = File.normalise(dirPath) || "/";
+        const filename = normalised === "/" ? "/" : posix.basename(normalised);
+        return new File({
+            filename,
+            location: normalised,
+            uploaded_at: new Date(),
+            size_kb: 0,
+            mime_super: "",
+            mime_sub: "",
+            min_role_read: DEFAULT_MIN_ROLE_READ,
+            s3_store: true,
+            isDirectory: true,
+        });
+    }
+    static relativeS3Path(key) {
+        const normalised = File.normalise(key);
+        const rel = (0, s3_helpers_1.relativeKeyToPath)(normalised);
+        return rel.startsWith("/") ? rel.slice(1) : rel;
+    }
     async is_symlink() {
         try {
             let stat = await fsp.lstat(this.location);
@@ -296,7 +468,7 @@ class File {
         if (typeof where === "string") {
             const { getState } = require("../db/state");
             const state = getState();
-            const useS3 = state?.getConfig("storage_s3_enabled");
+            const useS3 = (0, s3_helpers_1.isS3Enabled)();
             //legacy serving ids
             if (/^\d+$/.test(where)) {
                 const legacy_file_id_locations = state?.getConfig("legacy_file_id_locations");
@@ -307,26 +479,25 @@ class File {
                 }
             }
             if (useS3) {
-                const files = await File.find({ id: +where });
-                return files[0];
+                const relative = File.relativePathFromFieldValue(where);
+                const lookup = relative === null ? where : relative;
+                return await File.findOneS3(lookup);
             }
-            else {
-                const tenant = db_1.default.getTenantSchema();
-                const safeDir = path.normalize(where).replace(/^(\.\.(\/|\\|$))+/, "");
-                const absoluteFolder = path.join(db_1.default.connectObj.file_store, tenant, safeDir);
-                const name = path.basename(absoluteFolder);
-                const dir = path.dirname(absoluteFolder);
-                try {
-                    return await File.from_file_on_disk(name, dir);
-                }
-                catch (e) {
-                    state?.log(2, e?.toString ? e.toString() : e);
-                    return null;
-                }
+            const tenant = db_1.default.getTenantSchema();
+            const safeDir = path.normalize(where).replace(/^(\.\.(\/|\\|$))+/, "");
+            const absoluteFolder = path.join(db_1.default.connectObj.file_store, tenant, safeDir);
+            const name = path.basename(absoluteFolder);
+            const dir = path.dirname(absoluteFolder);
+            try {
+                return await File.from_file_on_disk(name, dir);
+            }
+            catch (e) {
+                state?.log(2, e?.toString ? e.toString() : e);
+                return null;
             }
         }
         const files = await File.find(where);
-        return files.length > 0 ? new File(files[0]) : null;
+        return files.length > 0 ? files[0] : null;
     }
     /**
      * Create new folder
@@ -334,6 +505,23 @@ class File {
      * @param inFolder
      */
     static async new_folder(name, inFolder = "") {
+        if ((0, s3_helpers_1.isS3Enabled)()) {
+            const folderName = File.normalise(name).replace(/\\/g, "/");
+            const parent = File.normalise(inFolder).replace(/\\/g, "/");
+            const target = [parent, folderName]
+                .filter((s) => s && s !== ".")
+                .join("/");
+            const placeholder = target
+                ? `${target.replace(/\/$/, "")}/.keep`
+                : ".keep";
+            await (0, s3_helpers_1.uploadBuffer)(placeholder, Buffer.alloc(0), "application/octet-stream", {
+                min_role_read: DEFAULT_MIN_ROLE_READ,
+                filename: ".keep",
+                mime_super: "application",
+                mime_sub: "octet-stream",
+            });
+            return;
+        }
         const tenant = db_1.default.getTenantSchema();
         const safeDir = path.normalize(name).replace(/^(\.\.(\/|\\|$))+/, "");
         const safeInDir = path.normalize(inFolder).replace(/^(\.\.(\/|\\|$))+/, "");
@@ -345,12 +533,19 @@ class File {
      * Get path to serve
      */
     get path_to_serve() {
-        if (this.s3_store && this.id)
-            return this.id;
+        if (this.s3_store) {
+            const rel = File.normalise(this.location);
+            return rel.startsWith("/") ? rel.slice(1) : rel;
+        }
         const tenant = db_1.default.getTenantSchema();
         const s = this.location.replace(path.join(db_1.default.connectObj.file_store, tenant), "");
         return s[0] === "/" ? s.substring(1) : s;
     }
+    get field_value() {
+        if (this.isDirectory)
+            return this.path_to_serve;
+        return File.fieldValueFromRelative(this.path_to_serve);
+    }
     /**
      * Get current folder
      */
@@ -372,26 +567,45 @@ class File {
      * @param min_role_read target Role for file to read
      */
     async set_role(min_role_read) {
+        this.min_role_read = min_role_read;
+        if (this.s3_store) {
+            await this.persistS3Metadata();
+            return;
+        }
         if (this.id) {
             await File.update(this.id, { min_role_read });
         }
         else {
             await xattr_set(this.location, "user.saltcorn.min_role_read", `${min_role_read}`);
         }
-        this.min_role_read = min_role_read;
     }
     /**
      * Set user for file
      * @param user_id target user_id
      */
     async set_user(user_id) {
+        this.user_id = user_id;
+        if (this.s3_store) {
+            await this.persistS3Metadata();
+            return;
+        }
         if (this.id) {
             await File.update(this.id, { user_id });
         }
         else {
             await xattr_set(this.location, "user.saltcorn.user_id", `${user_id}`);
         }
-        this.user_id = user_id;
+    }
+    async persistS3Metadata() {
+        if (!this.s3_store)
+            return;
+        await (0, s3_helpers_1.setObjectMetadata)(this.location, {
+            min_role_read: this.min_role_read,
+            user_id: this.user_id,
+            mime_super: this.mime_super,
+            mime_sub: this.mime_sub,
+            filename: this.filename,
+        });
     }
     /**
      * Rename file
@@ -399,6 +613,28 @@ class File {
      */
     async rename(filenameIn) {
         const filename = File.normalise(filenameIn);
+        if (this.s3_store) {
+            const currentDir = this.location.includes("/")
+                ? posix.dirname(this.location)
+                : "";
+            const newRel = currentDir
+                ? `${currentDir}/${filename}`.replace(/\\/g, "/")
+                : filename;
+            await (0, s3_helpers_1.copyObject)(this.location, newRel, {
+                metadata: {
+                    min_role_read: this.min_role_read,
+                    user_id: this.user_id,
+                    mime_super: this.mime_super,
+                    mime_sub: this.mime_sub,
+                    filename,
+                },
+            });
+            await (0, s3_helpers_1.deleteObject)(this.location);
+            await File.update_table_references(this.path_to_serve, newRel);
+            this.location = newRel;
+            this.filename = filename;
+            return;
+        }
         if (this.id) {
             await File.update(this.id, { filename });
         }
@@ -420,11 +656,18 @@ class File {
         const Table = require("./table");
         const fileFields = await Field.find({ type: "File" }, { cached: true });
         const schema = db_1.default.getTenantSchemaPrefix();
+        const targetValue = File.fieldValueFromRelative(to);
+        const fromValues = new Set([from]);
+        if ((0, s3_helpers_1.isS3Enabled)()) {
+            fromValues.add(File.fieldValueFromRelative(from));
+        }
         for (const field of fileFields) {
             const table = Table.findOne({ id: field.table_id });
             if (table.external || table.provider_name)
                 continue;
-            await db_1.default.query(`update ${schema}"${db_1.default.sqlsanitize(table.name)}" set "${field.name}" = $1 where "${field.name}" = $2`, [to, from]);
+            for (const fromValue of fromValues) {
+                await db_1.default.query(`update ${schema}"${db_1.default.sqlsanitize(table.name)}" set "${field.name}" = $1 where "${field.name}" = $2`, [targetValue, fromValue]);
+            }
         }
     }
     /**
@@ -433,6 +676,26 @@ class File {
      */
     async move_to_dir(newFolder) {
         const newFolderNormd = File.normalise(newFolder);
+        if (this.s3_store) {
+            const relative = newFolderNormd
+                .replace(/\\/g, "/")
+                .replace(/^(\/)+/, "")
+                .replace(/\/$/, "");
+            const newRel = relative ? `${relative}/${this.filename}` : this.filename;
+            await (0, s3_helpers_1.copyObject)(this.location, newRel, {
+                metadata: {
+                    min_role_read: this.min_role_read,
+                    user_id: this.user_id,
+                    mime_super: this.mime_super,
+                    mime_sub: this.mime_sub,
+                    filename: this.filename,
+                },
+            });
+            await (0, s3_helpers_1.deleteObject)(this.location);
+            await File.update_table_references(this.path_to_serve, newRel);
+            this.location = newRel;
+            return;
+        }
         const tenant = db_1.default.getTenantSchema();
         const file_store = db_1.default.connectObj.file_store;
         const newPath = path.join(file_store, tenant, newFolderNormd, this.filename);
@@ -448,12 +711,13 @@ class File {
      * @returns {string} - path to file
      */
     static get_new_path(suggest, renameIfExisting) {
-        const { getState } = require("../db/state");
-        const state = getState();
-        // Check if it uses S3, then use a default "saltcorn" folder
-        const useS3 = state?.getConfig("storage_s3_enabled");
+        const useS3 = (0, s3_helpers_1.isS3Enabled)();
         const tenant = db_1.default.getTenantSchema();
-        const file_store = !useS3 ? db_1.default.connectObj.file_store : "saltcorn/";
+        if (useS3) {
+            const relative = (suggest || (0, uuid_1.v4)()).replace(/\\/g, "/");
+            return (0, s3_helpers_1.buildKeyFromRelative)(relative);
+        }
+        const file_store = db_1.default.connectObj.file_store;
         const newFnm = suggest || (0, uuid_1.v4)();
         let newPath = (0, path_1.join)(file_store, tenant, newFnm);
         if (renameIfExisting && fs.existsSync(newPath)) {
@@ -529,6 +793,12 @@ class File {
         }
     }
     async get_contents(encoding) {
+        if (this.s3_store) {
+            const buffer = await (0, s3_helpers_1.downloadBuffer)(this.location);
+            return encoding
+                ? buffer.toString(encoding)
+                : buffer;
+        }
         return await fsp.readFile(this.location, encoding);
     }
     /**
@@ -548,6 +818,27 @@ class File {
         const [mime_super, mime_sub] = mimetype.split("/");
         // move file in file system to newPath
         const contents1 = contents instanceof ArrayBuffer ? Buffer.from(contents) : contents;
+        if ((0, s3_helpers_1.isS3Enabled)()) {
+            const relativePath = File.relativeS3Path(newPath);
+            await (0, s3_helpers_1.uploadBuffer)(relativePath, Buffer.from(contents1), mimetype, {
+                min_role_read,
+                user_id,
+                mime_super,
+                mime_sub,
+                filename: path.basename(name),
+            });
+            return await File.create({
+                filename: path.basename(name),
+                location: relativePath,
+                uploaded_at: new Date(),
+                size_kb: Math.round(contents1.length / 1024),
+                user_id,
+                mime_super,
+                mime_sub,
+                min_role_read,
+                s3_store: true,
+            });
+        }
         await fsp.writeFile(newPath, contents1);
         // create file
         const file = await File.create({
@@ -564,7 +855,18 @@ class File {
         return file;
     }
     async overwrite_contents(contents) {
-        await fsp.writeFile(this.location, contents);
+        const data = contents instanceof ArrayBuffer ? Buffer.from(contents) : contents;
+        if (this.s3_store) {
+            await (0, s3_helpers_1.uploadBuffer)(this.location, Buffer.from(data), this.mimetype, {
+                min_role_read: this.min_role_read,
+                user_id: this.user_id,
+                mime_super: this.mime_super,
+                mime_sub: this.mime_sub,
+                filename: this.filename,
+            });
+            return;
+        }
+        await fsp.writeFile(this.location, data);
     }
     /**
      * Delete file
@@ -576,6 +878,20 @@ class File {
             if (this.id)
                 await db_1.default.deleteWhere("_sc_files", { id: this.id });
             // delete name and possible file from file system
+            if (this.s3_store) {
+                if (this.isDirectory) {
+                    const { files } = await (0, s3_helpers_1.listS3Folder)(this.location, {
+                        recursive: true,
+                    });
+                    for (const obj of files) {
+                        await (0, s3_helpers_1.deleteObject)(obj.relativePath);
+                    }
+                }
+                else {
+                    await (0, s3_helpers_1.deleteObject)(this.location);
+                }
+                return;
+            }
             if (unlinker)
                 await unlinker(this);
             else if (this.isDirectory) {
@@ -615,13 +931,15 @@ class File {
      */
     static async create(f) {
         const file = new File(f);
-        //const { id, ...rest } = file;
-        // insert file descriptor row to database
-        //file.id = await db.insert("_sc_files", rest);
-        // refresh file list cache
-        await file.set_role(file.min_role_read);
-        if (file.user_id)
-            await file.set_user(file.user_id);
+        if (file.s3_store) {
+            file.location = File.relativeS3Path(file.location);
+            await file.persistS3Metadata();
+        }
+        else {
+            await file.set_role(file.min_role_read);
+            if (file.user_id)
+                await file.set_user(file.user_id);
+        }
         return file;
     }
     /**