@saltcorn/server 1.5.0-beta.12 → 1.5.0-beta.14

package/routes/api.js

@@ -31,6 +31,7 @@ const File = require("@saltcorn/data/models/file");
 //const load_plugins = require("../load_plugins");
 const passport = require("passport");
 const path = require("path");
+const s3storage = require("../s3storage");
 
 const {
   readState,
@@ -223,14 +224,16 @@ router.get(
           file &&
           (role <= file.min_role_read || (user_id && user_id === file.user_id))
         ) {
+          if (file.s3_store) {
+            await s3storage.redirectToObject(file, res, false);
+            return;
+          }
           res.type(file.mimetype);
           const cacheability =
             file.min_role_read === 100 ? "public" : "private";
           const maxAge = getState().getConfig("files_cache_maxage", 86400);
           res.set("Cache-Control", `${cacheability}, max-age=${maxAge}`);
-          if (file.s3_store)
-            res.status(404).json({ error: req.__("Not found") });
-          else res.sendFile(file.location, { dotfiles: "allow" });
+          res.sendFile(file.location, { dotfiles: "allow" });
         } else {
           res.status(404).json({ error: req.__("Not found") });
         }
@@ -275,15 +278,20 @@ router.post(
             folder ? File.normalise(folder) : undefined
           );
           const many = Array.isArray(f);
+          const formatLocation = (fl) => File.fieldValueFromRelative(fl.path_to_serve);
+          const formatUrl = (loc, filename) =>
+            File.pathToServeUrl(loc, { filename });
           jsonResp = {
             success: {
               filename: many ? f.map((fl) => fl.filename) : f.filename,
               location: many
-                ? f.map((fl) => fl.path_to_serve)
-                : f.path_to_serve,
+                ? f.map((fl) => formatLocation(fl))
+                : formatLocation(f),
               url: many
-                ? f.map((fl) => `/files/serve/${fl.path_to_serve}`)
-                : `/files/serve/${f.path_to_serve}`,
+                ? f.map((fl) =>
+                    formatUrl(formatLocation(fl), fl.filename)
+                  )
+                : formatUrl(formatLocation(f), f.filename),
             },
           };
           res.json(jsonResp);

package/routes/files.js

@@ -29,6 +29,8 @@ const {
   style,
   link,
   domReady,
+  img,
+  a,
 } = require("@saltcorn/markup/tags");
 const { editRoleForm, fileUploadForm } = require("../markup/forms.js");
 const { strictParseInt } = require("@saltcorn/data/plugin-helper");
@@ -160,7 +162,7 @@ router.get(
     }
 
     for (const file of rows) {
-      file.location = file.path_to_serve;
+      file.location = file.isDirectory ? file.path_to_serve : file.field_value;
     }
     if (file_exts) {
       const re = new RegExp(
@@ -217,7 +219,9 @@ router.get(
     }
     if (req.xhr) {
       for (const file of rows) {
-        file.location = file.path_to_serve;
+        file.location = file.isDirectory
+          ? file.path_to_serve
+          : file.field_value;
       }
       const directories = await File.allDirectories(true);
       for (const file of directories) {
@@ -268,9 +272,12 @@ router.get(
       file &&
       (role <= file.min_role_read || (user_id && user_id === file.user_id))
     ) {
+      if (file.s3_store) {
+        await s3storage.redirectToObject(file, res, true);
+        return;
+      }
       res.type(file.mimetype);
-      if (file.s3_store) s3storage.serveObject(file, res, true);
-      else res.download(file.location, file.filename, { dotfiles: "allow" });
+      res.download(file.location, file.filename, { dotfiles: "allow" });
     } else {
       res
         .status(404)
@@ -308,6 +315,62 @@ router.post(
   })
 );
 
+router.get(
+  "/view/*serve_path",
+  error_catcher(async (req, res) => {
+    const role = req.user && req.user.id ? req.user.role_id : 100;
+    const user_id = req.user && req.user.id;
+    const serve_path = path.join(...req.params.serve_path);
+    const file = await File.findOne(serve_path);
+    const authorized =
+      file &&
+      (role <= file.min_role_read || (user_id && user_id === file.user_id));
+    if (!file || !authorized) {
+      res
+        .status(404)
+        .sendWrap(req.__("Not found"), h1(req.__("File not found")));
+      return;
+    }
+    if (file.mime_super !== "image") {
+      res.redirect(`/files/serve/${file.path_to_serve}`);
+      return;
+    }
+    let imgSrc;
+    if (file.s3_store) {
+      try {
+        imgSrc = await s3storage.getObjectUrl(file, false);
+      } catch (e) {
+        getState().log(3, e?.message || e);
+        res.redirect(`/files/serve/${file.path_to_serve}`);
+        return;
+      }
+    } else {
+      imgSrc = `/files/serve/${file.path_to_serve}`;
+    }
+    const downloadUrl = `/files/download/${file.path_to_serve}`;
+    res.sendWrap(req.__("Image preview"), {
+      type: "card",
+      contents: [
+        div(
+          { class: "mb-3" },
+          a(
+            { href: downloadUrl, class: "btn btn-secondary" },
+            req.__("Download")
+          )
+        ),
+        div(
+          { class: "text-center" },
+          img({
+            src: imgSrc,
+            alt: file.filename,
+            style: "max-width: 100%; height: auto;",
+          })
+        ),
+      ],
+    });
+  })
+);
+
 /**
  * @name get/serve/:id
  * @function
@@ -328,6 +391,10 @@ router.get(
       file &&
       (role <= file.min_role_read || (user_id && user_id === file.user_id))
     ) {
+      if (file.s3_store) {
+        await s3storage.redirectToObject(file, res, false);
+        return;
+      }
       if (
         (file.mimetype === "text/html" ||
           file.mimetype === "application/xhtml+xml") &&
@@ -350,8 +417,7 @@ router.get(
         res.send(clean);
         return;
       }
-      if (file.s3_store) s3storage.serveObject(file, res, false);
-      else res.sendFile(file.location, { dotfiles: "allow" });
+      res.sendFile(file.location, { dotfiles: "allow" });
     } else {
       getState().log(
         5,
@@ -386,6 +452,10 @@ router.get(
       file &&
       (role <= file.min_role_read || (user_id && user_id === file.user_id))
     ) {
+      if (file.s3_store) {
+        await s3storage.redirectToObject(file, res, false);
+        return;
+      }
       if (
         (file.mimetype === "text/html" ||
           file.mimetype === "application/xhtml+xml") &&
@@ -396,31 +466,27 @@ router.get(
 
       const cacheability = file.min_role_read === 100 ? "public" : "private";
       res.set("Cache-Control", `${cacheability}, max-age=86400`);
-      //TODO s3
-      if (file.s3_store) s3storage.serveObject(file, res, false);
-      else {
-        const width = strictParseInt(width_str);
-        const height =
-          height_str && height_str !== "0" ? strictParseInt(height_str) : null;
-        if (!width) {
-          res.sendFile(file.location, { dotfiles: "allow" });
-          return;
-        }
-        const basenm = path.join(
-          path.dirname(file.location),
-          "_resized_" + path.basename(file.location)
-        );
-        const fnm = `${basenm}_w${width}${height ? `_h${height}` : ""}`;
-        if (!fs.existsSync(fnm)) {
-          await resizer({
-            fromFileName: file.location,
-            width,
-            height,
-            toFileName: fnm,
-          });
-        }
-        res.sendFile(fnm, { dotfiles: "allow" });
+      const width = strictParseInt(width_str);
+      const height =
+        height_str && height_str !== "0" ? strictParseInt(height_str) : null;
+      if (!width) {
+        res.sendFile(file.location, { dotfiles: "allow" });
+        return;
+      }
+      const basenm = path.join(
+        path.dirname(file.location),
+        "_resized_" + path.basename(file.location)
+      );
+      const fnm = `${basenm}_w${width}${height ? `_h${height}` : ""}`;
+      if (!fs.existsSync(fnm)) {
+        await resizer({
+          fromFileName: file.location,
+          width,
+          height,
+          toFileName: fnm,
+        });
       }
+      res.sendFile(fnm, { dotfiles: "allow" });
     } else {
       res
         .status(404)
@@ -563,15 +629,16 @@ router.post(
         `File %s uploaded`,
         many ? f.map((fl) => text(fl.filename)).join(", ") : text(f.filename)
       );
+      const asLocation = (fl) => File.fieldValueFromRelative(fl.path_to_serve);
+      const asUrl = (fl) =>
+        File.pathToServeUrl(asLocation(fl), { filename: fl.filename });
       if (!req.xhr) req.flash("success", successMsg);
       else
         jsonResp = {
           success: {
             filename: many ? f.map((fl) => fl.filename) : f.filename,
-            location: many ? f.map((fl) => fl.path_to_serve) : f.path_to_serve,
-            url: many
-              ? f.map((fl) => `/files/serve/${fl.path_to_serve}`)
-              : `/files/serve/${f.path_to_serve}`,
+            location: many ? f.map(asLocation) : asLocation(f),
+            url: many ? f.map(asUrl) : asUrl(f),
             msg: successMsg,
           },
         };
@@ -638,12 +705,12 @@ const storage_form = async (req) => {
     field_names: [
       "storage_s3_enabled",
       "storage_s3_bucket",
-      "storage_s3_path_prefix",
       "storage_s3_endpoint",
       "storage_s3_region",
       "storage_s3_access_key",
       "storage_s3_access_secret",
       "storage_s3_secure",
+      "files_direct_s3_links",
     ],
     action: "/files/storage",
   });
@@ -660,7 +727,7 @@ router.get(
   error_catcher(async (req, res) => {
     const form = await storage_form(req);
     form.blurb = [
-      `<div class="alert alert-warning">S3 storage options may not work for this release. Enabling S3 storage is not recommended</div>`,
+      `<div class="alert alert-warning">S3 storage is experimental</div>`,
     ];
     send_files_page({
       res,
@@ -721,6 +788,7 @@ const files_settings_form = async (req) => {
       "min_role_upload",
       "file_accept_filter_default",
       "files_cache_maxage",
+      // "files_direct_s3_links",
       "file_upload_debug",
       "file_upload_limit",
       "file_upload_timeout",

package/routes/notifications.js

@@ -14,6 +14,7 @@ const Form = require("@saltcorn/data/models/form");
 const File = require("@saltcorn/data/models/file");
 const User = require("@saltcorn/data/models/user");
 const Trigger = require("@saltcorn/data/models/trigger");
+const Table = require("@saltcorn/data/models/table");
 const { renderForm, post_btn } = require("@saltcorn/markup");
 const db = require("@saltcorn/data/db");
 
@@ -23,7 +24,10 @@ module.exports = router;
 router.use(
   error_catcher(async (req, res, next) => {
     const state = getState();
-    const maintenanceModeEnabled = state.getConfig("maintenance_mode_enabled", false);
+    const maintenanceModeEnabled = state.getConfig(
+      "maintenance_mode_enabled",
+      false
+    );
     if (maintenanceModeEnabled && (!req.user || req.user.role_id > 1)) {
       res.status(503).send("Page Unavailable: in maintenance mode");
       return;
@@ -348,55 +352,45 @@ router.post(
 );
 
 router.post(
-  "/fcm-token",
+  "/mobile-subscribe",
   loggedIn,
   error_catcher(async (req, res) => {
-    const enabled = getState().getConfig("enable_push_notify", false);
-    if (!enabled) {
-      res.status(403).json({
-        error: req.__("Notifications are not enabled on this server"),
+    const { token, deviceId, synchedTables } = req.body || {};
+    if (!token) {
+      res.status(400).json({
+        error: req.__("FCM token is required"),
+      });
+      return;
+    }
+    const user = req.user;
+    const allSubs = getState().getConfig("push_notification_subscriptions", {});
+    let userSubs = allSubs[user.id] || [];
+    const existingSub = userSubs.find(
+      (s) =>
+        s.type === "fcm-push" && s.token === token && s.deviceId === deviceId
+    );
+    if (existingSub) {
+      res.json({
+        success: "ok",
+        message: req.__("FCM token already uploaded"),
       });
     } else {
-      const { token, deviceId } = req.body || {};
-      if (!token) {
-        res.status(400).json({
-          error: req.__("FCM token is required"),
-        });
-        return;
-      }
-      const user = req.user;
-      const allSubs = getState().getConfig(
-        "push_notification_subscriptions",
-        {}
-      );
-      let userSubs = allSubs[user.id] || [];
-      const existingSub = userSubs.find(
-        (s) =>
-          s.type === "fcm-push" && s.token === token && s.deviceId === deviceId
+      userSubs = userSubs.filter(
+        (s) => s.type !== "fcm-push" || s.deviceId !== deviceId
       );
-      if (existingSub) {
-        res.json({
-          success: "ok",
-          message: req.__("FCM token already uploaded"),
-        });
-      } else {
-        userSubs = userSubs.filter(
-          (s) => s.type !== "fcm-push" || s.deviceId !== deviceId
-        );
-        userSubs.push({
-          type: "fcm-push",
-          token: token,
-          deviceId: deviceId,
-        });
-        await getState().setConfig("push_notification_subscriptions", {
-          ...allSubs,
-          [user.id]: userSubs,
-        });
-        res.json({
-          success: "ok",
-          message: req.__("FCM token uploaded"),
-        });
-      }
+      userSubs.push({
+        type: "fcm-push",
+        token: token,
+        deviceId: deviceId,
+      });
+      await getState().setConfig("push_notification_subscriptions", {
+        ...allSubs,
+        [user.id]: userSubs,
+      });
+      res.json({
+        success: "ok",
+        message: req.__("FCM token uploaded"),
+      });
     }
   })
 );
@@ -438,6 +432,48 @@ router.post(
   })
 );
 
+// TODO
+router.post(
+  "/mobile-remove-subscription",
+  loggedIn,
+  error_catcher(async (req, res) => {
+    res.json({
+      success: "ok",
+      message: req.__("Unsubscribed from notifications"),
+    });
+  })
+);
+
+router.post(
+  "/mobile-remove-subscription",
+  loggedIn,
+  error_catcher(async (req, res) => {
+    const { token, deviceId } = req.body || {};
+    if (!token) {
+      res.status(400).json({
+        error: req.__("FCM token is required"),
+      });
+      return;
+    }
+    const user = req.user;
+    const oldSubs = getState().getConfig("push_notification_subscriptions", {});
+    let userSubs = oldSubs[user.id];
+    if (userSubs) {
+      userSubs = userSubs.filter(
+        (s) => s.type !== "fcm-push" || s.deviceId !== deviceId
+      );
+      await getState().setConfig("push_notification_subscriptions", {
+        ...oldSubs,
+        [user.id]: userSubs,
+      });
+      res.json({
+        success: "ok",
+        message: req.__("Unsubscribed from notifications"),
+      });
+    }
+  })
+);
+
 router.post(
   "/generate-vapid-keys",
   isAdmin,
@@ -497,7 +533,9 @@ router.get(
     }
     if (Array.isArray(pwa_icons) && pwa_icons.length > 0)
       manifest.icons = pwa_icons.map(({ image, size, maskable }) => ({
-        src: `/files/serve/${image}`,
+        src: /^(?:[a-z]+:)?\/\//i.test(image)
+          ? image
+          : `/files/serve/${image}`,
         type: File.nameToMimeType(image),
         sizes: size ? `${size}x${size}` : "144x144",
         ...(maskable ? { purpose: "maskable" } : {}),
@@ -505,7 +543,9 @@ router.get(
     else if (site_logo)
       manifest.icons = [
         {
-          src: `/files/serve/${site_logo}`,
+          src: /^(?:[a-z]+:)?\/\//i.test(site_logo)
+            ? site_logo
+            : `/files/serve/${site_logo}`,
           type: File.nameToMimeType(site_logo),
           sizes: "144x144",
         },

package/routes/pageedit.js

@@ -172,7 +172,7 @@ const pageBuilderData = async (req, context) => {
   const pages = await Page.find();
   const page_groups = (await PageGroup.find()).map((g) => ({ name: g.name }));
   const images = await File.find({ mime_super: "image" });
-  images.forEach((im) => (im.location = im.path_to_serve));
+  images.forEach((im) => (im.location = im.field_value));
   const roles = await User.get_roles();
   const stateActions = getState().actions;
   const actions = [

package/routes/sync.js

@@ -1,4 +1,4 @@
-const { error_catcher } = require("./utils.js");
+const { error_catcher, loggedIn } = require("./utils.js");
 const Router = require("express-promise-router");
 const db = require("@saltcorn/data/db");
 const { getState } = require("@saltcorn/data/db/state");
@@ -337,3 +337,81 @@ router.post(
     }
   })
 );
+
+router.post(
+  "/push_subscribe",
+  loggedIn,
+  error_catcher(async (req, res) => {
+    const { token, deviceId, synchedTables } = req.body || {};
+    if (!token) {
+      res.status(400).json({
+        error: req.__("FCM token is required"),
+      });
+      return;
+    }
+
+    const user = req.user;
+    const state = getState();
+    const allSubs = state.getConfig("push_sync_subscriptions", {});
+    let userSubs = allSubs[user.id] || [];
+    const existingSub = userSubs.find(
+      (s) => s.token === token && s.deviceId === deviceId
+    );
+    if (existingSub) {
+      res.json({
+        success: "ok",
+        message: req.__("sync push subscription already exists"),
+      });
+    } else {
+      userSubs = userSubs.filter((s) => s.deviceId !== deviceId);
+      userSubs.push({
+        token,
+        deviceId,
+        synchedTables,
+      });
+      await getState().setConfig("push_sync_subscriptions", {
+        ...allSubs,
+        [user.id]: userSubs,
+      });
+      res.json({
+        success: "ok",
+        message: req.__("sync push subscription saved"),
+      });
+    }
+  })
+);
+
+router.post(
+  "/push_unsubscribe",
+  loggedIn,
+  error_catcher(async (req, res) => {
+    const { token, deviceId } = req.body || {};
+    if (!token) {
+      res.status(400).json({
+        error: req.__("FCM token is required"),
+      });
+      return;
+    }
+
+    const user = req.user;
+    const state = getState();
+    const allSubs = state.getConfig("push_sync_subscriptions", {});
+    let userSubs = allSubs[user.id] || [];
+    const newUserSubs = userSubs.filter((s) => s.deviceId !== deviceId);
+    if (newUserSubs.length === userSubs.length) {
+      res.json({
+        success: "ok",
+        message: req.__("FCM token not found"),
+      });
+    } else {
+      await getState().setConfig("push_sync_subscriptions", {
+        ...allSubs,
+        [user.id]: newUserSubs,
+      });
+      res.json({
+        success: "ok",
+        message: req.__("sync push subscription removed"),
+      });
+    }
+  })
+);

package/s3storage.js

@@ -1,21 +1,14 @@
-const { S3 } = require("@aws-sdk/client-s3");
 const multer = require("multer");
 const multerS3 = require("multer-s3");
 const { getState } = require("@saltcorn/data/db/state");
+const {
+  getS3Client,
+  getServeUrl: getS3ServeUrl,
+  deleteObject: deleteS3Object,
+} = require("@saltcorn/data/models/internal/s3_helpers");
 const fileUpload = require("express-fileupload");
 const { v4: uuidv4 } = require("uuid");
-const contentDisposition = require("content-disposition");
 const fs = require("fs");
-function createS3Client() {
-  return new S3({
-    credentials: {
-      secretAccessKey: getState().getConfig("storage_s3_access_secret"),
-      accessKeyId: getState().getConfig("storage_s3_access_key"),
-    },
-    region: getState().getConfig("storage_s3_region") || "us-east-1",
-    endpoint: getState().getConfig("storage_s3_endpoint"),
-  });
-}
 
 module.exports = {
   /**
@@ -34,7 +27,7 @@ module.exports = {
       // Create multer function
       const s3upload = multer({
         storage: multerS3({
-          s3: createS3Client(),
+          s3: getS3Client(),
           bucket: getState().getConfig("storage_s3_bucket"),
           metadata: function (req, file, cb) {
             cb(null, { fieldName: file.fieldname });
@@ -91,7 +84,7 @@ module.exports = {
     }
 
     // Create S3 object
-    const s3 = createS3Client();
+    const s3 = getS3Client();
     const bucket = getState().getConfig("storage_s3_bucket");
 
     let newFileObject = {};
@@ -140,44 +133,35 @@ module.exports = {
    */
   serveObject: function (file, res, download) {
     if (file.s3_store) {
-      const s3 = createS3Client();
-      const bucket = getState().getConfig("storage_s3_bucket");
-
-      const params = {
-        Bucket: bucket,
-        Key: file.location,
-      };
-
-      // Forward the object
-      s3.getObject(params)
-        .on("httpHeaders", function (statusCode, headers) {
-          if (download)
-            res.set("Content-Disposition", contentDisposition(file.filename));
-          res.set("Content-Length", headers["content-length"]);
-          this.response.httpResponse.createUnbufferedStream().pipe(res);
-        })
-        .send();
+      module.exports
+        .redirectToObject(file, res, download)
+        .catch((e) => {
+          getState().log(3, e.message || e);
+          res.status(500).send("Unable to redirect to object");
+        });
     } else {
-      // Use legacy file download
       res.download(file.location, file.filename, { dotfiles: "allow" });
     }
   },
 
+  redirectToObject: async function (file, res, download) {
+    const url = await getS3ServeUrl(file.location, {
+      download,
+      filename: file.filename,
+    });
+    res.redirect(url);
+  },
+
+  getObjectUrl: async function (file, download) {
+    return await getS3ServeUrl(file.location, {
+      download,
+      filename: file.filename,
+    });
+  },
+
   unlinkObject: function (file) {
     if (file.s3_store) {
-      const s3 = createS3Client();
-      return new Promise((resolve, reject) => {
-        s3.deleteObject(
-          {
-            Bucket: getState().getConfig("storage_s3_bucket"),
-            Key: file.location,
-          },
-          (err, data) => {
-            if (err) reject(err);
-            else resolve(data);
-          }
-        );
-      });
+      return deleteS3Object(file.location);
     } else {
       return fs.unlink(file.location);
     }

package/tests/api.test.js

@@ -866,7 +866,6 @@ describe("API upload-files", () => {
     expect(f).toBeDefined();
     expect(f.min_role_read).toBe(80);
     expect(f.user_id).toBe(1);
-    expect(f.path_to_serve).toBe(body.success.location);
   });
 
   it("should not allow public upload", async () => {