@blocklet/uploader-server 0.1.49

package/es/middlewares/static-resource.js

@@ -0,0 +1,107 @@
+const { existsSync } = require("fs");
+const { join } = require("path");
+const config = require("@blocklet/sdk/lib/config");
+const { getResources } = require("@blocklet/sdk/lib/component");
+const logger = console;
+const ImgResourceType = "imgpack";
+const ImageBinDid = "z8ia1mAXo8ZE7ytGF36L5uBf9kD2kenhqFGp9";
+let skipRunningCheck = false;
+let mediaKitInfo = null;
+let resourceTypes = [
+  {
+    type: ImgResourceType,
+    did: ImageBinDid,
+    folder: ""
+    // default is root, can be set to 'public' or 'assets' or any other folder
+  }
+];
+let canUseResources = [];
+export const mappingResource = async () => {
+  mediaKitInfo = config.components.find((item) => item.did === ImageBinDid);
+  if (mediaKitInfo) {
+    mediaKitInfo.uploadsDir = config.env.dataDir.replace(/\/[^/]*$/, "/image-bin/uploads");
+  }
+  try {
+    const resources = getResources({
+      types: resourceTypes,
+      skipRunningCheck
+    });
+    canUseResources = resources.map((resource) => {
+      const originDir = resource.path;
+      const resourceType = resourceTypes.find(({ type }) => originDir.endsWith(type));
+      if (!existsSync(originDir) || !resourceType) {
+        return false;
+      }
+      const { folder = "" } = resourceType;
+      return { originDir, dir: join(originDir, folder || ""), blockletInfo: resource };
+    }).filter(Boolean);
+    logger.info(
+      "Mapping can use resources count: ",
+      canUseResources.length
+      // canUseResources
+    );
+    return canUseResources;
+  } catch (error) {
+    console.error(error);
+  }
+  return false;
+};
+const { events, Events } = config;
+events.on(Events.componentAdded, () => mappingResource());
+events.on(Events.componentRemoved, () => mappingResource());
+events.on(Events.componentStarted, () => mappingResource());
+events.on(Events.componentStopped, () => mappingResource());
+events.on(Events.componentUpdated, () => mappingResource());
+export const initStaticResourceMiddleware = ({
+  options,
+  resourceTypes: _resourceTypes = resourceTypes,
+  express,
+  skipRunningCheck: _skipRunningCheck
+} = {}) => {
+  skipRunningCheck = !!_skipRunningCheck;
+  if (_resourceTypes?.length > 0) {
+    resourceTypes = _resourceTypes.map((item) => {
+      if (typeof item === "string") {
+        return {
+          type: item,
+          did: ImageBinDid,
+          // not set did, default is ImageBinDid
+          folder: ""
+          // not set folder, default is root
+        };
+      }
+      return item;
+    });
+  }
+  mappingResource();
+  return (req, res, next) => {
+    const urlPath = new URL(`http://localhost${req.url}`).pathname;
+    const matchCanUseResourceItem = canUseResources.find((item) => existsSync(join(item.dir, urlPath)));
+    if (matchCanUseResourceItem) {
+      express.static(matchCanUseResourceItem.dir, {
+        maxAge: "365d",
+        immutable: true,
+        index: false,
+        // fallthrough: false,
+        ...options
+      })(req, res, next);
+    } else {
+      res.status(404).end();
+    }
+  };
+};
+export const getCanUseResources = () => canUseResources;
+export const initProxyToMediaKitUploadsMiddleware = ({ options, express } = {}) => {
+  return (req, res, next) => {
+    if (!mediaKitInfo?.uploadsDir) {
+      return next();
+    }
+    return express.static(mediaKitInfo.uploadsDir, {
+      maxAge: "365d",
+      immutable: true,
+      index: false,
+      // fallthrough: false,
+      ...options
+    })(req, res, next);
+  };
+};

package/es/middlewares.d.ts

@@ -0,0 +1,3 @@
+export * from './middlewares/local-storage';
+export * from './middlewares/companion';
+export * from './middlewares/static-resource';

package/es/middlewares.js

@@ -0,0 +1,3 @@
+export * from "./middlewares/local-storage.js";
+export * from "./middlewares/companion.js";
+export * from "./middlewares/static-resource.js";

package/lib/index.d.ts

@@ -0,0 +1 @@
+export * from './middlewares';

package/lib/index.js

@@ -0,0 +1,16 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+var _middlewares = require("./middlewares");
+Object.keys(_middlewares).forEach(function (key) {
+  if (key === "default" || key === "__esModule") return;
+  if (key in exports && exports[key] === _middlewares[key]) return;
+  Object.defineProperty(exports, key, {
+    enumerable: true,
+    get: function () {
+      return _middlewares[key];
+    }
+  });
+});

package/lib/middlewares/companion.d.ts

@@ -0,0 +1,6 @@
+export declare function initCompanion({ path, express, providerOptions, ...restProps }: {
+    path: string;
+    express: Function;
+    providerOptions?: Object;
+}): any;
+export declare function proxyImageDownload(req: any, res: any, next?: Function): Promise<void>;

package/lib/middlewares/companion.js

@@ -0,0 +1,104 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.initCompanion = initCompanion;
+exports.proxyImageDownload = proxyImageDownload;
+const companion = require("@uppy/companion");
+const bodyParser = require("body-parser");
+const session = require("express-session");
+const axios = require("axios");
+const crypto = require("crypto");
+const secret = crypto.randomBytes(32).toString("hex");
+function initCompanion({
+  path,
+  express,
+  providerOptions,
+  ...restProps
+}) {
+  const app = express();
+  app.use(bodyParser.json());
+  app.use(session({
+    secret
+  }));
+  app.use("/proxy", proxyImageDownload);
+  let dynamicProviderOptions = providerOptions;
+  const companionOptions = {
+    secret,
+    providerOptions,
+    // unused
+    server: {
+      protocol: "https",
+      host: "UNUSED_HOST",
+      // unused
+      path: "UNUSED_PATH"
+      // unused
+    },
+    filePath: path,
+    streamingUpload: true,
+    metrics: false,
+    ...restProps
+  };
+  const newCompanion = companion.app(companionOptions);
+  const {
+    app: companionApp
+  } = newCompanion;
+  app.all("*", (req, res, next) => {
+    let hackerCompanion = {};
+    Object.defineProperty(req, "companion", {
+      get() {
+        return hackerCompanion;
+      },
+      set(value) {
+        hackerCompanion = value;
+        hackerCompanion.options.providerOptions = dynamicProviderOptions;
+      }
+    });
+    next();
+  }, companionApp);
+  newCompanion.handle = app;
+  newCompanion.setProviderOptions = options => {
+    dynamicProviderOptions = options;
+  };
+  return newCompanion;
+}
+async function proxyImageDownload(req, res, next) {
+  let {
+    url,
+    responseType = "stream"
+  } = {
+    ...req.query,
+    ...req.body
+  };
+  if (url) {
+    url = encodeURI(url);
+    try {
+      const {
+        headers,
+        data,
+        status
+      } = await axios.get(url, {
+        responseType
+      });
+      if (data && status >= 200 && status < 302) {
+        res.setHeader("Content-Type", headers["content-type"]);
+        try {} catch (error) {}
+        if (responseType === "stream") {
+          data.pipe(res);
+        } else if (responseType === "arraybuffer") {
+          res.end(data?.toString?.("binary"), "binary");
+        } else {
+          res.send(data);
+        }
+      } else {
+        throw new Error("download image error");
+      }
+    } catch (err) {
+      console.error("Proxy url failed: ", err);
+      res.status(500).send("Proxy url failed");
+    }
+  } else {
+    res.status(500).send('Parameter "url" is required');
+  }
+}

package/lib/middlewares/local-storage.d.ts

@@ -0,0 +1,18 @@
+import { type ServerOptions } from '@tus/server';
+export declare function initLocalStorageServer({ path: _path, onUploadFinish: _onUploadFinish, onUploadCreate: _onUploadCreate, express, expiredUploadTime, // default 3 days expire
+...restProps }: ServerOptions & {
+    path: string;
+    onUploadFinish?: Function;
+    onUploadCreate?: Function;
+    express: Function;
+    expiredUploadTime?: Number;
+}): any;
+export declare const getFileName: (req: any) => any;
+export declare function getFileNameParam(req: any, res: any, { isRequired }?: {
+    isRequired: boolean;
+}): any;
+export declare function getLocalStorageFile({ server }: any): (req: any, res: any, next: any) => Promise<void>;
+export declare function setHeaders(req: any, res: any, next?: Function): void;
+export declare function fileExistBeforeUpload(req: any, res: any, next?: Function): Promise<void>;
+export declare function getMetaDataByFilePath(filePath: string): Promise<any>;
+export declare function joinUrl(...args: string[]): any;

package/lib/middlewares/local-storage.js

@@ -0,0 +1,423 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.fileExistBeforeUpload = fileExistBeforeUpload;
+exports.getFileName = void 0;
+exports.getFileNameParam = getFileNameParam;
+exports.getLocalStorageFile = getLocalStorageFile;
+exports.getMetaDataByFilePath = getMetaDataByFilePath;
+exports.initLocalStorageServer = initLocalStorageServer;
+exports.joinUrl = joinUrl;
+exports.setHeaders = setHeaders;
+const {
+  Server,
+  EVENTS
+} = require("@tus/server");
+const {
+  FileStore
+} = require("@tus/file-store");
+const cron = require("@abtnode/cron");
+const fs = require("fs").promises;
+const path = require("path");
+const crypto = require("crypto");
+const mime = require("mime-types");
+const joinUrlLib = require("url-join");
+const {
+  default: queue
+} = require("p-queue");
+const validFilePathInDirPath = (dirPath, filePath) => {
+  const fileName = path.basename(filePath);
+  if (!filePath.startsWith(dirPath) || path.join(dirPath, fileName) !== filePath) {
+    console.error("Invalid file path: ", filePath);
+    throw new Error("Invalid file path");
+  }
+  return true;
+};
+function initLocalStorageServer({
+  path: _path,
+  onUploadFinish: _onUploadFinish,
+  onUploadCreate: _onUploadCreate,
+  express,
+  expiredUploadTime = 1e3 * 60 * 60 * 24 * 3,
+  // default 3 days expire
+  ...restProps
+}) {
+  const app = express();
+  const configstore = new RewriteFileConfigstore(_path);
+  const datastore = new RewriteFileStore({
+    directory: _path,
+    expirationPeriodInMilliseconds: expiredUploadTime,
+    configstore
+  });
+  const formatMetadata = uploadMetadata => {
+    const cloneUploadMetadata = {
+      ...uploadMetadata
+    };
+    if (cloneUploadMetadata.metadata?.name?.indexOf("/") > -1 && cloneUploadMetadata.metadata?.relativePath?.indexOf("/") > -1) {
+      cloneUploadMetadata.metadata.name = cloneUploadMetadata.metadata.name.split("/").pop();
+      cloneUploadMetadata.metadata.filename = cloneUploadMetadata.metadata.name;
+    }
+    if (cloneUploadMetadata.id && !cloneUploadMetadata.runtime) {
+      const {
+        id,
+        metadata,
+        size
+      } = cloneUploadMetadata;
+      cloneUploadMetadata.runtime = {
+        relativePath: metadata?.relativePath,
+        absolutePath: path.join(_path, id),
+        size,
+        hashFileName: id,
+        originFileName: metadata?.filename,
+        type: metadata?.type,
+        fileType: metadata?.filetype
+      };
+    }
+    return cloneUploadMetadata;
+  };
+  const rewriteMetaDataFile = async uploadMetadata => {
+    uploadMetadata = formatMetadata(uploadMetadata);
+    const {
+      id
+    } = uploadMetadata;
+    if (!id) {
+      return;
+    }
+    const oldMetadata = formatMetadata(await configstore.get(id));
+    if (JSON.stringify(oldMetadata) !== JSON.stringify(uploadMetadata)) {
+      await configstore.set(id, uploadMetadata);
+    }
+  };
+  const onUploadCreate = async (req, res, uploadMetadata) => {
+    uploadMetadata = formatMetadata(uploadMetadata);
+    await rewriteMetaDataFile(uploadMetadata);
+    if (uploadMetadata.offset === 0 && uploadMetadata.size === 0) {
+      res.status(200);
+      res.setHeader("Location", joinUrl(req.headers["x-uploader-base-url"], uploadMetadata.id));
+      res.setHeader("Upload-Offset", 0);
+      res.setHeader("Upload-Length", 0);
+      res.setHeader("x-uploader-file-exist", true);
+    }
+    if (_onUploadCreate) {
+      const result = await _onUploadCreate(req, res, uploadMetadata);
+      return result;
+    }
+    return res;
+  };
+  const onUploadFinish = async (req, res, uploadMetadata) => {
+    res.setHeader("x-uploader-file-exist", true);
+    uploadMetadata = formatMetadata(uploadMetadata);
+    await rewriteMetaDataFile(uploadMetadata);
+    if (_onUploadFinish) {
+      try {
+        const result = await _onUploadFinish(req, res, uploadMetadata);
+        return result;
+      } catch (err) {
+        console.error("@blocklet/uploader: onUploadFinish error: ", err);
+        newServer.delete(uploadMetadata.id);
+        res.setHeader("x-uploader-file-exist", false);
+        throw err;
+      }
+    }
+    return res;
+  };
+  const newServer = new Server({
+    path: "/",
+    // UNUSED
+    relativeLocation: true,
+    // respectForwardedHeaders: true,
+    namingFunction: req => {
+      const fileName = getFileName(req);
+      const filePath = path.join(_path, fileName);
+      validFilePathInDirPath(_path, filePath);
+      return fileName;
+    },
+    datastore,
+    onUploadFinish: async (req, res, uploadMetadata) => {
+      uploadMetadata = formatMetadata(uploadMetadata);
+      const result = await onUploadFinish(req, res, uploadMetadata);
+      if (result && !result.send) {
+        const body = typeof result === "string" ? result : JSON.stringify(result);
+        throw {
+          body,
+          status_code: 200
+        };
+      } else {
+        return result;
+      }
+    },
+    onUploadCreate,
+    ...restProps
+  });
+  app.use((req, res, next) => {
+    req.uploaderProps = {
+      server: newServer,
+      onUploadFinish,
+      onUploadCreate
+    };
+    next();
+  });
+  cron.init({
+    context: {},
+    jobs: [{
+      name: "auto-cleanup-expired-uploads",
+      time: "0 0 * * * *",
+      // each hour
+      fn: () => {
+        try {
+          newServer.cleanUpExpiredUploads().then(count => {
+            console.info(`@blocklet/uploader: cleanup expired uploads done: ${count}`);
+          }).catch(err => {
+            console.error(`@blocklet/uploader: cleanup expired uploads error`, err);
+          });
+        } catch (err) {
+          console.error(`@blocklet/uploader: cleanup expired uploads error`, err);
+        }
+      },
+      options: {
+        runOnInit: false
+      }
+    }],
+    onError: err => {
+      console.error("@blocklet/uploader: cleanup job failed", err);
+    }
+  });
+  newServer.delete = async key => {
+    try {
+      await configstore.delete(key);
+      await configstore.delete(key, false);
+    } catch (err) {
+      console.error("@blocklet/uploader: delete error: ", err);
+    }
+  };
+  newServer.on(EVENTS.POST_RECEIVE, async (req, res, uploadMetadata) => {
+    uploadMetadata = formatMetadata(uploadMetadata);
+    await rewriteMetaDataFile(uploadMetadata);
+  });
+  app.all("*", setHeaders, fileExistBeforeUpload, newServer.handle.bind(newServer));
+  newServer.handle = app;
+  return newServer;
+}
+const getFileName = req => {
+  const ext = req.headers["x-uploader-file-ext"];
+  const randomName = `${crypto.randomBytes(16).toString("hex")}${ext ? `.${ext}` : ""}`;
+  return req.headers["x-uploader-file-name"] || randomName;
+};
+exports.getFileName = getFileName;
+function getFileNameParam(req, res, {
+  isRequired = true
+} = {}) {
+  let {
+    fileName
+  } = req.params;
+  if (!fileName) {
+    fileName = req.originalUrl.replace(req.baseUrl, "");
+  }
+  if (!fileName && isRequired) {
+    res.status(400).json({
+      error: 'Parameter "fileName" is required'
+    });
+    return;
+  }
+  return fileName;
+}
+function getLocalStorageFile({
+  server
+}) {
+  return async (req, res, next) => {
+    const fileName = getFileNameParam(req, res);
+    const filePath = path.join(server.datastore.directory, fileName);
+    validFilePathInDirPath(server.datastore.directory, filePath);
+    const fileExists = await fs.stat(filePath).catch(() => false);
+    if (!fileExists) {
+      res.status(404).json({
+        error: "file not found"
+      });
+      return;
+    }
+    setHeaders(req, res);
+    const file = await fs.readFile(filePath);
+    res.send(file);
+    next?.();
+  };
+}
+function setHeaders(req, res, next) {
+  let {
+    method
+  } = req;
+  method = method.toUpperCase();
+  const fileName = getFileNameParam(req, res, {
+    isRequired: false
+  });
+  if (req.headers["x-uploader-endpoint-url"]) {
+    const query = new URL(req.headers["x-uploader-endpoint-url"]).searchParams;
+    req.query = {
+      ...Object.fromEntries(query),
+      // query params convert to object
+      ...req.query
+    };
+  }
+  if (method === "POST" && req.headers["x-uploader-base-url"]) {
+    req.baseUrl = req.headers["x-uploader-base-url"];
+  }
+  if (method === "GET" && fileName) {
+    const contentType = mime.lookup(fileName);
+    if (contentType) {
+      res.setHeader("Content-Type", contentType);
+    }
+  }
+  next?.();
+}
+async function fileExistBeforeUpload(req, res, next) {
+  let {
+    method,
+    uploaderProps
+  } = req;
+  method = method.toUpperCase();
+  if (["PATCH", "POST"].includes(method)) {
+    const _path = uploaderProps.server.datastore.directory;
+    const fileName = getFileName(req);
+    const filePath = path.join(_path, fileName);
+    validFilePathInDirPath(_path, filePath);
+    const isExist = await fs.stat(filePath).catch(() => false);
+    if (isExist) {
+      const metaData = await getMetaDataByFilePath(filePath);
+      if (isExist?.size >= 0 && isExist?.size === metaData?.size) {
+        const prepareUpload = method === "POST";
+        if (prepareUpload) {
+          res.status(200);
+          res.setHeader("Location", joinUrl(req.headers["x-uploader-base-url"], fileName));
+          res.setHeader("Upload-Offset", +metaData.offset);
+          res.setHeader("Upload-Length", +metaData.size);
+        }
+        if (req.headers["x-uploader-metadata"]) {
+          try {
+            const realMetaData = JSON.parse(req.headers["x-uploader-metadata"], (key, value) => {
+              if (typeof value === "string") {
+                return decodeURIComponent(value);
+              }
+              return value;
+            });
+            metaData.metadata = {
+              ...metaData.metadata,
+              ...realMetaData
+            };
+          } catch (err) {
+            console.error("@blocklet/uploader: parse metadata error: ", err);
+          }
+        }
+        const uploadResult = await uploaderProps.onUploadFinish(req, res, metaData);
+        res.json(uploadResult);
+        return;
+      }
+    }
+  }
+  next?.();
+}
+async function getMetaDataByFilePath(filePath) {
+  const metaDataPath = `${filePath}.json`;
+  const isExist = await fs.stat(filePath).catch(() => false);
+  if (isExist) {
+    try {
+      const metaData = await fs.readFile(metaDataPath, "utf-8");
+      const metaDataJson = JSON.parse(metaData);
+      return metaDataJson;
+    } catch (err) {
+      console.error("@blocklet/uploader: getMetaDataByPath error: ", err);
+    }
+  }
+  return null;
+}
+function joinUrl(...args) {
+  const realArgs = args.filter(Boolean).map(item => {
+    if (item === "/") {
+      return "";
+    }
+    return item;
+  });
+  return joinUrlLib(...realArgs);
+}
+class RewriteFileConfigstore {
+  directory;
+  queue;
+  constructor(path2) {
+    this.directory = path2;
+    this.queue = new queue({
+      concurrency: 1
+    });
+  }
+  async get(key) {
+    try {
+      const buffer = await this.queue.add(() => fs.readFile(this.resolve(key), "utf8"));
+      const metadata = JSON.parse(buffer);
+      if (metadata.offset !== metadata.size) {
+        const info = await fs.stat(this.resolve(key, false)).catch(() => false);
+        if (info?.size !== metadata?.offset) {
+          metadata.offset = info.size;
+          this.set(key, metadata);
+        }
+      }
+      return metadata;
+    } catch {
+      return void 0;
+    }
+  }
+  async set(key, value) {
+    if (value?.runtime) {
+      delete value.runtime;
+    }
+    if (value?.metadata?.runtime) {
+      delete value.metadata.runtime;
+    }
+    await this.queue.add(() => fs.writeFile(this.resolve(key), JSON.stringify(value)));
+  }
+  async safeDeleteFile(filePath) {
+    validFilePathInDirPath(this.directory, filePath);
+    try {
+      const isExist = await fs.stat(filePath).catch(() => false);
+      if (isExist) {
+        await fs.rm(filePath);
+      } else {
+        console.log("Can not remove file, the file not exist: ", filePath);
+      }
+    } catch (err) {
+      console.error("@blocklet/uploader: safeDeleteFile error: ", err);
+    }
+  }
+  async delete(key, isMetadata = true) {
+    try {
+      await this.queue.add(() => this.safeDeleteFile(this.resolve(key, isMetadata)));
+    } catch (err) {
+      console.error("@blocklet/uploader: delete error: ", err);
+    }
+  }
+  async list() {
+    return this.queue.add(async () => {
+      const files = await fs.readdir(this.directory, {
+        withFileTypes: true
+      });
+      const promises = files.filter(file => file.isFile() && file.name.endsWith(".json")).map(file => {
+        return file.name.replace(".json", "");
+      });
+      return Promise.all(promises);
+    });
+  }
+  resolve(key, isMetadata = true) {
+    let fileKey = key;
+    if (isMetadata) {
+      fileKey = `${key}.json`;
+    }
+    return path.join(this.directory, fileKey);
+  }
+}
+class RewriteFileStore extends FileStore {
+  constructor(options) {
+    super(options);
+  }
+  async remove(key) {
+    this.configstore.delete(key);
+    this.configstore.delete(key, false);
+  }
+}