@blocklet/uploader-server 0.1.49

package/LICENSE

@@ -0,0 +1,13 @@
+Copyright 2018-2020 ArcBlock
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

package/README.md

@@ -0,0 +1,101 @@
+# @blocklet/uploader-server-server
+
+**@blocklet/uploader-server** is a package that integrates the **uppy** service to provide universal upload capability for blocklets. For more information about uppy, refer to the [official documentation](https://uppy.io/docs/quick-start/).
+
+## Package Structure
+
+The package is composed of both frontend and backend components. The backend code can be found in the `middlewares` folder.
+
+## Development
+
+### Install In Blocklet
+
+```
+# You can use npm / yarn
+pnpm add @blocklet/uploader-server
+```
+
+### Install Dependencies
+
+To install the required dependencies, run the following command:
+
+```
+pnpm i
+```
+
+### Build Packages
+
+To build the packages, execute the following command:
+
+```
+pnpm build
+```
+
+### Build, Watch, and Run Development Server
+
+For building, watching changes, and running the development server, use the following command:
+
+```
+pnpm run dev
+```
+
+## Backend Example
+
+```javascript
+const { initLocalStorageServer, initCompanion } = require('@blocklet/uploader-server');
+
+// init uploader server
+const localStorageServer = initLocalStorageServer({
+  path: env.uploadDir,
+  express,
+  onUploadFinish: async (req, res, uploadMetadata) => {
+    const {
+      id: filename,
+      size,
+      metadata: { filename: originalname, filetype: mimetype },
+    } = uploadMetadata;
+
+    const obj = new URL(env.appUrl);
+    obj.protocol = req.get('x-forwarded-proto') || req.protocol;
+    obj.pathname = joinUrl(req.headers['x-path-prefix'] || '/', '/uploads', filename);
+
+    const doc = await Upload.insert({
+      mimetype,
+      originalname,
+      filename,
+      size,
+      remark: req.body.remark || '',
+      tags: (req.body.tags || '')
+        .split(',')
+        .map((x) => x.trim())
+        .filter(Boolean),
+      folderId: req.componentDid,
+      createdAt: new Date().toISOString(),
+      updatedAt: new Date().toISOString(),
+      createdBy: req.user.did,
+      updatedBy: req.user.did,
+    });
+
+    const resData = { url: obj.href, ...doc };
+
+    return resData;
+  },
+});
+
+router.use('/uploads', user, auth, ensureComponentDid, localStorageServer.handle);
+
+// if you need to load file from remote
+// companion
+const companion = initCompanion({
+  path: env.uploadDir,
+  express,
+  providerOptions: env.providerOptions,
+  uploadUrls: [env.appUrl],
+});
+
+router.use('/companion', user, auth, ensureComponentDid, companion.handle);
+```
+
+## License
+
+This package is licensed under the MIT license.

package/es/index.d.ts

@@ -0,0 +1 @@
+export * from './middlewares';

package/es/index.js

@@ -0,0 +1 @@
+export * from "./middlewares.js";

package/es/middlewares/companion.d.ts

@@ -0,0 +1,6 @@
+export declare function initCompanion({ path, express, providerOptions, ...restProps }: {
+    path: string;
+    express: Function;
+    providerOptions?: Object;
+}): any;
+export declare function proxyImageDownload(req: any, res: any, next?: Function): Promise<void>;

package/es/middlewares/companion.js

@@ -0,0 +1,92 @@
+const companion = require("@uppy/companion");
+const bodyParser = require("body-parser");
+const session = require("express-session");
+const axios = require("axios");
+const crypto = require("crypto");
+const secret = crypto.randomBytes(32).toString("hex");
+export function initCompanion({
+  path,
+  express,
+  providerOptions,
+  ...restProps
+}) {
+  const app = express();
+  app.use(bodyParser.json());
+  app.use(session({ secret }));
+  app.use("/proxy", proxyImageDownload);
+  let dynamicProviderOptions = providerOptions;
+  const companionOptions = {
+    secret,
+    providerOptions,
+    // unused
+    server: {
+      protocol: "https",
+      host: "UNUSED_HOST",
+      // unused
+      path: "UNUSED_PATH"
+      // unused
+    },
+    filePath: path,
+    streamingUpload: true,
+    metrics: false,
+    ...restProps
+  };
+  const newCompanion = companion.app(companionOptions);
+  const { app: companionApp } = newCompanion;
+  app.all(
+    "*",
+    (req, res, next) => {
+      let hackerCompanion = {};
+      Object.defineProperty(req, "companion", {
+        get() {
+          return hackerCompanion;
+        },
+        set(value) {
+          hackerCompanion = value;
+          hackerCompanion.options.providerOptions = dynamicProviderOptions;
+        }
+      });
+      next();
+    },
+    companionApp
+  );
+  newCompanion.handle = app;
+  newCompanion.setProviderOptions = (options) => {
+    dynamicProviderOptions = options;
+  };
+  return newCompanion;
+}
+export async function proxyImageDownload(req, res, next) {
+  let { url, responseType = "stream" } = {
+    ...req.query,
+    ...req.body
+  };
+  if (url) {
+    url = encodeURI(url);
+    try {
+      const { headers, data, status } = await axios.get(url, {
+        responseType
+      });
+      if (data && status >= 200 && status < 302) {
+        res.setHeader("Content-Type", headers["content-type"]);
+        try {
+        } catch (error) {
+        }
+        if (responseType === "stream") {
+          data.pipe(res);
+        } else if (responseType === "arraybuffer") {
+          res.end(data?.toString?.("binary"), "binary");
+        } else {
+          res.send(data);
+        }
+      } else {
+        throw new Error("download image error");
+      }
+    } catch (err) {
+      console.error("Proxy url failed: ", err);
+      res.status(500).send("Proxy url failed");
+    }
+  } else {
+    res.status(500).send('Parameter "url" is required');
+  }
+}

package/es/middlewares/local-storage.d.ts

@@ -0,0 +1,18 @@
+import { type ServerOptions } from '@tus/server';
+export declare function initLocalStorageServer({ path: _path, onUploadFinish: _onUploadFinish, onUploadCreate: _onUploadCreate, express, expiredUploadTime, // default 3 days expire
+...restProps }: ServerOptions & {
+    path: string;
+    onUploadFinish?: Function;
+    onUploadCreate?: Function;
+    express: Function;
+    expiredUploadTime?: Number;
+}): any;
+export declare const getFileName: (req: any) => any;
+export declare function getFileNameParam(req: any, res: any, { isRequired }?: {
+    isRequired: boolean;
+}): any;
+export declare function getLocalStorageFile({ server }: any): (req: any, res: any, next: any) => Promise<void>;
+export declare function setHeaders(req: any, res: any, next?: Function): void;
+export declare function fileExistBeforeUpload(req: any, res: any, next?: Function): Promise<void>;
+export declare function getMetaDataByFilePath(filePath: string): Promise<any>;
+export declare function joinUrl(...args: string[]): any;

package/es/middlewares/local-storage.js

@@ -0,0 +1,374 @@
+const { Server, EVENTS } = require("@tus/server");
+const { FileStore } = require("@tus/file-store");
+const cron = require("@abtnode/cron");
+const fs = require("fs").promises;
+const path = require("path");
+const crypto = require("crypto");
+const mime = require("mime-types");
+const joinUrlLib = require("url-join");
+const { default: queue } = require("p-queue");
+const validFilePathInDirPath = (dirPath, filePath) => {
+  const fileName = path.basename(filePath);
+  if (!filePath.startsWith(dirPath) || path.join(dirPath, fileName) !== filePath) {
+    console.error("Invalid file path: ", filePath);
+    throw new Error("Invalid file path");
+  }
+  return true;
+};
+export function initLocalStorageServer({
+  path: _path,
+  onUploadFinish: _onUploadFinish,
+  onUploadCreate: _onUploadCreate,
+  express,
+  expiredUploadTime = 1e3 * 60 * 60 * 24 * 3,
+  // default 3 days expire
+  ...restProps
+}) {
+  const app = express();
+  const configstore = new RewriteFileConfigstore(_path);
+  const datastore = new RewriteFileStore({
+    directory: _path,
+    expirationPeriodInMilliseconds: expiredUploadTime,
+    configstore
+  });
+  const formatMetadata = (uploadMetadata) => {
+    const cloneUploadMetadata = {
+      ...uploadMetadata
+    };
+    if (cloneUploadMetadata.metadata?.name?.indexOf("/") > -1 && cloneUploadMetadata.metadata?.relativePath?.indexOf("/") > -1) {
+      cloneUploadMetadata.metadata.name = cloneUploadMetadata.metadata.name.split("/").pop();
+      cloneUploadMetadata.metadata.filename = cloneUploadMetadata.metadata.name;
+    }
+    if (cloneUploadMetadata.id && !cloneUploadMetadata.runtime) {
+      const { id, metadata, size } = cloneUploadMetadata;
+      cloneUploadMetadata.runtime = {
+        relativePath: metadata?.relativePath,
+        absolutePath: path.join(_path, id),
+        size,
+        hashFileName: id,
+        originFileName: metadata?.filename,
+        type: metadata?.type,
+        fileType: metadata?.filetype
+      };
+    }
+    return cloneUploadMetadata;
+  };
+  const rewriteMetaDataFile = async (uploadMetadata) => {
+    uploadMetadata = formatMetadata(uploadMetadata);
+    const { id } = uploadMetadata;
+    if (!id) {
+      return;
+    }
+    const oldMetadata = formatMetadata(await configstore.get(id));
+    if (JSON.stringify(oldMetadata) !== JSON.stringify(uploadMetadata)) {
+      await configstore.set(id, uploadMetadata);
+    }
+  };
+  const onUploadCreate = async (req, res, uploadMetadata) => {
+    uploadMetadata = formatMetadata(uploadMetadata);
+    await rewriteMetaDataFile(uploadMetadata);
+    if (uploadMetadata.offset === 0 && uploadMetadata.size === 0) {
+      res.status(200);
+      res.setHeader("Location", joinUrl(req.headers["x-uploader-base-url"], uploadMetadata.id));
+      res.setHeader("Upload-Offset", 0);
+      res.setHeader("Upload-Length", 0);
+      res.setHeader("x-uploader-file-exist", true);
+    }
+    if (_onUploadCreate) {
+      const result = await _onUploadCreate(req, res, uploadMetadata);
+      return result;
+    }
+    return res;
+  };
+  const onUploadFinish = async (req, res, uploadMetadata) => {
+    res.setHeader("x-uploader-file-exist", true);
+    uploadMetadata = formatMetadata(uploadMetadata);
+    await rewriteMetaDataFile(uploadMetadata);
+    if (_onUploadFinish) {
+      try {
+        const result = await _onUploadFinish(req, res, uploadMetadata);
+        return result;
+      } catch (err) {
+        console.error("@blocklet/uploader: onUploadFinish error: ", err);
+        newServer.delete(uploadMetadata.id);
+        res.setHeader("x-uploader-file-exist", false);
+        throw err;
+      }
+    }
+    return res;
+  };
+  const newServer = new Server({
+    path: "/",
+    // UNUSED
+    relativeLocation: true,
+    // respectForwardedHeaders: true,
+    namingFunction: (req) => {
+      const fileName = getFileName(req);
+      const filePath = path.join(_path, fileName);
+      validFilePathInDirPath(_path, filePath);
+      return fileName;
+    },
+    datastore,
+    onUploadFinish: async (req, res, uploadMetadata) => {
+      uploadMetadata = formatMetadata(uploadMetadata);
+      const result = await onUploadFinish(req, res, uploadMetadata);
+      if (result && !result.send) {
+        const body = typeof result === "string" ? result : JSON.stringify(result);
+        throw { body, status_code: 200 };
+      } else {
+        return result;
+      }
+    },
+    onUploadCreate,
+    ...restProps
+  });
+  app.use((req, res, next) => {
+    req.uploaderProps = {
+      server: newServer,
+      onUploadFinish,
+      onUploadCreate
+    };
+    next();
+  });
+  cron.init({
+    context: {},
+    jobs: [
+      {
+        name: "auto-cleanup-expired-uploads",
+        time: "0 0 * * * *",
+        // each hour
+        fn: () => {
+          try {
+            newServer.cleanUpExpiredUploads().then((count) => {
+              console.info(`@blocklet/uploader: cleanup expired uploads done: ${count}`);
+            }).catch((err) => {
+              console.error(`@blocklet/uploader: cleanup expired uploads error`, err);
+            });
+          } catch (err) {
+            console.error(`@blocklet/uploader: cleanup expired uploads error`, err);
+          }
+        },
+        options: { runOnInit: false }
+      }
+    ],
+    onError: (err) => {
+      console.error("@blocklet/uploader: cleanup job failed", err);
+    }
+  });
+  newServer.delete = async (key) => {
+    try {
+      await configstore.delete(key);
+      await configstore.delete(key, false);
+    } catch (err) {
+      console.error("@blocklet/uploader: delete error: ", err);
+    }
+  };
+  newServer.on(EVENTS.POST_RECEIVE, async (req, res, uploadMetadata) => {
+    uploadMetadata = formatMetadata(uploadMetadata);
+    await rewriteMetaDataFile(uploadMetadata);
+  });
+  app.all("*", setHeaders, fileExistBeforeUpload, newServer.handle.bind(newServer));
+  newServer.handle = app;
+  return newServer;
+}
+export const getFileName = (req) => {
+  const ext = req.headers["x-uploader-file-ext"];
+  const randomName = `${crypto.randomBytes(16).toString("hex")}${ext ? `.${ext}` : ""}`;
+  return req.headers["x-uploader-file-name"] || randomName;
+};
+export function getFileNameParam(req, res, { isRequired = true } = {}) {
+  let { fileName } = req.params;
+  if (!fileName) {
+    fileName = req.originalUrl.replace(req.baseUrl, "");
+  }
+  if (!fileName && isRequired) {
+    res.status(400).json({ error: 'Parameter "fileName" is required' });
+    return;
+  }
+  return fileName;
+}
+export function getLocalStorageFile({ server }) {
+  return async (req, res, next) => {
+    const fileName = getFileNameParam(req, res);
+    const filePath = path.join(server.datastore.directory, fileName);
+    validFilePathInDirPath(server.datastore.directory, filePath);
+    const fileExists = await fs.stat(filePath).catch(() => false);
+    if (!fileExists) {
+      res.status(404).json({ error: "file not found" });
+      return;
+    }
+    setHeaders(req, res);
+    const file = await fs.readFile(filePath);
+    res.send(file);
+    next?.();
+  };
+}
+export function setHeaders(req, res, next) {
+  let { method } = req;
+  method = method.toUpperCase();
+  const fileName = getFileNameParam(req, res, {
+    isRequired: false
+  });
+  if (req.headers["x-uploader-endpoint-url"]) {
+    const query = new URL(req.headers["x-uploader-endpoint-url"]).searchParams;
+    req.query = {
+      ...Object.fromEntries(query),
+      // query params convert to object
+      ...req.query
+    };
+  }
+  if (method === "POST" && req.headers["x-uploader-base-url"]) {
+    req.baseUrl = req.headers["x-uploader-base-url"];
+  }
+  if (method === "GET" && fileName) {
+    const contentType = mime.lookup(fileName);
+    if (contentType) {
+      res.setHeader("Content-Type", contentType);
+    }
+  }
+  next?.();
+}
+export async function fileExistBeforeUpload(req, res, next) {
+  let { method, uploaderProps } = req;
+  method = method.toUpperCase();
+  if (["PATCH", "POST"].includes(method)) {
+    const _path = uploaderProps.server.datastore.directory;
+    const fileName = getFileName(req);
+    const filePath = path.join(_path, fileName);
+    validFilePathInDirPath(_path, filePath);
+    const isExist = await fs.stat(filePath).catch(() => false);
+    if (isExist) {
+      const metaData = await getMetaDataByFilePath(filePath);
+      if (isExist?.size >= 0 && isExist?.size === metaData?.size) {
+        const prepareUpload = method === "POST";
+        if (prepareUpload) {
+          res.status(200);
+          res.setHeader("Location", joinUrl(req.headers["x-uploader-base-url"], fileName));
+          res.setHeader("Upload-Offset", +metaData.offset);
+          res.setHeader("Upload-Length", +metaData.size);
+        }
+        if (req.headers["x-uploader-metadata"]) {
+          try {
+            const realMetaData = JSON.parse(req.headers["x-uploader-metadata"], (key, value) => {
+              if (typeof value === "string") {
+                return decodeURIComponent(value);
+              }
+              return value;
+            });
+            metaData.metadata = {
+              ...metaData.metadata,
+              ...realMetaData
+            };
+          } catch (err) {
+            console.error("@blocklet/uploader: parse metadata error: ", err);
+          }
+        }
+        const uploadResult = await uploaderProps.onUploadFinish(req, res, metaData);
+        res.json(uploadResult);
+        return;
+      }
+    }
+  }
+  next?.();
+}
+export async function getMetaDataByFilePath(filePath) {
+  const metaDataPath = `${filePath}.json`;
+  const isExist = await fs.stat(filePath).catch(() => false);
+  if (isExist) {
+    try {
+      const metaData = await fs.readFile(metaDataPath, "utf-8");
+      const metaDataJson = JSON.parse(metaData);
+      return metaDataJson;
+    } catch (err) {
+      console.error("@blocklet/uploader: getMetaDataByPath error: ", err);
+    }
+  }
+  return null;
+}
+export function joinUrl(...args) {
+  const realArgs = args.filter(Boolean).map((item) => {
+    if (item === "/") {
+      return "";
+    }
+    return item;
+  });
+  return joinUrlLib(...realArgs);
+}
+class RewriteFileConfigstore {
+  directory;
+  queue;
+  constructor(path2) {
+    this.directory = path2;
+    this.queue = new queue({ concurrency: 1 });
+  }
+  async get(key) {
+    try {
+      const buffer = await this.queue.add(() => fs.readFile(this.resolve(key), "utf8"));
+      const metadata = JSON.parse(buffer);
+      if (metadata.offset !== metadata.size) {
+        const info = await fs.stat(this.resolve(key, false)).catch(() => false);
+        if (info?.size !== metadata?.offset) {
+          metadata.offset = info.size;
+          this.set(key, metadata);
+        }
+      }
+      return metadata;
+    } catch {
+      return void 0;
+    }
+  }
+  async set(key, value) {
+    if (value?.runtime) {
+      delete value.runtime;
+    }
+    if (value?.metadata?.runtime) {
+      delete value.metadata.runtime;
+    }
+    await this.queue.add(() => fs.writeFile(this.resolve(key), JSON.stringify(value)));
+  }
+  async safeDeleteFile(filePath) {
+    validFilePathInDirPath(this.directory, filePath);
+    try {
+      const isExist = await fs.stat(filePath).catch(() => false);
+      if (isExist) {
+        await fs.rm(filePath);
+      } else {
+        console.log("Can not remove file, the file not exist: ", filePath);
+      }
+    } catch (err) {
+      console.error("@blocklet/uploader: safeDeleteFile error: ", err);
+    }
+  }
+  async delete(key, isMetadata = true) {
+    try {
+      await this.queue.add(() => this.safeDeleteFile(this.resolve(key, isMetadata)));
+    } catch (err) {
+      console.error("@blocklet/uploader: delete error: ", err);
+    }
+  }
+  async list() {
+    return this.queue.add(async () => {
+      const files = await fs.readdir(this.directory, { withFileTypes: true });
+      const promises = files.filter((file) => file.isFile() && file.name.endsWith(".json")).map((file) => {
+        return file.name.replace(".json", "");
+      });
+      return Promise.all(promises);
+    });
+  }
+  resolve(key, isMetadata = true) {
+    let fileKey = key;
+    if (isMetadata) {
+      fileKey = `${key}.json`;
+    }
+    return path.join(this.directory, fileKey);
+  }
+}
+class RewriteFileStore extends FileStore {
+  constructor(options) {
+    super(options);
+  }
+  async remove(key) {
+    this.configstore.delete(key);
+    this.configstore.delete(key, false);
+  }
+}

package/es/middlewares/static-resource.d.ts

@@ -0,0 +1,11 @@
+export declare const mappingResource: () => Promise<any>;
+type initStaticResourceMiddlewareOptions = {
+    options?: any;
+    resourceTypes?: string[] | Object[];
+    express: any;
+    skipRunningCheck?: boolean;
+};
+export declare const initStaticResourceMiddleware: ({ options, resourceTypes: _resourceTypes, express, skipRunningCheck: _skipRunningCheck, }?: initStaticResourceMiddlewareOptions) => (req: any, res: any, next: Function) => void;
+export declare const getCanUseResources: () => any;
+export declare const initProxyToMediaKitUploadsMiddleware: ({ options, express }?: any) => (req: any, res: any, next: Function) => any;
+export {};