@blocklet/uploader-server 0.1.72 → 0.1.74

package/es/constants.d.ts

@@ -0,0 +1 @@
+export declare const ImageBinDid = "z8ia1mAXo8ZE7ytGF36L5uBf9kD2kenhqFGp9";

package/es/constants.js

@@ -0,0 +1 @@
+export const ImageBinDid = "z8ia1mAXo8ZE7ytGF36L5uBf9kD2kenhqFGp9";

package/es/middlewares/static-resource.d.ts

@@ -7,5 +7,5 @@ type initStaticResourceMiddlewareOptions = {
 };
 export declare const initStaticResourceMiddleware: ({ options, resourceTypes: _resourceTypes, express, skipRunningCheck: _skipRunningCheck, }?: initStaticResourceMiddlewareOptions) => (req: any, res: any, next: Function) => void;
 export declare const getCanUseResources: () => any;
-export declare const initProxyToMediaKitUploadsMiddleware: ({ options, express }?: any) => (req: any, res: any, next: Function) => any;
+export declare const initProxyToMediaKitUploadsMiddleware: ({ options, express }?: any) => (req: any, res: any, next: Function) => Promise<any>;
 export {};

package/es/middlewares/static-resource.js

@@ -2,15 +2,13 @@ const { existsSync } = require("fs");
 const { join, basename } = require("path");
 const config = require("@blocklet/sdk/lib/config");
 const { getResources } = require("@blocklet/sdk/lib/component");
-const httpProxy = require("http-proxy");
 const joinUrl = require("url-join");
+const component = require("@blocklet/sdk/lib/component");
 const { setPDFDownloadHeader } = require("../utils");
-const proxy = httpProxy.createProxyServer();
+const { ImageBinDid } = require("../constants");
 const logger = console;
 const ImgResourceType = "imgpack";
-const ImageBinDid = "z8ia1mAXo8ZE7ytGF36L5uBf9kD2kenhqFGp9";
 let skipRunningCheck = false;
-let mediaKitInfo = null;
 let resourceTypes = [
   {
     type: ImgResourceType,
@@ -21,10 +19,6 @@ let resourceTypes = [
 ];
 let canUseResources = [];
 export const mappingResource = async () => {
-  mediaKitInfo = config.components.find((item) => item.did === ImageBinDid);
-  if (mediaKitInfo) {
-    mediaKitInfo.uploadsDir = config.env.dataDir.replace(/\/[^/]*$/, "/image-bin/uploads");
-  }
   try {
     const resources = getResources({
       types: resourceTypes,
@@ -87,6 +81,13 @@ export const initStaticResourceMiddleware = ({
   return (req, res, next) => {
     const fileName = basename(req.url);
     const matchCanUseResourceItem = canUseResources.find((item) => {
+      const normalizedPath = join(item.dir, fileName);
+      if (!normalizedPath.startsWith(item.dir)) {
+        return false;
+      }
+      if (!existsSync(normalizedPath)) {
+        return false;
+      }
       const { whitelist, blacklist } = item;
       if (whitelist?.length && !whitelist.some((ext) => fileName.endsWith(ext))) {
         return false;
@@ -94,9 +95,6 @@ export const initStaticResourceMiddleware = ({
       if (blacklist?.length && blacklist.some((ext) => fileName.endsWith(ext))) {
         return false;
       }
-      if (!existsSync(join(item.dir, fileName))) {
-        return false;
-      }
       return true;
     });
     if (matchCanUseResourceItem) {
@@ -113,32 +111,30 @@ export const initStaticResourceMiddleware = ({
 };
 export const getCanUseResources = () => canUseResources;
 export const initProxyToMediaKitUploadsMiddleware = ({ options, express } = {}) => {
-  return (req, res, next) => {
-    if (!mediaKitInfo?.webEndpoint) {
+  return async (req, res, next) => {
+    if (!component.getComponentWebEndpoint(ImageBinDid)) {
       return next();
     }
     setPDFDownloadHeader(req, res);
-    proxy.once("proxyRes", (proxyRes, req2, res2) => {
-      if (proxyRes.statusCode >= 200 && proxyRes.statusCode < 400) {
-        res2.writeHead(proxyRes.statusCode, proxyRes.headers);
-        proxyRes.pipe(res2);
+    try {
+      const { data, status, headers } = await component.call({
+        name: ImageBinDid,
+        path: joinUrl("/uploads", basename(req.url)),
+        responseType: "stream",
+        method: "GET"
+      });
+      if (data && status >= 200 && status < 400) {
+        res.set("Content-Type", headers["content-type"]);
+        data.on("error", (err) => {
+          next();
+        }).pipe(res).on("error", (err) => {
+          next();
+        });
       } else {
         next();
       }
-    });
-    proxy.once("error", (err, req2, res2) => {
-      next(err);
-    });
-    proxy.web(
-      req,
-      res,
-      {
-        target: joinUrl(mediaKitInfo.webEndpoint, "/uploads", basename(req.url)),
-        changeOrigin: true,
-        selfHandleResponse: true,
-        ...options
-      },
-      next
-    );
+    } catch (error) {
+      next();
+    }
   };
 };

package/es/utils.d.ts

@@ -1,3 +1,14 @@
+import type { Method } from 'axios';
+type CallComponentOptions<D = any, P = any> = {
+    name?: string;
+    method?: Method;
+    path: string;
+    data?: D;
+    params?: P;
+    headers?: {
+        [key: string]: any;
+    };
+};
 export declare function getTrustedDomainsCache({ forceUpdate, ttl, }?: {
     forceUpdate?: boolean;
     ttl?: number;
@@ -5,3 +16,12 @@ export declare function getTrustedDomainsCache({ forceUpdate, ttl, }?: {
 export declare function checkTrustedReferer(req: any, res: any, next?: Function): Promise<any>;
 export declare function proxyImageDownload(req: any, res: any, next?: Function): Promise<void>;
 export declare function setPDFDownloadHeader(req: any, res: any): void;
+export declare const getFileHash: (filePath: string, maxBytes?: number) => Promise<any>;
+export declare function uploadToMediaKit({ filePath, fileName, base64, extraComponentCallOptions, }: {
+    filePath?: string;
+    fileName?: string;
+    base64?: string;
+    extraComponentCallOptions?: CallComponentOptions;
+}): Promise<import("axios").AxiosResponse<import("http").IncomingMessage, any> | undefined>;
+export declare function getMediaKitFileStream(filePath: string): Promise<import("axios").AxiosResponse<import("http").IncomingMessage, any>>;
+export {};

package/es/utils.js

@@ -2,6 +2,13 @@ import axios from "axios";
 import path from "path";
 import joinUrl from "url-join";
 import { isbot } from "isbot";
+import component from "@blocklet/sdk/lib/component";
+import { ImageBinDid } from "./constants.js";
+import { createReadStream } from "fs";
+import crypto from "crypto";
+import { getSignData } from "@blocklet/sdk/lib/util/verify-sign";
+import FormData from "form-data";
+import omit from "lodash/omit";
 const DEFAULT_TTL = 5 * 60 * 1e3;
 const appUrl = process.env.BLOCKLET_APP_URL || "";
 const trustedDomainsCache = {
@@ -78,3 +85,84 @@ export function setPDFDownloadHeader(req, res) {
     res.setHeader("Content-Disposition", `attachment; ${filename ? `filename="${filename}"` : ""}`);
   }
 }
+export const getFileHash = async (filePath, maxBytes = 5 * 1024 * 1024) => {
+  const hash = crypto.createHash("md5");
+  const readStream = createReadStream(filePath, {
+    start: 0,
+    end: maxBytes - 1,
+    highWaterMark: 1024 * 1024
+    // 1MB chunks
+  });
+  for await (const chunk of readStream) {
+    hash.update(chunk.toString());
+  }
+  return hash.digest("hex");
+};
+export async function uploadToMediaKit({
+  filePath,
+  fileName,
+  base64,
+  extraComponentCallOptions
+}) {
+  if (!filePath && !base64) {
+    throw new Error("filePath or base64 is required");
+  }
+  if (base64) {
+    if (!fileName) {
+      throw new Error("fileName is required when base64 is provided");
+    }
+    const res = await component.call({
+      name: ImageBinDid,
+      path: "/api/sdk/uploads",
+      data: {
+        base64,
+        filename: fileName
+      },
+      ...omit(extraComponentCallOptions, ["name", "path", "data"])
+    });
+    return res;
+  }
+  if (filePath) {
+    const fileStream = createReadStream(filePath);
+    const filename = fileName || path.basename(filePath);
+    const form = new FormData();
+    const fileHash = await getFileHash(filePath);
+    form.append("file", fileStream);
+    form.append("filename", filename);
+    form.append("hash", fileHash);
+    const res = await component.call(
+      {
+        name: ImageBinDid,
+        path: "/api/sdk/uploads",
+        data: form,
+        headers: {
+          "x-component-upload-sig": getSignData({
+            data: {
+              filename,
+              hash: fileHash
+            },
+            method: "POST",
+            url: "/api/sdk/uploads",
+            params: extraComponentCallOptions?.params || {}
+          }).sig,
+          ...extraComponentCallOptions?.headers
+        },
+        ...omit(extraComponentCallOptions, ["name", "path", "data", "headers"])
+      },
+      {
+        retries: 0
+      }
+    );
+    return res;
+  }
+}
+export async function getMediaKitFileStream(filePath) {
+  const fileName = path.basename(filePath);
+  const res = await component.call({
+    name: ImageBinDid,
+    path: joinUrl("/uploads", fileName),
+    responseType: "stream",
+    method: "GET"
+  });
+  return res;
+}

package/lib/constants.d.ts

@@ -0,0 +1 @@
+export declare const ImageBinDid = "z8ia1mAXo8ZE7ytGF36L5uBf9kD2kenhqFGp9";

package/lib/constants.js

@@ -0,0 +1,7 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.ImageBinDid = void 0;
+const ImageBinDid = exports.ImageBinDid = "z8ia1mAXo8ZE7ytGF36L5uBf9kD2kenhqFGp9";

package/lib/middlewares/static-resource.d.ts

@@ -7,5 +7,5 @@ type initStaticResourceMiddlewareOptions = {
 };
 export declare const initStaticResourceMiddleware: ({ options, resourceTypes: _resourceTypes, express, skipRunningCheck: _skipRunningCheck, }?: initStaticResourceMiddlewareOptions) => (req: any, res: any, next: Function) => void;
 export declare const getCanUseResources: () => any;
-export declare const initProxyToMediaKitUploadsMiddleware: ({ options, express }?: any) => (req: any, res: any, next: Function) => any;
+export declare const initProxyToMediaKitUploadsMiddleware: ({ options, express }?: any) => (req: any, res: any, next: Function) => Promise<any>;
 export {};

package/lib/middlewares/static-resource.js

@@ -15,17 +15,17 @@ const config = require("@blocklet/sdk/lib/config");
 const {
   getResources
 } = require("@blocklet/sdk/lib/component");
-const httpProxy = require("http-proxy");
 const joinUrl = require("url-join");
+const component = require("@blocklet/sdk/lib/component");
 const {
   setPDFDownloadHeader
 } = require("../utils");
-const proxy = httpProxy.createProxyServer();
+const {
+  ImageBinDid
+} = require("../constants");
 const logger = console;
 const ImgResourceType = "imgpack";
-const ImageBinDid = "z8ia1mAXo8ZE7ytGF36L5uBf9kD2kenhqFGp9";
 let skipRunningCheck = false;
-let mediaKitInfo = null;
 let resourceTypes = [{
   type: ImgResourceType,
   did: ImageBinDid,
@@ -34,10 +34,6 @@ let resourceTypes = [{
 }];
 let canUseResources = [];
 const mappingResource = async () => {
-  mediaKitInfo = config.components.find(item => item.did === ImageBinDid);
-  if (mediaKitInfo) {
-    mediaKitInfo.uploadsDir = config.env.dataDir.replace(/\/[^/]*$/, "/image-bin/uploads");
-  }
   try {
     const resources = getResources({
       types: resourceTypes,
@@ -104,6 +100,13 @@ const initStaticResourceMiddleware = ({
   return (req, res, next) => {
     const fileName = basename(req.url);
     const matchCanUseResourceItem = canUseResources.find(item => {
+      const normalizedPath = join(item.dir, fileName);
+      if (!normalizedPath.startsWith(item.dir)) {
+        return false;
+      }
+      if (!existsSync(normalizedPath)) {
+        return false;
+      }
       const {
         whitelist,
         blacklist
@@ -114,9 +117,6 @@ const initStaticResourceMiddleware = ({
       if (blacklist?.length && blacklist.some(ext => fileName.endsWith(ext))) {
         return false;
       }
-      if (!existsSync(join(item.dir, fileName))) {
-        return false;
-      }
       return true;
     });
     if (matchCanUseResourceItem) {
@@ -138,28 +138,35 @@ const initProxyToMediaKitUploadsMiddleware = ({
   options,
   express
 } = {}) => {
-  return (req, res, next) => {
-    if (!mediaKitInfo?.webEndpoint) {
+  return async (req, res, next) => {
+    if (!component.getComponentWebEndpoint(ImageBinDid)) {
       return next();
     }
     setPDFDownloadHeader(req, res);
-    proxy.once("proxyRes", (proxyRes, req2, res2) => {
-      if (proxyRes.statusCode >= 200 && proxyRes.statusCode < 400) {
-        res2.writeHead(proxyRes.statusCode, proxyRes.headers);
-        proxyRes.pipe(res2);
+    try {
+      const {
+        data,
+        status,
+        headers
+      } = await component.call({
+        name: ImageBinDid,
+        path: joinUrl("/uploads", basename(req.url)),
+        responseType: "stream",
+        method: "GET"
+      });
+      if (data && status >= 200 && status < 400) {
+        res.set("Content-Type", headers["content-type"]);
+        data.on("error", err => {
+          next();
+        }).pipe(res).on("error", err => {
+          next();
+        });
       } else {
         next();
       }
-    });
-    proxy.once("error", (err, req2, res2) => {
-      next(err);
-    });
-    proxy.web(req, res, {
-      target: joinUrl(mediaKitInfo.webEndpoint, "/uploads", basename(req.url)),
-      changeOrigin: true,
-      selfHandleResponse: true,
-      ...options
-    }, next);
+    } catch (error) {
+      next();
+    }
   };
 };
 exports.initProxyToMediaKitUploadsMiddleware = initProxyToMediaKitUploadsMiddleware;

package/lib/utils.d.ts

@@ -1,3 +1,14 @@
+import type { Method } from 'axios';
+type CallComponentOptions<D = any, P = any> = {
+    name?: string;
+    method?: Method;
+    path: string;
+    data?: D;
+    params?: P;
+    headers?: {
+        [key: string]: any;
+    };
+};
 export declare function getTrustedDomainsCache({ forceUpdate, ttl, }?: {
     forceUpdate?: boolean;
     ttl?: number;
@@ -5,3 +16,12 @@ export declare function getTrustedDomainsCache({ forceUpdate, ttl, }?: {
 export declare function checkTrustedReferer(req: any, res: any, next?: Function): Promise<any>;
 export declare function proxyImageDownload(req: any, res: any, next?: Function): Promise<void>;
 export declare function setPDFDownloadHeader(req: any, res: any): void;
+export declare const getFileHash: (filePath: string, maxBytes?: number) => Promise<any>;
+export declare function uploadToMediaKit({ filePath, fileName, base64, extraComponentCallOptions, }: {
+    filePath?: string;
+    fileName?: string;
+    base64?: string;
+    extraComponentCallOptions?: CallComponentOptions;
+}): Promise<import("axios").AxiosResponse<import("http").IncomingMessage, any> | undefined>;
+export declare function getMediaKitFileStream(filePath: string): Promise<import("axios").AxiosResponse<import("http").IncomingMessage, any>>;
+export {};

package/lib/utils.js

@@ -4,13 +4,23 @@ Object.defineProperty(exports, "__esModule", {
   value: true
 });
 exports.checkTrustedReferer = checkTrustedReferer;
+exports.getFileHash = void 0;
+exports.getMediaKitFileStream = getMediaKitFileStream;
 exports.getTrustedDomainsCache = getTrustedDomainsCache;
 exports.proxyImageDownload = proxyImageDownload;
 exports.setPDFDownloadHeader = setPDFDownloadHeader;
+exports.uploadToMediaKit = uploadToMediaKit;
 var _axios = _interopRequireDefault(require("axios"));
 var _path = _interopRequireDefault(require("path"));
 var _urlJoin = _interopRequireDefault(require("url-join"));
 var _isbot = require("isbot");
+var _component = _interopRequireDefault(require("@blocklet/sdk/lib/component"));
+var _constants = require("./constants");
+var _fs = require("fs");
+var _crypto = _interopRequireDefault(require("crypto"));
+var _verifySign = require("@blocklet/sdk/lib/util/verify-sign");
+var _formData = _interopRequireDefault(require("form-data"));
+var _omit = _interopRequireDefault(require("lodash/omit"));
 function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
 const DEFAULT_TTL = 5 * 60 * 1e3;
 const appUrl = process.env.BLOCKLET_APP_URL || "";
@@ -93,4 +103,83 @@ function setPDFDownloadHeader(req, res) {
     const filename = req.query?.filename ?? req?.path;
     res.setHeader("Content-Disposition", `attachment; ${filename ? `filename="${filename}"` : ""}`);
   }
+}
+const getFileHash = async (filePath, maxBytes = 5 * 1024 * 1024) => {
+  const hash = _crypto.default.createHash("md5");
+  const readStream = (0, _fs.createReadStream)(filePath, {
+    start: 0,
+    end: maxBytes - 1,
+    highWaterMark: 1024 * 1024
+    // 1MB chunks
+  });
+  for await (const chunk of readStream) {
+    hash.update(chunk.toString());
+  }
+  return hash.digest("hex");
+};
+exports.getFileHash = getFileHash;
+async function uploadToMediaKit({
+  filePath,
+  fileName,
+  base64,
+  extraComponentCallOptions
+}) {
+  if (!filePath && !base64) {
+    throw new Error("filePath or base64 is required");
+  }
+  if (base64) {
+    if (!fileName) {
+      throw new Error("fileName is required when base64 is provided");
+    }
+    const res = await _component.default.call({
+      name: _constants.ImageBinDid,
+      path: "/api/sdk/uploads",
+      data: {
+        base64,
+        filename: fileName
+      },
+      ...(0, _omit.default)(extraComponentCallOptions, ["name", "path", "data"])
+    });
+    return res;
+  }
+  if (filePath) {
+    const fileStream = (0, _fs.createReadStream)(filePath);
+    const filename = fileName || _path.default.basename(filePath);
+    const form = new _formData.default();
+    const fileHash = await getFileHash(filePath);
+    form.append("file", fileStream);
+    form.append("filename", filename);
+    form.append("hash", fileHash);
+    const res = await _component.default.call({
+      name: _constants.ImageBinDid,
+      path: "/api/sdk/uploads",
+      data: form,
+      headers: {
+        "x-component-upload-sig": (0, _verifySign.getSignData)({
+          data: {
+            filename,
+            hash: fileHash
+          },
+          method: "POST",
+          url: "/api/sdk/uploads",
+          params: extraComponentCallOptions?.params || {}
+        }).sig,
+        ...extraComponentCallOptions?.headers
+      },
+      ...(0, _omit.default)(extraComponentCallOptions, ["name", "path", "data", "headers"])
+    }, {
+      retries: 0
+    });
+    return res;
+  }
+}
+async function getMediaKitFileStream(filePath) {
+  const fileName = _path.default.basename(filePath);
+  const res = await _component.default.call({
+    name: _constants.ImageBinDid,
+    path: (0, _urlJoin.default)("/uploads", fileName),
+    responseType: "stream",
+    method: "GET"
+  });
+  return res;
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@blocklet/uploader-server",
-  "version": "0.1.72",
+  "version": "0.1.74",
   "description": "blocklet upload server",
   "publishConfig": {
     "access": "public"
@@ -47,7 +47,6 @@
     "body-parser": "^1.20.3",
     "crypto": "^1.0.1",
     "express-session": "1.17.3",
-    "http-proxy": "^1.18.1",
     "isbot": "^5.1.17",
     "mime-types": "^2.1.35",
     "p-queue": "6.6.2",
@@ -57,7 +56,6 @@
   "devDependencies": {
     "@arcblock/eslint-config-ts": "^0.2.4",
     "@types/express": "^4.17.21",
-    "@types/http-proxy": "^1.17.15",
     "@types/mime-types": "^2.1.4",
     "@types/node": "^20.17.9",
     "@types/url-join": "^4.0.3",
@@ -74,6 +72,7 @@
     "build:watch": "npx nodemon --ext 'ts,tsx,json,js,jsx' --exec 'pnpm run build' --ignore 'lib/*' --ignore 'es/*' ",
     "dev": "pnpm run build:watch",
     "prepublish": "pnpm run build",
-    "prebuild:dep": "pnpm run build"
+    "prebuild:dep": "pnpm run build",
+    "dev:yalc": "npx nodemon --ext 'ts,tsx,json,js,jsx' --exec 'pnpm run build && yalc push' --ignore 'lib/*' --ignore 'es/*'"
   }
 }