@nocobase/plugin-file-manager 2.0.57 → 2.0.59

package/dist/server/commands/repair-filenames.js

@@ -0,0 +1,283 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var repair_filenames_exports = {};
+__export(repair_filenames_exports, {
+  getRepairedAttachmentValues: () => getRepairedAttachmentValues,
+  registerRepairFilenamesCommand: () => registerRepairFilenamesCommand,
+  repairAttachmentFilenames: () => repairAttachmentFilenames,
+  replaceInvisibleChars: () => replaceInvisibleChars
+});
+module.exports = __toCommonJS(repair_filenames_exports);
+var import_sequelize = require("sequelize");
+var import_utils = require("../utils");
+function containsInvisibleChars(value) {
+  return Array.from(value || "").some((char) => {
+    const code = char.charCodeAt(0);
+    return code <= 31 || code >= 127 && code <= 159;
+  });
+}
+function replaceInvisibleChars(value) {
+  return Array.from(value || "").map((char) => {
+    const code = char.charCodeAt(0);
+    return code <= 31 || code >= 127 && code <= 159 ? "-" : char;
+  }).join("");
+}
+function getRepairedAttachmentValues(record) {
+  return {
+    path: replaceInvisibleChars(record.path),
+    filename: replaceInvisibleChars(record.filename)
+  };
+}
+function updateUrlValue(url, oldKey, newKey) {
+  if (!url) {
+    return url;
+  }
+  const encodedOldKey = oldKey.split("/").map((segment) => encodeURIComponent(segment)).join("/");
+  const encodedNewKey = newKey.split("/").map((segment) => encodeURIComponent(segment)).join("/");
+  return url.split(oldKey).join(newKey).split(encodedOldKey).join(encodedNewKey);
+}
+function formatItem(item) {
+  return {
+    collectionName: item.collectionName,
+    id: item.id,
+    storageId: item.storageId,
+    storageType: item.storageType,
+    oldKey: item.oldKey,
+    newKey: item.newKey,
+    sourceExists: item.sourceExists,
+    targetExists: item.targetExists,
+    status: item.status,
+    reason: item.reason
+  };
+}
+async function getFileCollectionNames(app) {
+  var _a;
+  const repository = app.db.getRepository("collections") || ((_a = app.db.getCollection("collections")) == null ? void 0 : _a.repository);
+  if (repository) {
+    const collections = await repository.find({
+      filter: {
+        "options.template": "file"
+      }
+    });
+    return Array.from(/* @__PURE__ */ new Set(["attachments", ...collections.map((collection) => collection.get("name"))]));
+  }
+  return Array.from(
+    /* @__PURE__ */ new Set([
+      "attachments",
+      ...Array.from(app.db.collections.values()).filter((collection) => collection.options.template === "file").map((collection) => collection.name)
+    ])
+  );
+}
+async function loadFileCollections(app) {
+  var _a, _b;
+  const collection = app.db.getCollection("collections");
+  if (!collection) {
+    return;
+  }
+  const repository = collection.repository;
+  await ((_a = repository.setApp) == null ? void 0 : _a.call(repository, app));
+  await ((_b = repository.load) == null ? void 0 : _b.call(repository));
+}
+async function repairCollectionAttachmentFilenames({
+  app,
+  fileManager,
+  collectionName,
+  result,
+  apply,
+  batchSize,
+  limit
+}) {
+  const collection = app.db.getCollection(collectionName);
+  if (!collection) {
+    return;
+  }
+  const Model = collection.model;
+  let lastId = null;
+  while (result.scanned < limit) {
+    const rows = await Model.findAll({
+      attributes: ["id", "path", "filename", "storageId", "url"],
+      where: lastId ? { id: { [import_sequelize.Op.gt]: lastId } } : void 0,
+      order: [["id", "ASC"]],
+      limit: Math.min(batchSize, limit - result.scanned)
+    });
+    if (!rows.length) {
+      break;
+    }
+    for (const row of rows) {
+      result.scanned += 1;
+      lastId = row.get("id");
+      const record = row.toJSON();
+      if (!containsInvisibleChars(record.path) && !containsInvisibleChars(record.filename)) {
+        continue;
+      }
+      const storage = fileManager.storagesCache.get(record.storageId);
+      const { path: newPath, filename: newFilename } = getRepairedAttachmentValues(record);
+      const oldKey = (0, import_utils.getFileKey)(record);
+      const newRecord = { ...record, path: newPath, filename: newFilename };
+      const newKey = (0, import_utils.getFileKey)(newRecord);
+      const item = {
+        collectionName,
+        id: lastId,
+        storageId: record.storageId,
+        storageType: (storage == null ? void 0 : storage.type) || "",
+        oldPath: record.path || "",
+        oldFilename: record.filename,
+        newPath,
+        newFilename,
+        oldKey,
+        newKey,
+        status: "pending"
+      };
+      result.candidates.push(item);
+      if (!storage) {
+        item.status = "skipped";
+        item.reason = "storage_not_found";
+        result.skipped.push(item);
+        continue;
+      }
+      if (oldKey === newKey) {
+        item.status = "skipped";
+        item.reason = "unchanged";
+        result.skipped.push(item);
+        continue;
+      }
+      try {
+        const StorageClass = fileManager.storageTypes.get(storage.type);
+        if (!StorageClass) {
+          item.status = "skipped";
+          item.reason = "storage_type_not_found";
+          result.skipped.push(item);
+          continue;
+        }
+        const storageInstance = new StorageClass(storage);
+        item.sourceExists = await storageInstance.exists(record);
+        if (!item.sourceExists) {
+          item.status = "skipped";
+          item.reason = "source_not_found";
+          result.skipped.push(item);
+          continue;
+        }
+        item.targetExists = await storageInstance.exists(newRecord);
+        if (item.targetExists) {
+          item.status = "skipped";
+          item.reason = "target_exists";
+          result.skipped.push(item);
+          continue;
+        }
+        if (apply) {
+          await storageInstance.copy(record, newRecord);
+          await row.update({
+            path: newPath,
+            filename: newFilename,
+            url: updateUrlValue(record.url, oldKey, newKey)
+          });
+          try {
+            const [deleted] = await storageInstance.delete([record]);
+            if (!deleted) {
+              item.reason = "delete_old_failed";
+            }
+          } catch (error) {
+            item.reason = `delete_old_failed: ${error.message}`;
+          }
+          item.status = "repaired";
+          result.repaired.push(item);
+        }
+      } catch (error) {
+        item.status = "failed";
+        item.reason = error.message;
+        result.failed.push(item);
+      }
+    }
+  }
+}
+async function repairAttachmentFilenames(app, fileManager, options = {}) {
+  const apply = !!options.apply;
+  const batchSize = options.batchSize || 500;
+  const limit = options.limit || Number.POSITIVE_INFINITY;
+  const result = {
+    dryRun: !apply,
+    scanned: 0,
+    candidates: [],
+    repaired: [],
+    skipped: [],
+    failed: []
+  };
+  if (!fileManager.storagesCache.size) {
+    await fileManager.loadStorages();
+  }
+  await loadFileCollections(app);
+  for (const collectionName of await getFileCollectionNames(app)) {
+    if (result.scanned >= limit) {
+      break;
+    }
+    await repairCollectionAttachmentFilenames({
+      app,
+      fileManager,
+      collectionName,
+      result,
+      apply,
+      batchSize,
+      limit
+    });
+  }
+  return result;
+}
+function registerRepairFilenamesCommand(app) {
+  const command = app.findCommand("file-manager") || app.command("file-manager");
+  command.command("repair-filenames").preload().option("--apply", "rename objects and update attachment records").option("--batch-size [batchSize]", "batch size for scanning attachments").option("--limit [limit]", "maximum number of attachment records to scan").action(async (options) => {
+    const fileManager = app.pm.get("file-manager");
+    const result = await repairAttachmentFilenames(app, fileManager, {
+      apply: !!options.apply,
+      batchSize: options.batchSize ? Number(options.batchSize) : void 0,
+      limit: options.limit ? Number(options.limit) : void 0
+    });
+    console.log(
+      JSON.stringify(
+        {
+          dryRun: result.dryRun,
+          scanned: result.scanned,
+          candidates: result.candidates.length,
+          repaired: result.repaired.length,
+          skipped: result.skipped.length,
+          failed: result.failed.length
+        },
+        null,
+        2
+      )
+    );
+    if (result.candidates.length) {
+      console.table(result.candidates.map(formatItem));
+    }
+  });
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  getRepairedAttachmentValues,
+  registerRepairFilenamesCommand,
+  repairAttachmentFilenames,
+  replaceInvisibleChars
+});

package/dist/server/server.d.ts

@@ -27,6 +27,7 @@ export type UploadFileOptions = {
 export declare class PluginFileManagerServer extends Plugin {
     storageTypes: Registry<StorageClassType>;
     storagesCache: Map<string | number, StorageModel>;
+    static staticImport(): Promise<void>;
     afterDestroy: (record: Model, options: any) => Promise<void>;
     registerStorageType(type: string, Type: StorageClassType): void;
     createFileRecord(options: FileRecordOptions): Promise<any>;

package/dist/server/server.js

@@ -54,6 +54,7 @@ var import_local = __toESM(require("./storages/local"));
 var import_s3 = __toESM(require("./storages/s3"));
 var import_tx_cos = __toESM(require("./storages/tx-cos"));
 var import_utils2 = require("./utils");
+var import_repair_filenames = require("./commands/repair-filenames");
 const DEFAULT_STORAGE_TYPE = import_constants.STORAGE_TYPE_LOCAL;
 class FileDeleteError extends Error {
   data;
@@ -66,6 +67,9 @@ class FileDeleteError extends Error {
 class PluginFileManagerServer extends import_server.Plugin {
   storageTypes = new import_utils.Registry();
   storagesCache = /* @__PURE__ */ new Map();
+  static async staticImport() {
+    import_server.Application.addCommand(import_repair_filenames.registerRepairFilenamesCommand);
+  }
   afterDestroy = async (record, options) => {
     var _a;
     const { collection } = record.constructor;

package/dist/server/storages/ali-oss.d.ts

@@ -18,7 +18,7 @@ declare class AliYunOssStorage {
         filename?: typeof getRandomFilename;
     });
     _handleFile(req: any, file: any, cb: any): any;
-    _removeFile(req: any, file: any, cb: any): any;
+    _removeFile(req: any, file: any, cb: any): Promise<any>;
 }
 export default class extends StorageType {
     static defaults(): {
@@ -34,6 +34,8 @@ export default class extends StorageType {
         };
     };
     make(): AliYunOssStorage;
+    exists(record: AttachmentModel): Promise<boolean>;
+    copy(source: AttachmentModel, target: AttachmentModel): Promise<void>;
     delete(records: AttachmentModel[]): Promise<[number, AttachmentModel[]]>;
 }
 export {};

package/dist/server/storages/ali-oss.js

@@ -90,11 +90,16 @@ class AliYunOssStorage {
       });
     }).catch(cb);
   }
-  _removeFile(req, file, cb) {
+  async _removeFile(req, file, cb) {
     if (!this.client) {
       return cb(ERROR_NO_CLIENT);
     }
-    this.client.delete(file.filename).then((result) => cb(null, result)).catch(cb);
+    try {
+      const result = await this.client.delete(file.filename);
+      cb(null, result);
+    } catch (error) {
+      cb(error);
+    }
   }
 }
 class ali_oss_default extends import__.StorageType {
@@ -118,6 +123,22 @@ class ali_oss_default extends import__.StorageType {
       filename: (0, import_utils.cloudFilenameGetter)(this.storage)
     });
   }
+  async exists(record) {
+    const { client } = this.make();
+    try {
+      await client.head((0, import_utils.getFileKey)(record));
+      return true;
+    } catch (error) {
+      if (["NoSuchKey", "NotFoundError"].includes(error.name)) {
+        return false;
+      }
+      throw error;
+    }
+  }
+  async copy(source, target) {
+    const { client } = this.make();
+    await client.copy((0, import_utils.getFileKey)(target), (0, import_utils.getFileKey)(source));
+  }
   async delete(records) {
     const { client } = this.make();
     const { deleted } = await client.deleteMulti(records.map(import_utils.getFileKey));

package/dist/server/storages/index.d.ts

@@ -24,6 +24,7 @@ export interface StorageModel {
     settings?: Record<string, any>;
 }
 export interface AttachmentModel {
+    id?: number;
     title: string;
     filename: string;
     mimetype?: string;
@@ -38,6 +39,8 @@ export declare abstract class StorageType {
     constructor(storage: StorageModel);
     abstract make(): StorageEngine;
     abstract delete(records: AttachmentModel[]): [number, AttachmentModel[]] | Promise<[number, AttachmentModel[]]>;
+    exists(record: AttachmentModel): Promise<boolean>;
+    copy(source: AttachmentModel, target: AttachmentModel): Promise<void>;
     getFileKey(record: AttachmentModel): any;
     getFileData(file: any, meta?: {}): {
         title: string;

package/dist/server/storages/index.js

@@ -52,6 +52,12 @@ class StorageType {
     return {};
   }
   static filenameKey;
+  async exists(record) {
+    throw new Error(`Storage type "${this.storage.type}" does not support object existence checks`);
+  }
+  async copy(source, target) {
+    throw new Error(`Storage type "${this.storage.type}" does not support object copy`);
+  }
   getFileKey(record) {
     return (0, import_utils2.getFileKey)(record);
   }

package/dist/server/storages/local.d.ts

@@ -27,6 +27,8 @@ export default class extends StorageType {
         };
     };
     make(): multer.StorageEngine;
+    exists(record: AttachmentModel): Promise<boolean>;
+    copy(source: AttachmentModel, target: AttachmentModel): Promise<void>;
     delete(records: AttachmentModel[]): Promise<[number, AttachmentModel[]]>;
     getFileURL(file: AttachmentModel, preview?: boolean): Promise<any>;
     getFileStream(file: AttachmentModel): Promise<{

package/dist/server/storages/local.js

@@ -92,6 +92,24 @@ class local_default extends import__.StorageType {
       filename: (0, import_utils2.diskFilenameGetter)(this.storage)
     });
   }
+  async exists(record) {
+    try {
+      await import_promises.default.stat(resolveSafePath(getDocumentRoot(this.storage), record.path, record.filename));
+      return true;
+    } catch (error) {
+      if (error.code === "ENOENT") {
+        return false;
+      }
+      throw error;
+    }
+  }
+  async copy(source, target) {
+    const documentRoot = getDocumentRoot(this.storage);
+    const sourcePath = resolveSafePath(documentRoot, source.path, source.filename);
+    const targetPath = resolveSafePath(documentRoot, target.path, target.filename);
+    await import_promises.default.mkdir(import_path.default.dirname(targetPath), { recursive: true });
+    await import_promises.default.copyFile(sourcePath, targetPath);
+  }
   async delete(records) {
     const documentRoot = getDocumentRoot(this.storage);
     let count = 0;

package/dist/server/storages/s3.d.ts

@@ -32,5 +32,7 @@ export default class extends StorageType {
     deleteS3Objects(bucketName: string, objects: string[]): Promise<{
         Deleted: any[];
     }>;
+    exists(record: AttachmentModel): Promise<boolean>;
+    copy(source: AttachmentModel, target: AttachmentModel): Promise<void>;
     delete(records: AttachmentModel[]): Promise<[number, AttachmentModel[]]>;
 }

package/dist/server/storages/s3.js

@@ -173,6 +173,32 @@ class s3_default extends import__.StorageType {
       Deleted
     };
   }
+  async exists(record) {
+    try {
+      await this.client.send(
+        new import_client_s3.HeadObjectCommand({
+          Bucket: this.storage.options.bucket,
+          Key: this.getFileKey(record)
+        })
+      );
+      return true;
+    } catch (error) {
+      if (["NotFound", "NoSuchKey", "NoSuchBucket"].includes(error.name)) {
+        return false;
+      }
+      throw error;
+    }
+  }
+  async copy(source, target) {
+    const sourceKey = this.getFileKey(source);
+    await this.client.send(
+      new import_client_s3.CopyObjectCommand({
+        Bucket: this.storage.options.bucket,
+        Key: this.getFileKey(target),
+        CopySource: `${this.storage.options.bucket}/${sourceKey.split("/").map((segment) => encodeURIComponent(segment)).join("/")}`
+      })
+    );
+  }
   async delete(records) {
     const { Deleted } = await this.deleteS3Objects(
       this.storage.options.bucket,

package/dist/server/storages/tx-cos.d.ts

@@ -36,6 +36,8 @@ export default class extends StorageType {
     };
     static filenameKey: string;
     make(): TxCosStorage;
+    exists(record: AttachmentModel): Promise<boolean>;
+    copy(source: AttachmentModel, target: AttachmentModel): Promise<void>;
     delete(records: AttachmentModel[]): Promise<[number, AttachmentModel[]]>;
 }
 export {};

package/dist/server/storages/tx-cos.js

@@ -160,6 +160,33 @@ class tx_cos_default extends import__.StorageType {
       filename: (0, import_utils.cloudFilenameGetter)(this.storage)
     });
   }
+  async exists(record) {
+    const { cos } = this.make();
+    try {
+      await (0, import_util.promisify)(cos.headObject).call(cos, {
+        Region: this.storage.options.Region,
+        Bucket: this.storage.options.Bucket,
+        Key: (0, import_utils.getFileKey)(record)
+      });
+      return true;
+    } catch (error) {
+      if (["NoSuchKey", "NotFound"].includes(error.name)) {
+        return false;
+      }
+      throw error;
+    }
+  }
+  async copy(source, target) {
+    const { cos } = this.make();
+    const sourceKey = (0, import_utils.getFileKey)(source);
+    const copySource = `${this.storage.options.Bucket}.cos.${this.storage.options.Region}.myqcloud.com/${sourceKey.split("/").map((segment) => encodeURIComponent(segment)).join("/")}`;
+    await (0, import_util.promisify)(cos.putObjectCopy).call(cos, {
+      Region: this.storage.options.Region,
+      Bucket: this.storage.options.Bucket,
+      Key: (0, import_utils.getFileKey)(target),
+      CopySource: copySource
+    });
+  }
   async delete(records) {
     const { cos } = this.make();
     const { Deleted } = await (0, import_util.promisify)(cos.deleteMultipleObject).call(cos, {

package/dist/server/utils.js

@@ -48,6 +48,13 @@ var import_utils = require("@nocobase/utils");
 var import_crypto = __toESM(require("crypto"));
 var import_path = __toESM(require("path"));
 var import_url_join = __toESM(require("url-join"));
+const INVALID_FILENAME_CHARS = /* @__PURE__ */ new Set(["<", ">", "?", "*", "|", ":", '"', "\\", "/"]);
+function sanitizeFilename(value) {
+  return Array.from(value).map((char) => {
+    const code = char.charCodeAt(0);
+    return code < 32 || code === 127 || INVALID_FILENAME_CHARS.has(char) ? "-" : char;
+  }).join("");
+}
 function normalizeOriginalname(file) {
   const originalname = file == null ? void 0 : file.originalname;
   if (!originalname) {
@@ -56,6 +63,9 @@ function normalizeOriginalname(file) {
   if (Buffer.isBuffer(originalname)) {
     return originalname.toString("utf8");
   }
+  if (Array.from(originalname).some((char) => char.charCodeAt(0) > 255)) {
+    return originalname;
+  }
   const decoded = Buffer.from(originalname, "binary").toString("utf8");
   if (decoded.includes("\uFFFD")) {
     return originalname;
@@ -64,13 +74,13 @@ function normalizeOriginalname(file) {
 }
 function getFilename(req, file, cb) {
   const originalname = normalizeOriginalname(file);
-  const baseName = import_path.default.basename(originalname.replace(/[<>?*|:"\\/]/g, "-"), import_path.default.extname(originalname));
+  const baseName = import_path.default.basename(sanitizeFilename(originalname), import_path.default.extname(originalname));
   cb(null, `${baseName}-${(0, import_utils.uid)(6)}${import_path.default.extname(originalname)}`);
 }
 function getOriginalFilename(file) {
   const originalname = normalizeOriginalname(file);
   const extname = import_path.default.extname(originalname);
-  const baseName = import_path.default.basename(originalname.replace(/[<>?*|:"\\/]/g, "-"), extname);
+  const baseName = import_path.default.basename(sanitizeFilename(originalname), extname);
   return `${baseName}${extname}`;
 }
 const cloudFilenameGetter = (storage) => (req, file, cb) => {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nocobase/plugin-file-manager",
-  "version": "2.0.57",
+  "version": "2.0.59",
   "displayName": "File manager",
   "displayName.ru-RU": "Менеджер файлов",
   "displayName.zh-CN": "文件管理器",
@@ -27,7 +27,6 @@
     "antd": "5.x",
     "axios": "^1.7.0",
     "cos-nodejs-sdk-v5": "^2.11.14",
-    "file-type": "^21.0.0",
     "koa-static": "^5.0.0",
     "mime-match": "^1.0.2",
     "mime-types": "^3.0.1",
@@ -41,6 +40,9 @@
     "supertest": "^6.1.6",
     "url-join": "4.0.1"
   },
+  "dependencies": {
+    "file-type": "^21.0.0"
+  },
   "peerDependencies": {
     "@nocobase/actions": "2.x",
     "@nocobase/client": "2.x",
@@ -53,5 +55,5 @@
     "Collections",
     "Collection fields"
   ],
-  "gitHead": "629bf05e63bca0cbd60fffb87057d45bda5d9222"
+  "gitHead": "8b3a0cfadf6cb2da4a7e9213ef0d8d092d885c4a"
 }