@kevisual/oss 0.0.16 → 0.0.18

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@kevisual/oss",
-  "version": "0.0.16",
+  "version": "0.0.18",
   "main": "dist/index.js",
   "scripts": {
     "build": "bun run bun.config.ts"
@@ -14,13 +14,13 @@
   "license": "MIT",
   "type": "module",
   "devDependencies": {
-    "@types/bun": "^1.3.5",
-    "@types/node": "^25.0.3",
+    "@aws-sdk/client-s3": "^3.978.0",
+    "@kevisual/use-config": "^1.0.28",
+    "@types/bun": "^1.3.8",
+    "@types/node": "^25.1.0",
     "bun-plugin-dts": "^0.3.0",
     "dotenv": "^17.2.3",
-    "minio": "^8.0.6",
-    "@aws-sdk/client-s3": "^3.0.0",
-    "es-toolkit": "^1.43.0",
+    "es-toolkit": "^1.44.0",
     "fast-glob": "^3.3.3"
   },
   "exports": {
@@ -28,18 +28,13 @@
       "import": "./dist/index.js",
       "types": "./dist/index.d.ts"
     },
-    "./config": {
-      "import": "./dist/services/config.js",
-      "types": "./dist/services/config.d.ts"
-    },
     "./services": {
-      "import": "./dist/services/index.js",
-      "types": "./dist/services/index.d.ts"
+      "import": "./dist/services.js",
+      "types": "./dist/services.d.ts"
     },
     "./s3.ts": "./src/s3/core.ts"
   },
   "publishConfig": {
     "access": "public"
-  },
-  "dependencies": {}
+  }
 }

package/src/index.ts

@@ -1,244 +1,3 @@
-import { Client, ItemBucketMetadata } from 'minio';
-import { ListFileObject, ListObjectResult, OssBaseOperation } from './core/type.ts';
-import { hash } from './util/hash.ts';
-import { copyObject } from './core/copy-object.ts';
-import { getContentType } from './util/get-content-type.ts';
-import { omit } from 'es-toolkit'
-export type OssBaseOptions<T = { [key: string]: any }> = {
-  /**
-   * 已经初始化好的minio client
-   */
-  client: Client;
-  /**
-   * 桶名
-   */
-  bucketName: string;
-  /**
-   * 前缀
-   */
-  prefix?: string;
-} & T;
+export * from './s3/core.ts'
 
-export class OssBase implements OssBaseOperation {
-  client?: Client;
-  bucketName: string;
-  prefix = '';
-  /**
-   * 计算字符串或者对象的的md5值
-   */
-  hash = hash;
-  constructor(opts: OssBaseOptions) {
-    if (!opts.client) {
-      throw new Error('client is required');
-    }
-    this.bucketName = opts.bucketName;
-    this.client = opts.client;
-    this.prefix = opts?.prefix ?? '';
-  }
-
-  setPrefix(prefix: string) {
-    this.prefix = prefix;
-  }
-
-  async getObject(objectName: string) {
-    const bucketName = this.bucketName;
-    const obj = await this.client.getObject(bucketName, `${this.prefix}${objectName}`);
-    return obj;
-  }
-
-  async getJson(objectName: string): Promise<Record<string, any>> {
-    const obj = await this.getObject(objectName);
-    return new Promise((resolve, reject) => {
-      let data = '';
-      obj.on('data', (chunk) => {
-        data += chunk;
-      });
-      obj.on('end', () => {
-        try {
-          const jsonData = JSON.parse(data);
-          resolve(jsonData);
-        } catch (error) {
-          reject(new Error('Failed to parse JSON'));
-        }
-      });
-      obj.on('error', (err) => {
-        reject(err);
-      });
-    });
-  }
-  /**
-   * 上传文件, 当是流的时候，中断之后的etag会变，所以传递的时候不要嵌套async await，例如 busboy 监听文件流内部的时候，不要用check
-   * @param objectName
-   * @param data
-   * @param metaData
-   * @param options 如果文件本身存在，则复制原有的meta的内容
-   * @returns
-   */
-  async putObject(
-    objectName: string,
-    data: Buffer | string | Object,
-    metaData: ItemBucketMetadata = {},
-    opts?: { check?: boolean; isStream?: boolean; size?: number },
-  ) {
-    let putData: Buffer | string;
-    let size: number = opts?.size;
-    const isStream = opts?.isStream;
-    if (!isStream) {
-      if (typeof data === 'string') {
-        putData = data;
-        size = putData.length;
-      } else {
-        putData = JSON.stringify(data);
-        size = putData.length;
-      }
-    } else {
-      putData = data as any;
-      // 对于流式上传，如果没有提供 size，会导致多部分上传，ETag 会是 ****-1 格式
-      // 必须提供准确的 size 才能得到标准的 MD5 格式 ETag
-      if (!size) {
-        throw new Error('Stream upload requires size parameter to avoid multipart upload and get standard MD5 ETag');
-      }
-    }
-    if (opts?.check) {
-      const obj = await this.statObject(objectName, true);
-      if (obj) {
-        const omitMeda = ['size', 'content-type', 'cache-control', 'app-source'];
-        const objMeta = JSON.parse(JSON.stringify(omit(obj.metaData, omitMeda)));
-        metaData = {
-          ...objMeta,
-          ...metaData,
-        };
-      }
-    }
-
-    const bucketName = this.bucketName;
-    const obj = await this.client.putObject(bucketName, `${this.prefix}${objectName}`, putData, size, metaData);
-    return obj;
-  }
-
-  async deleteObject(objectName: string) {
-    const bucketName = this.bucketName;
-    const obj = await this.client.removeObject(bucketName, `${this.prefix}${objectName}`);
-    return obj;
-  }
-
-  async listObjects<IS_FILE = false>(objectName: string, opts?: { recursive?: boolean; startAfter?: string }) {
-    const bucketName = this.bucketName;
-    const prefix = `${this.prefix}${objectName}`;
-    const res = await new Promise((resolve, reject) => {
-      let res: any[] = [];
-      let hasError = false;
-      this.client
-        .listObjectsV2(bucketName, prefix, opts?.recursive ?? false, opts?.startAfter)
-        .on('data', (data) => {
-          res.push(data);
-        })
-        .on('error', (err) => {
-          console.error('minio error', prefix, err);
-          hasError = true;
-        })
-        .on('end', () => {
-          if (hasError) {
-            reject();
-            return;
-          } else {
-            resolve(res);
-          }
-        });
-    });
-    return res as IS_FILE extends true ? ListFileObject[] : ListObjectResult[];
-  }
-
-  async fPutObject(objectName: string, filePath: string, metaData?: ItemBucketMetadata) {
-    const bucketName = this.bucketName;
-    const obj = await this.client.fPutObject(bucketName, `${this.prefix}${objectName}`, filePath, metaData);
-    return obj as any;
-  }
-  /**
-   * 获取完整的对象名称
-   * @param objectName
-   * @returns
-   */
-  async getObjectName(objectName: string) {
-    return `${this.prefix}${objectName}`;
-  }
-  async statObject(objectName: string, checkFile = true) {
-    const bucketName = this.bucketName;
-    try {
-      const obj = await this.client.statObject(bucketName, `${this.prefix}${objectName}`);
-      return obj;
-    } catch (e) {
-      if (e.code === 'NotFound') {
-        return null;
-      }
-      throw e;
-    }
-  }
-  /**
-   * 检查文件hash是否一致
-   * @param objectName
-   * @param hash
-   * @returns
-   */
-  async checkObjectHash(
-    objectName: string,
-    hash: string,
-    meta?: ItemBucketMetadata,
-  ): Promise<{ success: boolean; metaData: ItemBucketMetadata | null; obj: any; equalMeta?: boolean }> {
-    const obj = await this.statObject(`${this.prefix}${objectName}`, true);
-    if (!obj) {
-      return { success: false, metaData: null, obj: null, equalMeta: false };
-    }
-    let metaData: ItemBucketMetadata = {};
-    const omitMeda = ['content-type', 'cache-control', 'app-source'];
-    const objMeta = omit(obj.metaData, omitMeda);
-    metaData = {
-      ...objMeta,
-    };
-    let equalMeta = false;
-    if (meta) {
-      equalMeta = JSON.stringify(metaData) === JSON.stringify(meta);
-    }
-    return { success: obj.etag === hash, metaData, obj, equalMeta };
-  }
-  getMetadata(pathname: string, meta: ItemBucketMetadata = { 'app-source': 'user-app' }) {
-    const isHtml = pathname.endsWith('.html');
-    if (isHtml) {
-      meta = {
-        ...meta,
-        'content-type': 'text/html; charset=utf-8',
-        'cache-control': 'no-cache',
-      };
-    } else {
-      meta = {
-        ...meta,
-        'content-type': getContentType(pathname),
-        'cache-control': 'max-age=31536000, immutable',
-      };
-    }
-    return meta;
-  }
-
-  async copyObject(sourceObject: any, targetObject: any) {
-    const bucketName = this.bucketName;
-    const obj = await this.client.copyObject(bucketName, sourceObject, targetObject);
-    return obj;
-  }
-  async replaceObject(objectName: string, meta: { [key: string]: string }) {
-    const { bucketName, client } = this;
-    return copyObject({ bucketName, client, objectName: `${this.prefix}${objectName}`, newMetadata: meta });
-  }
-  static create<T extends OssBase, U>(this: new (opts: OssBaseOptions<U>) => T, opts: OssBaseOptions<U>): T {
-    return new this(opts);
-  }
-
-  static fromBase<T extends OssBase, U>(this: new (opts: OssBaseOptions<U>) => T, createOpts: { oss: OssBase; opts: Partial<OssBaseOptions<U>> }): T {
-    const base = createOpts.oss;
-    const opts = createOpts.opts as any;
-    return new this({
-      client: base.client,
-      bucketName: base.bucketName,
-      ...opts,
-    });
-  }
-}
+export * from './s3/type.ts'

package/src/s3/core.ts

@@ -127,9 +127,16 @@ export class OssBase implements OssBaseOperation {
         contentLength = Buffer.byteLength(putData);
       }
     } else {
-      putData = data as Readable;
+      // Stream 上传：自动读取到 Buffer 以获取 contentLength
       if (!contentLength) {
-        throw new Error('Stream upload requires size parameter');
+        const chunks: Buffer[] = [];
+        for await (const chunk of data as Readable) {
+          chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
+        }
+        putData = Buffer.concat(chunks);
+        contentLength = putData.length;
+      } else {
+        putData = data as Readable;
       }
     }
 
@@ -180,24 +187,45 @@ export class OssBase implements OssBaseOperation {
     filePath: string,
     metaData?: ItemBucketMetadata,
   ): Promise<UploadedObjectInfo> {
-    const fileStream = fs.createReadStream(filePath);
     const stat = fs.statSync(filePath);
-
     const { standardHeaders, customMetadata } = extractStandardHeaders(metaData || {});
 
-    const command = new PutObjectCommand({
-      Bucket: this.bucketName,
-      Key: `${this.prefix}${objectName}`,
-      Body: fileStream,
-      ContentLength: stat.size,
-      ContentType: standardHeaders.ContentType || getContentType(filePath),
-      CacheControl: standardHeaders.CacheControl,
-      ContentDisposition: standardHeaders.ContentDisposition,
-      ContentEncoding: standardHeaders.ContentEncoding,
-      ContentLanguage: standardHeaders.ContentLanguage,
-      Expires: standardHeaders.Expires,
-      Metadata: customMetadata,
-    });
+    const THRESHOLD = 5 * 1024 * 1024; // 5MB
+    let command: PutObjectCommand;
+
+    if (stat.size < THRESHOLD) {
+      // 小文件：读取到内存再上传，避免流式传输导致的 IncompleteBody 错误
+      const fileBuffer = await fs.promises.readFile(filePath);
+      command = new PutObjectCommand({
+        Bucket: this.bucketName,
+        Key: `${this.prefix}${objectName}`,
+        Body: fileBuffer,
+        ContentLength: fileBuffer.length,
+        ContentType: standardHeaders.ContentType || getContentType(filePath),
+        CacheControl: standardHeaders.CacheControl,
+        ContentDisposition: standardHeaders.ContentDisposition,
+        ContentEncoding: standardHeaders.ContentEncoding,
+        ContentLanguage: standardHeaders.ContentLanguage,
+        Expires: standardHeaders.Expires,
+        Metadata: customMetadata,
+      });
+    } else {
+      // 大文件：使用流式上传
+      const fileStream = fs.createReadStream(filePath);
+      command = new PutObjectCommand({
+        Bucket: this.bucketName,
+        Key: `${this.prefix}${objectName}`,
+        Body: fileStream,
+        ContentLength: stat.size,
+        ContentType: standardHeaders.ContentType || getContentType(filePath),
+        CacheControl: standardHeaders.CacheControl,
+        ContentDisposition: standardHeaders.ContentDisposition,
+        ContentEncoding: standardHeaders.ContentEncoding,
+        ContentLanguage: standardHeaders.ContentLanguage,
+        Expires: standardHeaders.Expires,
+        Metadata: customMetadata,
+      });
+    }
 
     const response = await this.client.send(command);
     return {
@@ -217,6 +245,11 @@ export class OssBase implements OssBaseOperation {
     });
     await this.client.send(command);
   }
+  async deleteObjects(objectNameList: string[]): Promise<void> {
+    for (const objectName of objectNameList) {
+      await this.deleteObject(objectName);
+    }
+  }
 
   /**
    * 列出对象
@@ -285,16 +318,34 @@ export class OssBase implements OssBaseOperation {
       const response = await this.client.send(command);
 
       return {
+        standardHeaders: {
+          contentType: response.ContentType,
+          cacheControl: response.CacheControl,
+          contentDisposition: response.ContentDisposition,
+          contentEncoding: response.ContentEncoding,
+          contentLanguage: response.ContentLanguage,
+        },
         size: response.ContentLength || 0,
-        etag: response.ETag?.replace(/"/g, '') || '',
+        etag: response.ETag?.replace?.(/"/g, '') || '',
         lastModified: response.LastModified || new Date(),
         metaData: (response.Metadata as ItemBucketMetadata) || {},
         versionId: response.VersionId || null,
       };
     } catch (e: any) {
-      if (checkFile && (e.name === 'NotFound' || e.$metadata?.httpStatusCode === 404)) {
+      // 检查是否是 404 错误 - 支持多种 S3 兼容存储的错误格式
+      const isNotFound =
+        e.name === 'NotFound' ||
+        e.name === 'NoSuchBucket' ||
+        e.name === 'NoSuchKey' ||
+        e.code === 'NotFound' ||
+        e.code === 'NoSuchBucket' ||
+        e.code === 'NoSuchKey' ||
+        e.$metadata?.httpStatusCode === 404;
+
+      if (checkFile && isNotFound) {
         return null;
       }
+      console.error('statObject error', e);
       throw e;
     }
   }
@@ -322,7 +373,7 @@ export class OssBase implements OssBaseOperation {
     if (!obj) {
       return { success: false, metaData: null, obj: null, equalMeta: false };
     }
-    const omitMeta = ['content-type', 'cache-control', 'app-source'];
+    const omitMeta = ['Content-Type', 'Cache-Control', 'app-source'];
     const metaData = omit(obj.metaData, omitMeta);
     let equalMeta = false;
     if (meta) {
@@ -341,14 +392,14 @@ export class OssBase implements OssBaseOperation {
     if (isHtml) {
       meta = {
         ...meta,
-        'content-type': 'text/html; charset=utf-8',
-        'cache-control': 'no-cache',
+        'Content-Type': 'text/html; charset=utf-8',
+        'Cache-Control': 'no-cache',
       };
     } else {
       meta = {
         ...meta,
-        'content-type': getContentType(pathname),
-        'cache-control': 'max-age=31536000, immutable',
+        'Content-Type': getContentType(pathname),
+        'Cache-Control': 'max-age=31536000, immutable',
       };
     }
     return meta;
@@ -422,4 +473,11 @@ export class OssBase implements OssBaseOperation {
       ...opts,
     });
   }
+}
+
+export const getStram = (data: GetObjectCommandOutput) => {
+  if (data.Body) {
+    return data.Body as Readable;
+  }
+  throw new Error('Object body is empty');
 }

package/src/s3/type.ts

@@ -17,6 +17,13 @@ export type StatObjectResult = {
   lastModified: Date;
   metaData: ItemBucketMetadata;
   versionId?: string | null;
+  standardHeaders: {
+    contentType?: string;
+    cacheControl?: string;
+    contentDisposition?: string;
+    contentEncoding?: string;
+    contentLanguage?: string;
+  }
 };
 
 export type ListFileObject = {

package/src/services/config.ts

@@ -1,5 +1,5 @@
-import { OssBase, OssBaseOptions } from '../index.ts';
-import { OssService } from '../core/type.ts';
+import { OssBase, OssBaseOptions } from '../s3/core.ts';
+import { OssService } from '../s3/type.ts';
 import * as util from '../util/index.ts';
 
 export class ConfigOssService extends OssBase implements OssService {

package/src/services/index.ts

@@ -6,4 +6,4 @@ export * from '../util/download.ts';
 
 export * from '../util/index.ts';
 
-export * from '../core/type.ts';
+export * from '../s3/type.ts';

package/src/test/common.ts

@@ -6,7 +6,7 @@ import { S3Client, ListObjectsV2Command, GetBucketMetadataConfigurationCommand,
 export const s3Client = new S3Client({
   credentials: {
     accessKeyId: process.env.S3_ACCESS_KEY_ID || '',
-    secretAccessKey: process.env.S3_SECRET_ACCESS_KEY || '',
+    secretAccessKey: process.env.S3_ACCESS_KEY_SECRET || '',
   },
   region: process.env.S3_REGION,
   endpoint: 'https://tos-s3-cn-shanghai.volces.com',

package/src/util/download.ts

@@ -1,73 +1,73 @@
-import { ServerResponse } from 'node:http';
-import { BucketItemStat } from 'minio';
-import fs from 'node:fs';
-import path from 'node:path';
+// import { ServerResponse } from 'node:http';
+// import { BucketItemStat } from 'minio';
+// import fs from 'node:fs';
+// import path from 'node:path';
 
-const viewableExtensions = ['jpg', 'jpeg', 'png', 'gif', 'svg', 'webp', 'mp4', 'webm', 'mp3', 'wav', 'ogg', 'pdf', 'doc', 'docx', 'xls', 'xlsx', 'ppt', 'pptx'];
-import { OssBase } from '../index.ts';
-/**
- * 过滤 metaData 中的 key, 去除 password, accesskey, secretkey，
- * 并返回过滤后的 metaData
- * @param metaData
- * @returns
- */
-export const filterMetaDataKeys = (metaData: Record<string, string>, clearKeys: string[] = []) => {
-  const keys = Object.keys(metaData);
-  // remove X-Amz- meta data
-  const removeKeys = ['password', 'accesskey', 'secretkey', ...clearKeys];
-  const filteredKeys = keys.filter((key) => !removeKeys.includes(key));
-  return filteredKeys.reduce((acc, key) => {
-    acc[key] = metaData[key];
-    return acc;
-  }, {} as Record<string, string>);
-};
-type SendObjectOptions = {
-  res: ServerResponse;
-  client: OssBase;
-  objectName: string;
-  isDownload?: boolean;
-};
-export const NotFoundFile = (res: ServerResponse, msg?: string, code = 404) => {
-  res.writeHead(code, { 'Content-Type': 'text/plain' });
-  res.end(msg || 'Not Found File');
-  return;
-};
-export const sendObject = async ({ res, objectName, client, isDownload = false }: SendObjectOptions) => {
-  let stat: BucketItemStat;
-  try {
-    stat = await client.statObject(objectName);
-  } catch (e) {
-  } finally {
-    if (!stat || stat.size === 0) {
-      return NotFoundFile(res);
-    }
-    const contentLength = stat.size;
-    const etag = stat.etag;
-    const lastModified = stat.lastModified.toISOString();
-    const filename = objectName.split('/').pop() || 'no-file-name-download'; // Extract filename from objectName
-    const fileExtension = filename.split('.').pop()?.toLowerCase() || '';
-    const filteredMetaData = filterMetaDataKeys(stat.metaData, ['size', 'etag', 'last-modified']);
-    const contentDisposition = viewableExtensions.includes(fileExtension) && !isDownload ? 'inline' : `attachment; filename="${filename}"`;
+// const viewableExtensions = ['jpg', 'jpeg', 'png', 'gif', 'svg', 'webp', 'mp4', 'webm', 'mp3', 'wav', 'ogg', 'pdf', 'doc', 'docx', 'xls', 'xlsx', 'ppt', 'pptx'];
+// import { OssBase } from '../index.ts';
+// /**
+//  * 过滤 metaData 中的 key, 去除 password, accesskey, secretkey，
+//  * 并返回过滤后的 metaData
+//  * @param metaData
+//  * @returns
+//  */
+// export const filterMetaDataKeys = (metaData: Record<string, string>, clearKeys: string[] = []) => {
+//   const keys = Object.keys(metaData);
+//   // remove X-Amz- meta data
+//   const removeKeys = ['password', 'accesskey', 'secretkey', ...clearKeys];
+//   const filteredKeys = keys.filter((key) => !removeKeys.includes(key));
+//   return filteredKeys.reduce((acc, key) => {
+//     acc[key] = metaData[key];
+//     return acc;
+//   }, {} as Record<string, string>);
+// };
+// type SendObjectOptions = {
+//   res: ServerResponse;
+//   client: OssBase;
+//   objectName: string;
+//   isDownload?: boolean;
+// };
+// export const NotFoundFile = (res: ServerResponse, msg?: string, code = 404) => {
+//   res.writeHead(code, { 'Content-Type': 'text/plain' });
+//   res.end(msg || 'Not Found File');
+//   return;
+// };
+// export const sendObject = async ({ res, objectName, client, isDownload = false }: SendObjectOptions) => {
+//   let stat: BucketItemStat;
+//   try {
+//     stat = await client.statObject(objectName);
+//   } catch (e) {
+//   } finally {
+//     if (!stat || stat.size === 0) {
+//       return NotFoundFile(res);
+//     }
+//     const contentLength = stat.size;
+//     const etag = stat.etag;
+//     const lastModified = stat.lastModified.toISOString();
+//     const filename = objectName.split('/').pop() || 'no-file-name-download'; // Extract filename from objectName
+//     const fileExtension = filename.split('.').pop()?.toLowerCase() || '';
+//     const filteredMetaData = filterMetaDataKeys(stat.metaData, ['size', 'etag', 'last-modified']);
+//     const contentDisposition = viewableExtensions.includes(fileExtension) && !isDownload ? 'inline' : `attachment; filename="${filename}"`;
 
-    res.writeHead(200, {
-      'Content-Length': contentLength,
-      etag,
-      'last-modified': lastModified,
-      'Content-Disposition': contentDisposition,
-      ...filteredMetaData,
-    });
-    const objectStream = await client.getObject(objectName);
+//     res.writeHead(200, {
+//       'Content-Length': contentLength,
+//       etag,
+//       'last-modified': lastModified,
+//       'Content-Disposition': contentDisposition,
+//       ...filteredMetaData,
+//     });
+//     const objectStream = await client.getObject(objectName);
 
-    objectStream.pipe(res, { end: true });
-  }
-};
+//     objectStream.pipe(res, { end: true });
+//   }
+// };
 
-export const downloadObject = async ({ objectName, client, filePath }: Pick<SendObjectOptions, 'objectName' | 'client'> & { filePath: string }) => {
-  const objectStream = await client.getObject(objectName);
-  const dir = path.dirname(filePath);
-  if (!fs.existsSync(dir)) {
-    fs.mkdirSync(dir, { recursive: true });
-  }
-  objectStream.pipe(fs.createWriteStream(filePath));
-  return objectStream;
-};
+// export const downloadObject = async ({ objectName, client, filePath }: Pick<SendObjectOptions, 'objectName' | 'client'> & { filePath: string }) => {
+//   const objectStream = await client.getObject(objectName);
+//   const dir = path.dirname(filePath);
+//   if (!fs.existsSync(dir)) {
+//     fs.mkdirSync(dir, { recursive: true });
+//   }
+//   objectStream.pipe(fs.createWriteStream(filePath));
+//   return objectStream;
+// };