@dd-code/oss-uploader 0.1.0 → 0.1.2

package/src/providers/huawei/HuaweiObsClient.ts

@@ -10,25 +10,59 @@ const ObsClient = require('esdk-obs-nodejs');
  */
 export class HuaweiObsClient implements StorageClient {
   private readonly client: InstanceType<typeof ObsClient>;
-
+  private readonly whiteList: string[];
   constructor(private readonly config: HuaweiObsConfig) {
     this.client = new ObsClient({
       access_key_id: config.accessKey,
       secret_access_key: config.secretKey,
       server: config.endpoint,
     });
+    this.whiteList = JSON.parse(process.env.FILE_RE_WHITE_LIST as string)
+  }
+
+  /**
+   * 按前缀列举对象 key（分页拉全），用于上传前一次拉取、内存判断存在，避免每个文件都 head。
+   */
+  async listObjectKeys(bucket: string, prefix: string): Promise<string[]> {
+    const keys: string[] = [];
+    let marker: string | undefined;
+    const pageSize = 1000;
+    do {
+      const param: Record<string, unknown> = {
+        Bucket: bucket,
+        MaxKeys: pageSize,
+        Prefix: prefix || undefined,
+      };
+      if (marker) param.Marker = marker;
+      const result = await this.client.listObjects(param);
+      if (result.CommonMsg.Status > 300) {
+        throw new Error(
+          `OBS listObjects 失败: ${result.CommonMsg.Status} ${result.CommonMsg.Code} ${result.CommonMsg.Message}`,
+        );
+      }
+      const contents = result.InterfaceResult?.Contents ?? [];
+      for (const item of contents) {
+        if (item.Key) keys.push(item.Key);
+      }
+      const truncated = result.InterfaceResult?.IsTruncated === 'true';
+      marker = truncated ? result.InterfaceResult?.NextMarker : undefined;
+    } while (marker);
+    return keys;
   }
 
   /**
    * 调用 OBS getObjectMetadata 查元数据，判断对象是否存在（同路径即跳过上传）。
+   * 当使用 listObjectKeys 时，上传流程会优先用内存 Set 判断，不再逐文件调用本方法。
    */
   async headObject(bucket: string, key: string): Promise<HeadObjectResult> {
     try {
+      if(this.whiteList.some(item => key.includes(item))) {
+        return { exists: false };
+      }
       const result = await this.client.getObjectMetadata({
         Bucket: bucket,
         Key: key,
       });
-
       if (result.CommonMsg.Status <= 300) {
         return { exists: true };
       }
@@ -49,16 +83,23 @@ export class HuaweiObsClient implements StorageClient {
     }
   }
 
-  /** 调用 OBS putObject 上传对象 */
+  /** 调用 OBS putObject 上传对象（支持 Body 或 SourceFile 本地路径，SourceFile 由 SDK 直接读文件避免 Buffer 问题） */
   async putObject(options: PutObjectOptions): Promise<void> {
-    const { bucket, key, body, contentType } = options;
-    // console.log('putObject', { bucket, key, endpoint: this.config.endpoint, accessKey: this.config.accessKey, secretKey: this.config.secretKey })
-    const result = await this.client.putObject({
+    const { bucket, key, contentType } = options;
+    const param: Record<string, unknown> = {
       Bucket: bucket,
       Key: key,
-      Body: body,
-      // ContentType: contentType,
-    });
+      ContentType: contentType,
+    };
+    if (options.sourceFile) {
+      param.SourceFile = options.sourceFile;
+    } else if (options.body) {
+      param.Body = options.body;
+    } else {
+      throw new Error('putObject 需要 body 或 sourceFile');
+    }
+
+    const result = await this.client.putObject(param);
 
     if (result.CommonMsg.Status > 300) {
       const { Status, Code, Message, RequestId } = result.CommonMsg;

package/src/core/UploadFlowTemplate.ts

@@ -1,119 +0,0 @@
-import fs from 'fs/promises';
-import path from 'path';
-import { Reporter, UploadSummary, UploadFileResult } from './reporter';
-import { StorageClient } from './StorageClient';
-import { shouldInclude } from './filters';
-import type { FileProcessor } from './fileProcessor';
-
-/** 上传流程的上下文：环境、bucket、前缀、本地目录、存储客户端、Reporter、CDN 根地址、上传前处理器等 */
-export interface UploadContext {
-  env?: string;
-  bucket: string;
-  basePrefix: string;
-  pathPrefix?: string;
-  localDir: string;
-  include?: string[];
-  exclude?: string[];
-  storageClient: StorageClient;
-  reporter?: Reporter;
-  /** CDN 加速根地址，配置后可为每个文件生成 accessUrl */
-  cdnBaseUrl?: string;
-  /** 上传前对文件内容的处理器（如图片压缩），未配置则直接上传原内容 */
-  fileProcessor?: FileProcessor;
-}
-
-/** 本地文件信息：绝对路径与相对路径 */
-export interface LocalFile {
-  absolutePath: string;
-  relativePath: string;
-}
-
-/**
- * 上传流程模板（Template Method）。
- * 定义固定步骤：onStart -> collectFiles -> filterFiles -> uploadFiles -> onComplete，
- * 子类实现 collectFiles 与 uploadFiles，其余步骤可复用或重写。
- */
-export abstract class UploadFlowTemplate {
-  protected constructor(protected readonly context: UploadContext) {}
-
-  /** 执行完整上传流程 */
-  async execute(): Promise<UploadSummary> {
-    const { reporter } = this.context;
-
-    if (reporter?.onStart) {
-      await reporter.onStart({
-        env: this.context.env,
-        bucket: this.context.bucket,
-        basePrefix: this.context.basePrefix,
-        pathPrefix: this.context.pathPrefix,
-        localDir: this.context.localDir,
-        cdnBaseUrl: this.context.cdnBaseUrl,
-      });
-    }
-
-    const files = await this.collectFiles();
-    const filtered = this.filterFiles(files);
-    const summary = await this.uploadFiles(filtered);
-
-    if (reporter?.onComplete) {
-      await reporter.onComplete(summary);
-    }
-
-    return summary;
-  }
-
-  protected abstract collectFiles(): Promise<LocalFile[]>;
-  protected abstract uploadFiles(files: LocalFile[]): Promise<UploadSummary>;
-
-  /** 使用 include/exclude 过滤文件列表 */
-  protected filterFiles(files: LocalFile[]): LocalFile[] {
-    const { include, exclude } = this.context;
-    return files.filter((f) => shouldInclude(f.relativePath, include, exclude));
-  }
-
-  /**
-   * 构造远端 key：basePrefix / env / pathPrefix / relativePath。
-   * 保持目录结构，且 base 前缀来自 config，不对外暴露。
-   */
-  protected buildKey(relativePath: string): string {
-    const parts: string[] = [];
-    if (this.context.basePrefix) parts.push(this.context.basePrefix);
-    if (this.context.env) parts.push(this.context.env);
-    if (this.context.pathPrefix) parts.push(this.context.pathPrefix);
-
-    parts.push(relativePath.replace(/\\/g, '/'));
-    return parts.join('/');
-  }
-
-  /** 递归遍历目录，收集所有文件的绝对路径与相对路径 */
-  protected async walkDir(rootDir: string, currentDir = ''): Promise<LocalFile[]> {
-    const dirPath = path.join(rootDir, currentDir);
-    const entries = await fs.readdir(dirPath, { withFileTypes: true });
-    const files: LocalFile[] = [];
-
-    for (const entry of entries) {
-      const rel = path.join(currentDir, entry.name);
-      const abs = path.join(rootDir, rel);
-
-      if (entry.isDirectory()) {
-        files.push(...(await this.walkDir(rootDir, rel)));
-      } else if (entry.isFile()) {
-        files.push({
-          absolutePath: abs,
-          relativePath: rel.replace(/\\/g, '/'),
-        });
-      }
-    }
-
-    return files;
-  }
-
-  /** 将单个文件的上传结果交给 Reporter */
-  protected async reportFileResult(result: UploadFileResult): Promise<void> {
-    const { reporter } = this.context;
-    if (reporter?.onFileResult) {
-      await reporter.onFileResult(result);
-    }
-  }
-}
-