@dd-code/oss-uploader 0.1.0 → 0.1.2

package/src/core/DirectoryUploadFlow.ts

@@ -1,96 +1,17 @@
-import fs from 'fs/promises';
-import { UploadFlowTemplate, UploadContext, LocalFile } from './UploadFlowTemplate';
-import { UploadSummary } from './reporter';
-import { buildCdnAccessUrl } from './urlHelper';
+import { UploadService, type UploadDirectoryOptions } from './UploadService';
 
 /**
- * 目录上传流程的具体实现：遍历本地目录，按 key 规则上传，云端同名则跳过。
- * 规则：先 headObject，exists 则 skipped，否则 putObject。
+ * 目录上传的具体实现：继承 UploadService，按目录遍历上传，云端已存在则跳过。
+ * 未覆盖任何方法时行为与基类一致；子类可覆盖 collectFiles、uploadOneFile 等。
  */
-export class DirectoryUploadFlow extends UploadFlowTemplate {
-  constructor(context: UploadContext) {
-    super(context);
-  }
-
-  /** 递归收集 localDir 下所有文件 */
-  protected async collectFiles(): Promise<LocalFile[]> {
-    return this.walkDir(this.context.localDir);
-  }
-
-  /**
-   * 逐个文件上传：先 head 判断是否存在，存在则跳过，否则读取本地文件并 putObject。
-   * 每个文件的结果通过 reportFileResult 交给 Reporter。
-   */
-  protected async uploadFiles(files: LocalFile[]): Promise<UploadSummary> {
-    const { bucket, storageClient, cdnBaseUrl } = this.context;
-
-    const summary: UploadSummary = {
-      total: files.length,
-      success: 0,
-      failed: 0,
-      skipped: 0,
-    };
-
-    for (const file of files) {
-      const key = this.buildKey(file.relativePath);
-
-      try {
-        const head = await storageClient.headObject(bucket, key);
-
-        // 云端已有同名对象则跳过，不比较内容
-        if (head.exists) {
-          summary.skipped += 1;
-          await this.reportFileResult({
-            localPath: file.absolutePath,
-            relativePath: file.relativePath,
-            bucket,
-            key,
-            status: 'skipped',
-            ...(cdnBaseUrl && { accessUrl: buildCdnAccessUrl(cdnBaseUrl, key) }),
-          });
-          continue;
-        }
-
-        const data = await fs.readFile(file.absolutePath);
-
-        const { buffer, contentType } = this.context.fileProcessor
-          ? await this.context.fileProcessor.process({
-              localPath: file.absolutePath,
-              relativePath: file.relativePath,
-              buffer: data,
-            })
-          : { buffer: data, contentType: undefined as string | undefined };
-
-        await storageClient.putObject({
-          bucket,
-          key,
-          body: buffer,
-          contentType,
-        });
-
-        summary.success += 1;
-        await this.reportFileResult({
-          localPath: file.absolutePath,
-          relativePath: file.relativePath,
-          bucket,
-          key,
-          status: 'success',
-          ...(cdnBaseUrl && { accessUrl: buildCdnAccessUrl(cdnBaseUrl, key) }),
-        });
-      } catch (err: any) {
-        summary.failed += 1;
-        await this.reportFileResult({
-          localPath: file.absolutePath,
-          relativePath: file.relativePath,
-          bucket,
-          key,
-          status: 'failed',
-          error: err instanceof Error ? err : new Error(String(err)),
-        });
-      }
-    }
-
-    return summary;
+export class DirectoryUploadFlow extends UploadService {
+  constructor(
+    storageClient: UploadService['storageClient'],
+    basePrefix: string,
+    cdnBaseUrl?: string,
+    reporter?: UploadService['reporter'],
+    fileProcessor?: UploadService['fileProcessor'],
+  ) {
+    super(storageClient, basePrefix, cdnBaseUrl, reporter, fileProcessor);
   }
 }
-

package/src/core/StorageClient.ts

@@ -3,11 +3,14 @@
  * 实现类负责将 putObject/headObject 翻译为具体云厂商的 API 调用。
  */
 
-/** 上传单个对象时的参数 */
+/** 上传单个对象时的参数（body 与 sourceFile 二选一：sourceFile 为本地路径时由 SDK 直接读文件上传，避免先读成 Buffer） */
 export interface PutObjectOptions {
   bucket: string;
   key: string;
-  body: Buffer | NodeJS.ReadableStream;
+  /** 文件内容，与 sourceFile 二选一 */
+  body?: Buffer | NodeJS.ReadableStream;
+  /** 本地文件路径，与 body 二选一；设置时由实现方用 SDK 的 SourceFile 等方式直接上传 */
+  sourceFile?: string;
   contentType?: string;
   headers?: Record<string, string>;
 }
@@ -21,5 +24,7 @@ export interface HeadObjectResult {
 export interface StorageClient {
   putObject(options: PutObjectOptions): Promise<void>;
   headObject(bucket: string, key: string): Promise<HeadObjectResult>;
+  /** 可选：按前缀列举 key，用于一次拉取后内存判断存在，避免每个文件都 head */
+  listObjectKeys?(bucket: string, prefix: string): Promise<string[]>;
 }
 

package/src/core/UploadService.ts

@@ -0,0 +1,239 @@
+import fs from 'fs/promises';
+import path from 'path';
+import { Reporter, UploadSummary, UploadFileResult } from './reporter';
+import { StorageClient } from './StorageClient';
+import { shouldInclude } from './filters';
+import { buildCdnAccessUrl } from './urlHelper';
+import type { FileProcessor } from './fileProcessor';
+
+/** 常见扩展名 -> Content-Type，避免图片等以错误类型上传导致损坏或无法预览 */
+const EXT_TO_MIME: Record<string, string> = {
+  png: 'image/png',
+  jpg: 'image/jpeg',
+  jpeg: 'image/jpeg',
+  gif: 'image/gif',
+  webp: 'image/webp',
+  svg: 'image/svg+xml',
+  ico: 'image/x-icon',
+  bmp: 'image/bmp',
+  tiff: 'image/tiff',
+  tif: 'image/tiff',
+  js: 'application/javascript',
+  mjs: 'application/javascript',
+  json: 'application/json',
+  css: 'text/css',
+  html: 'text/html',
+  htm: 'text/html',
+  txt: 'text/plain',
+  woff: 'font/woff',
+  woff2: 'font/woff2',
+  ttf: 'font/ttf',
+  eot: 'application/vnd.ms-fontobject',
+};
+
+function getContentType(relativePath: string): string | undefined {
+  const ext = path.extname(relativePath).slice(1).toLowerCase();
+  return ext ? EXT_TO_MIME[ext] : undefined;
+}
+
+/** 单次上传的目录参数 */
+export interface UploadDirectoryOptions {
+  bucket: string;
+  localDir: string;
+  env?: string;
+  pathPrefix?: string;
+  include?: string[];
+  exclude?: string[];
+  /** 匹配到的 key 强制上传（不跳过），如从 FILE_RE_WHITE_LIST 解析的列表 */
+  forceUploadPatterns?: string[];
+}
+
+export interface LocalFile {
+  absolutePath: string;
+  relativePath: string;
+}
+
+/**
+ * 上传流程基类：封装「收集文件 → 过滤 → 逐个上传 → 上报」。
+ * 子类可覆盖 collectFiles、uploadOneFile 等扩展行为。
+ */
+export abstract class UploadService {
+  protected constructor(
+    protected readonly storageClient: StorageClient,
+    protected readonly basePrefix: string,
+    protected readonly cdnBaseUrl?: string,
+    protected readonly reporter?: Reporter,
+    protected readonly fileProcessor?: FileProcessor,
+  ) {}
+
+  async uploadDirectory(options: UploadDirectoryOptions): Promise<UploadSummary> {
+    const files = await this.collectFiles(
+      options.localDir,
+      options.include,
+      options.exclude,
+    );
+
+    const summary: UploadSummary = {
+      total: files.length,
+      success: [],
+      failed: [],
+      skipped: [],
+    };
+
+    // 若有 listObjectKeys：一次拉取前缀下已有 key，用 Set 判断存在，避免每个文件都 head
+    const prefix = this.buildKey(this.basePrefix, options.env, options.pathPrefix, '');
+    let existingKeysSet: Set<string> | undefined;
+    if (typeof this.storageClient.listObjectKeys === 'function') {
+      existingKeysSet = new Set(await this.storageClient.listObjectKeys(options.bucket, prefix));
+    }
+
+    if (this.reporter?.onStart) {
+      await this.reporter.onStart({
+        env: options.env,
+        bucket: options.bucket,
+        basePrefix: this.basePrefix,
+        pathPrefix: options.pathPrefix,
+        localDir: options.localDir,
+        cdnBaseUrl: this.cdnBaseUrl,
+      });
+    }
+
+    for (const file of files) {
+      const result = await this.uploadOneFile(file, options, existingKeysSet);
+      summary[result.status].push(result.key);
+      const current = summary.success.length + summary.failed.length + summary.skipped.length;
+      if (this.reporter?.onProgress) {
+        await this.reporter.onProgress(current, files.length);
+      }
+    }
+
+    if (this.reporter?.onComplete) {
+      await this.reporter.onComplete(summary);
+    }
+
+    return summary;
+  }
+
+  /** 收集并过滤要上传的文件，子类可覆盖 */
+  protected async collectFiles(
+    localDir: string,
+    include?: string[],
+    exclude?: string[],
+  ): Promise<LocalFile[]> {
+    const files = await this.walkDir(localDir);
+    return files.filter((f) => shouldInclude(f.relativePath, include, exclude));
+  }
+
+  /** 上传单个文件：有 existingKeysSet 时用 Set 判断存在，否则 head；存在则跳过，否则 put；子类可覆盖 */
+  protected async uploadOneFile(
+    file: LocalFile,
+    options: UploadDirectoryOptions,
+    existingKeysSet?: Set<string>,
+  ): Promise<{ status: 'success' | 'skipped' | 'failed'; key: string }> {
+    const key = this.buildKey(
+      this.basePrefix,
+      options.env,
+      options.pathPrefix,
+      file.relativePath,
+    );
+
+    try {
+      const forceUpload = options.forceUploadPatterns?.some((p) => key.includes(p));
+      if (existingKeysSet !== undefined) {
+        if (!forceUpload && existingKeysSet.has(key)) {
+          await this.reportFile(file, options.bucket, key, 'skipped');
+          return { status: 'skipped', key };
+        }
+      } else {
+        const head = await this.storageClient.headObject(options.bucket, key);
+        if (head.exists) {
+          await this.reportFile(file, options.bucket, key, 'skipped');
+          return { status: 'skipped', key };
+        }
+      }
+
+      if (this.fileProcessor) {
+        const data = await fs.readFile(file.absolutePath);
+        const { buffer, contentType } = await this.fileProcessor.process({
+          localPath: file.absolutePath,
+          relativePath: file.relativePath,
+          buffer: data,
+        });
+        await this.storageClient.putObject({
+          bucket: options.bucket,
+          key,
+          body: buffer,
+          contentType,
+        });
+      } else {
+        await this.storageClient.putObject({
+          bucket: options.bucket,
+          key,
+          sourceFile: file.absolutePath,
+          contentType: getContentType(file.relativePath),
+        });
+      }
+
+      await this.reportFile(file, options.bucket, key, 'success');
+      return { status: 'success', key };
+    } catch (err: any) {
+      const error = err instanceof Error ? err : new Error(String(err));
+      await this.reportFile(file, options.bucket, key, 'failed', error);
+      return { status: 'failed', key };
+    }
+  }
+
+  protected buildKey(
+    basePrefix: string,
+    env?: string,
+    pathPrefix?: string,
+    relativePath?: string,
+  ): string {
+    const parts: string[] = [];
+    if (basePrefix) parts.push(basePrefix);
+    if (env) parts.push(env);
+    if (pathPrefix) parts.push(pathPrefix);
+    parts.push((relativePath ?? '').replace(/\\/g, '/'));
+    return parts.join('/');
+  }
+
+  protected async walkDir(rootDir: string, currentDir = ''): Promise<LocalFile[]> {
+    const dirPath = path.join(rootDir, currentDir);
+    const entries = await fs.readdir(dirPath, { withFileTypes: true });
+    const files: LocalFile[] = [];
+
+    for (const entry of entries) {
+      const rel = path.join(currentDir, entry.name);
+      const abs = path.join(rootDir, rel);
+      if (entry.isDirectory()) {
+        files.push(...(await this.walkDir(rootDir, rel)));
+      } else if (entry.isFile()) {
+        files.push({
+          absolutePath: abs,
+          relativePath: rel.replace(/\\/g, '/'),
+        });
+      }
+    }
+    return files;
+  }
+
+  protected async reportFile(
+    file: LocalFile,
+    bucket: string,
+    key: string,
+    status: UploadFileResult['status'],
+    error?: Error,
+  ): Promise<void> {
+    if (!this.reporter?.onFileResult) return;
+    const result: UploadFileResult = {
+      localPath: file.absolutePath,
+      relativePath: file.relativePath,
+      bucket,
+      key,
+      status,
+      ...(this.cdnBaseUrl && { accessUrl: buildCdnAccessUrl(this.cdnBaseUrl, key) }),
+      ...(error && { error }),
+    };
+    await this.reporter.onFileResult(result);
+  }
+}

package/src/core/reporter.ts

@@ -3,6 +3,8 @@
  * 默认实现为控制台输出，可扩展为企业微信、钉钉等通知。
  */
 
+import { SingleBar, Presets } from 'cli-progress';
+
 export type UploadStatus = 'success' | 'failed' | 'skipped';
 
 /** 单个文件的上传结果，供 Reporter 使用 */
@@ -20,12 +22,12 @@ export interface UploadFileResult {
 /** 整次上传的汇总统计 */
 export interface UploadSummary {
   total: number;
-  success: number;
-  failed: number;
-  skipped: number;
+  success: string[];
+  failed: string[];
+  skipped: string[];
 }
 
-/** Reporter 接口：上传开始、每个文件结果、全部完成时回调，便于接入不同通知方式 */
+/** Reporter 接口：上传开始、进度、每个文件结果、全部完成时回调 */
 export interface Reporter {
   onStart?(context: {
     env?: string;
@@ -36,6 +38,9 @@ export interface Reporter {
     cdnBaseUrl?: string;
   }): void | Promise<void>;
 
+  /** 每完成一个文件后调用，便于显示进度（当前数、总数） */
+  onProgress?(current: number, total: number): void | Promise<void>;
+
   onFileResult?(result: UploadFileResult): void | Promise<void>;
 
   onComplete?(summary: UploadSummary): void | Promise<void>;
@@ -43,6 +48,8 @@ export interface Reporter {
 
 /** 默认实现：将上传进度与结果输出到控制台 */
 export class ConsoleReporter implements Reporter {
+  private progressBar: SingleBar | null = null;
+
   onStart(context: {
     env?: string;
     bucket: string;
@@ -51,38 +58,41 @@ export class ConsoleReporter implements Reporter {
     localDir: string;
     cdnBaseUrl?: string;
   }): void {
-    const { env, bucket, basePrefix, pathPrefix, localDir, cdnBaseUrl } = context;
-    // console.log('开始上传目录:');
-    // console.log(`  环境: ${env ?? '未指定'}`);
-    // console.log(`  Bucket: ${bucket}`);
-    // console.log(`  Base 前缀: ${basePrefix || '(空)'}`);
-    // console.log(`  业务前缀(pathPrefix): ${pathPrefix || '(空)'}`);
-    // console.log(`  本地目录: ${localDir}`);
+    const { cdnBaseUrl } = context;
     if (cdnBaseUrl) {
       console.log(`  CDN 根地址: ${cdnBaseUrl}`);
     }
   }
 
-  onFileResult(result: UploadFileResult): void {
-    const { status, localPath, key, error, accessUrl } = result;
-    if (status === 'success') {
-      console.log(`[SUCCESS] ${localPath} -> ${key}`);
-      if (accessUrl) console.log(`  访问地址: ${accessUrl}`);
-    } else if (status === 'skipped') {
-      console.log(`[SKIPPED] ${localPath} -> ${key} (云端同名已存在)`);
-      if (accessUrl) console.log(`  访问地址: ${accessUrl}`);
-    } else {
-      console.error(`[FAILED]  ${localPath} -> ${key}`);
-      if (error) console.error(`  Error: ${error.message}`);
+  onProgress(current: number, total: number): void {
+    if (this.progressBar === null) {
+      this.progressBar = new SingleBar(
+        {
+          format: '  上传进度 |{bar}| {percentage}% | {value}/{total} 文件',
+          barCompleteChar: '\u2588',
+          barIncompleteChar: '\u2591',
+          hideCursor: true,
+        },
+        Presets.shades_classic,
+      );
+      this.progressBar.start(total, 0);
+    }
+    this.progressBar.update(current);
+    if (current >= total) {
+      this.progressBar.stop();
+      this.progressBar = null;
     }
   }
 
+  onFileResult(_result: UploadFileResult): void {}
+
   onComplete(summary: UploadSummary): void {
     console.log('上传完成:');
     console.log(`  总数: ${summary.total}`);
-    console.log(`  成功: ${summary.success}`);
-    console.log(`  失败: ${summary.failed}`);
-    console.log(`  跳过: ${summary.skipped}`);
+    console.log(`\n  跳过: ${summary.skipped.length}`);
+    console.log(`  成功:\n ${summary.success.join('\n')}`);
+    console.log(`\n\n\n  失败:\n ${summary.failed.join('\n')}`);
+    
   }
 }
 

package/src/index.ts

@@ -1,63 +1,32 @@
 /**
- * 包入口：显式导出所有对外 API 与类型，便于调用方按需引用。
+ * 包入口：仅暴露上传方法及参数、返回值类型，内部实现固定。
  */
 
-export { buildCdnAccessUrl } from './core/urlHelper';
-
-export {
-  type StorageClient,
-  type PutObjectOptions,
-  type HeadObjectResult,
-} from './core/StorageClient';
-
-export {
-  type Reporter,
-  type UploadFileResult,
-  type UploadSummary,
-  type UploadStatus,
-  ConsoleReporter,
-} from './core/reporter';
-
-export {
-  type FileProcessor,
-  type ProcessFileInput,
-  type ProcessFileResult,
-  NoOpFileProcessor,
-} from './core/fileProcessor';
-
-export {
-  UploadFlowTemplate,
-  type UploadContext,
-  type LocalFile,
-} from './core/UploadFlowTemplate';
-
-export {
-  DirectoryUploadFlow,
-} from './core/DirectoryUploadFlow';
-
-export {
-  createStorageClient,
-} from './core/StorageFactory';
-
-export {
-  OssService,
-  type UploadDirectoryInput,
-} from './middleware/OssService';
-
-export {
-  type HuaweiObsConfig,
-  type ProviderConfig,
-  type ProviderType,
-  type EnvConfigResolver,
-  type ResolvedProviderConfig,
-  type ProviderConfigResolver,
-} from './config/types';
-
-export {
-  EnvConfigResolverImpl,
-} from './config/EnvConfigResolverImpl';
-
-export {
-  ProviderConfigResolverImpl,
-} from './config/ProviderConfigResolverImpl';
+import { uploadDirectory } from './middleware/OssService';
+import type { UploadSummary } from './core/reporter';
+
+/** 上传参数 */
+export type UploadOptions = {
+  /** 本地目录路径 */
+  localDir: string;
+  /** Bucket 名称 */
+  bucket: string;
+  /** 环境标识，如 dev / test / prod */
+  env?: string;
+  /** 业务路径前缀 */
+  pathPrefix?: string;
+  /** 包含的文件模式 */
+  include?: string[];
+  /** 排除的文件模式 */
+  exclude?: string[];
+};
+
+/** 上传结果：总数、成功/失败/跳过的 key 列表 */
+export type UploadResult = UploadSummary;
 
+/**
+ * 上传本地目录到 OSS（华为 OBS）。配置从环境变量/配置文件读取，进度输出到控制台。
+ */
+export async function upload(options: UploadOptions): Promise<UploadResult> {
+  return uploadDirectory(options);
+}

package/src/middleware/OssService.ts

@@ -1,54 +1,66 @@
-import { Reporter, UploadSummary } from '../core/reporter';
-import { ProviderConfigResolver } from '../config/types';
+import { Reporter, UploadSummary, ConsoleReporter } from '../core/reporter';
 import { createStorageClient } from '../core/StorageFactory';
-import { DirectoryUploadFlow } from '../core/DirectoryUploadFlow';
 import type { FileProcessor } from '../core/fileProcessor';
+import { ProviderConfigResolverImpl } from '../config/ProviderConfigResolverImpl';
+import { type UploadDirectoryOptions as BaseOptions } from '../core/UploadService';
+import { DirectoryUploadFlow } from '../core/DirectoryUploadFlow';
 
-/** 上传目录的入参：环境、bucket、本地目录、路径前缀、过滤规则、Reporter、上传前文件处理器 */
-export interface UploadDirectoryInput {
-  env?: string;
-  bucket: string;
-  localDir: string;
-  pathPrefix?: string;
-  include?: string[];
-  exclude?: string[];
+// ============ 对外类型与门面 ============
+
+/** 上传目录入参：含可选 reporter / fileProcessor，不传则用默认控制台上报 */
+export interface UploadDirectoryOptions extends BaseOptions {
   reporter?: Reporter;
-  /** 上传前对文件内容的处理器（如图片压缩），未配置则直接上传原内容 */
   fileProcessor?: FileProcessor;
 }
 
 /**
- * 中间件：对主流程暴露统一 API（Facade）。
- * 内部只依赖 ProviderConfigResolver 得到当前 Provider 配置，再创建 StorageClient 与流程并执行。
- * 切换云厂商时只需改 ProviderConfigResolver 的【一个实现文件】。
+ * 一行调用即可上传：内部使用默认配置解析器 + 控制台 Reporter。
+ */
+export async function uploadDirectory(
+  options: UploadDirectoryOptions,
+): Promise<UploadSummary> {
+  return new OssService().uploadDirectory(options);
+}
+
+// ============ 具体服务类：默认装配 ============
+
+/**
+ * OSS 上传服务：解析配置并委托 DirectoryUploadFlow（继承 UploadService）执行上传。
  */
 export class OssService {
-  constructor(private readonly providerConfigResolver: ProviderConfigResolver) {}
-
-  /** 上传指定本地目录到 OSS，key 规则为 basePrefix/env/pathPrefix/相对路径，同名则跳过 */
-  async uploadDirectory(input: UploadDirectoryInput): Promise<UploadSummary> {
-    const { env, bucket= 'hr-uat', localDir, pathPrefix, include, exclude, reporter, fileProcessor } = input;
-
-    const resolved = this.providerConfigResolver.resolve();
-    const { providerConfig, basePrefix, cdnBaseUrl } = resolved;
-
-    const client = createStorageClient(providerConfig);
-
-    const flow = new DirectoryUploadFlow({
-      env,
-      bucket,
-      localDir,
-      pathPrefix,
-      include,
-      exclude,
-      basePrefix,
-      storageClient: client,
+  private readonly resolver = new ProviderConfigResolverImpl();
+
+  async uploadDirectory(options: UploadDirectoryOptions): Promise<UploadSummary> {
+    const resolved = this.resolver.resolve();
+    const client = createStorageClient(resolved.providerConfig);
+    const reporter = options.reporter ?? new ConsoleReporter();
+
+    const flow = new DirectoryUploadFlow(
+      client,
+      resolved.basePrefix,
+      resolved.cdnBaseUrl,
       reporter,
-      cdnBaseUrl,
-      fileProcessor,
-    });
+      options.fileProcessor,
+    );
 
-    return flow.execute();
+    return flow.uploadDirectory({
+      bucket: options.bucket,
+      localDir: options.localDir,
+      env: options.env,
+      pathPrefix: options.pathPrefix,
+      include: options.include,
+      exclude: options.exclude,
+      forceUploadPatterns: getForceUploadPatterns(),
+    });
   }
 }
 
+function getForceUploadPatterns(): string[] {
+  try {
+    const raw = process.env.FILE_RE_WHITE_LIST;
+    if (!raw) return [];
+    return JSON.parse(raw);
+  } catch {
+    return [];
+  }
+}