@dd-code/oss-uploader 0.1.1 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.cjs +1 -1
- package/dist/core/StorageClient.d.ts +2 -0
- package/dist/core/StorageClient.d.ts.map +1 -1
- package/dist/core/UploadService.d.ts +4 -2
- package/dist/core/UploadService.d.ts.map +1 -1
- package/dist/core/reporter.d.ts.map +1 -1
- package/dist/index.cjs +1 -1
- package/dist/middleware/OssService.d.ts.map +1 -1
- package/dist/providers/huawei/HuaweiObsClient.d.ts +5 -0
- package/dist/providers/huawei/HuaweiObsClient.d.ts.map +1 -1
- package/package.json +1 -1
- package/src/core/StorageClient.ts +2 -0
- package/src/core/UploadService.ts +24 -6
- package/src/core/reporter.ts +2 -1
- package/src/middleware/OssService.ts +11 -0
- package/src/providers/huawei/HuaweiObsClient.ts +31 -0
package/dist/cli.cjs
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
"use strict";var e=require("cli-progress"),t=require("fs/promises"),s=require("path"),r=require("micromatch"),o=require("js-yaml");(async()=>{const[{Command:e},{uploadDirectory:t}]=await Promise.all([import("commander"),Promise.resolve().then(function(){return
|
|
2
|
+
"use strict";var e=require("cli-progress"),t=require("fs/promises"),s=require("path"),r=require("micromatch"),o=require("js-yaml");(async()=>{const[{Command:e},{uploadDirectory:t}]=await Promise.all([import("commander"),Promise.resolve().then(function(){return w})]),s=new e;s.name("oss-upload").description("上传本地目录到 OSS(当前为华为 OBS)").argument("<localDir>","本地目录").requiredOption("-b, --bucket <bucket>","Bucket 名称","hr-uat").option("-e, --env <env>","环境标识,如 dev/test/prod").option("--path-prefix <pathPrefix>","业务路径前缀,将拼在 base/env 后面").option("--include <pattern...>","包含的文件模式(可多次)").option("--exclude <pattern...>","排除的文件模式(可多次)").action(async(e,s)=>{const{loadObsCredentialsFromUrlIfNeeded:r}=await Promise.resolve().then(function(){return y});await r();try{(await t({env:s.env,bucket:s.bucket,localDir:e,pathPrefix:s.pathPrefix,include:s.include,exclude:s.exclude})).failed.length>0&&(process.exitCode=1)}catch(e){console.error("上传过程中发生错误:",e?.message||e),process.exitCode=1}}),s.parse(process.argv)})();class i{constructor(){this.progressBar=null}onStart(e){const{cdnBaseUrl:t}=e;t&&console.log(` CDN 根地址: ${t}`)}onProgress(t,s){null===this.progressBar&&(this.progressBar=new e.SingleBar({format:" 上传进度 |{bar}| {percentage}% | {value}/{total} 文件",barCompleteChar:"█",barIncompleteChar:"░",hideCursor:!0},e.Presets.shades_classic),this.progressBar.start(s,0)),this.progressBar.update(t),t>=s&&(this.progressBar.stop(),this.progressBar=null)}onFileResult(e){}onComplete(e){console.log("上传完成:"),console.log(` 总数: ${e.total}`),console.log(`\n 跳过: ${e.skipped.length}`),console.log(` 成功:\n ${e.success.join("\n")}`),console.log(`\n\n\n 失败:\n ${e.failed.join("\n")}`)}}const n=require("esdk-obs-nodejs");class a{constructor(e){this.config=e,this.client=new n({access_key_id:e.accessKey,secret_access_key:e.secretKey,server:e.endpoint}),this.whiteList=JSON.parse('["/index.html"]')}async listObjectKeys(e,t){const s=[];let r;do{const o={Bucket:e,MaxKeys:1e3,Prefix:t||void 0};r&&(o.Marker=r);const i=await this.client.listObjects(o);if(i.CommonMsg.Status>300)throw new Error(`OBS listObjects 失败: ${i.CommonMsg.Status} ${i.CommonMsg.Code} ${i.CommonMsg.Message}`);const n=i.InterfaceResult?.Contents??[];for(const e of n)e.Key&&s.push(e.Key);r="true"===i.InterfaceResult?.IsTruncated?i.InterfaceResult?.NextMarker:void 0}while(r);return s}async headObject(e,t){try{if(this.whiteList.some(e=>t.includes(e)))return{exists:!1};const s=await this.client.getObjectMetadata({Bucket:e,Key:t});if(s.CommonMsg.Status<=300)return{exists:!0};if(404===s.CommonMsg.Status||"NoSuchKey"===s.CommonMsg.Code||"NotFound"===s.CommonMsg.Code)return{exists:!1};throw new Error(`OBS getObjectMetadata 失败: ${s.CommonMsg.Status} ${s.CommonMsg.Code} ${s.CommonMsg.Message}`)}catch(e){const t=e&&"string"==typeof e.message?e.message:String(e),s=e?.Code??e?.code;if("NoSuchKey"===s||"NotFound"===s||t.includes("404")||t.includes("NoSuchKey"))return{exists:!1};throw e}}async putObject(e){const{bucket:t,key:s,contentType:r}=e,o={Bucket:t,Key:s,ContentType:r};if(e.sourceFile)o.SourceFile=e.sourceFile;else{if(!e.body)throw new Error("putObject 需要 body 或 sourceFile");o.Body=e.body}const i=await this.client.putObject(o);if(i.CommonMsg.Status>300){const{Status:e,Code:t,Message:s,RequestId:r}=i.CommonMsg,o=[e,t,s,r].filter(Boolean).join(" ");throw new Error(`OBS putObject 失败 (${o||"无详情"}). 请检查: 1) 桶名是否正确且已创建 2) OBS_ENDPOINT 是否与该桶所在区域一致,如 https://obs.xx-xx-1.myhuaweicloud.com`)}}}class c{resolveHuaweiConfig(e){const t=process.env.OBS_ACCESS_KEY,s=process.env.OBS_SECRET_KEY;if(!t||!s)throw new Error("缺少华为 OBS 配置: OBS_ENDPOINT / OBS_ACCESS_KEY / OBS_SECRET_KEY");return{endpoint:"https://obs.cn-east-3.myhuaweicloud.com",accessKey:t,secretKey:s,region:"cn-east-3",basePrefix:"jhr-static",cdnBaseUrl:"https://hruat-cos.jtexpress.com.cn"}}}class l{constructor(){this.huaweiResolver=new c}resolve(){{const e=this.huaweiResolver.resolveHuaweiConfig();return{providerConfig:{type:"huawei",options:e},basePrefix:e.basePrefix,cdnBaseUrl:e.cdnBaseUrl}}}}function u(e,t){const s=e.replace(/\/+$/,""),r=t.replace(/^\/+/,"");return r?`${s}/${r}`:s}const p={png:"image/png",jpg:"image/jpeg",jpeg:"image/jpeg",gif:"image/gif",webp:"image/webp",svg:"image/svg+xml",ico:"image/x-icon",bmp:"image/bmp",tiff:"image/tiff",tif:"image/tiff",js:"application/javascript",mjs:"application/javascript",json:"application/json",css:"text/css",html:"text/html",htm:"text/html",txt:"text/plain",woff:"font/woff",woff2:"font/woff2",ttf:"font/ttf",eot:"application/vnd.ms-fontobject"};function h(e){const t=s.extname(e).slice(1).toLowerCase();return t?p[t]:void 0}class f{constructor(e,t,s,r,o){this.storageClient=e,this.basePrefix=t,this.cdnBaseUrl=s,this.reporter=r,this.fileProcessor=o}async uploadDirectory(e){const t=await this.collectFiles(e.localDir,e.include,e.exclude),s={total:t.length,success:[],failed:[],skipped:[]},r=this.buildKey(this.basePrefix,e.env,e.pathPrefix,"");let o;"function"==typeof this.storageClient.listObjectKeys&&(o=new Set(await this.storageClient.listObjectKeys(e.bucket,r))),this.reporter?.onStart&&await this.reporter.onStart({env:e.env,bucket:e.bucket,basePrefix:this.basePrefix,pathPrefix:e.pathPrefix,localDir:e.localDir,cdnBaseUrl:this.cdnBaseUrl});for(const r of t){const i=await this.uploadOneFile(r,e,o);s[i.status].push(i.key);const n=s.success.length+s.failed.length+s.skipped.length;this.reporter?.onProgress&&await this.reporter.onProgress(n,t.length)}return this.reporter?.onComplete&&await this.reporter.onComplete(s),s}async collectFiles(e,t,s){return(await this.walkDir(e)).filter(e=>function(e,t,s){let o=!0;return t&&t.length>0&&(o=r.isMatch(e,t)),!(!o||s&&s.length>0&&r.isMatch(e,s))}(e.relativePath,t,s))}async uploadOneFile(e,s,r){const o=this.buildKey(this.basePrefix,s.env,s.pathPrefix,e.relativePath);try{const i=s.forceUploadPatterns?.some(e=>o.includes(e));if(void 0!==r){if(!i&&r.has(o))return await this.reportFile(e,s.bucket,o,"skipped"),{status:"skipped",key:o}}else{if((await this.storageClient.headObject(s.bucket,o)).exists)return await this.reportFile(e,s.bucket,o,"skipped"),{status:"skipped",key:o}}if(this.fileProcessor){const r=await t.readFile(e.absolutePath),{buffer:i,contentType:n}=await this.fileProcessor.process({localPath:e.absolutePath,relativePath:e.relativePath,buffer:r});await this.storageClient.putObject({bucket:s.bucket,key:o,body:i,contentType:n})}else await this.storageClient.putObject({bucket:s.bucket,key:o,sourceFile:e.absolutePath,contentType:h(e.relativePath)});return await this.reportFile(e,s.bucket,o,"success"),{status:"success",key:o}}catch(t){const r=t instanceof Error?t:new Error(String(t));return await this.reportFile(e,s.bucket,o,"failed",r),{status:"failed",key:o}}}buildKey(e,t,s,r){const o=[];return e&&o.push(e),t&&o.push(t),s&&o.push(s),o.push((r??"").replace(/\\/g,"/")),o.join("/")}async walkDir(e,r=""){const o=s.join(e,r),i=await t.readdir(o,{withFileTypes:!0}),n=[];for(const t of i){const o=s.join(r,t.name),i=s.join(e,o);t.isDirectory()?n.push(...await this.walkDir(e,o)):t.isFile()&&n.push({absolutePath:i,relativePath:o.replace(/\\/g,"/")})}return n}async reportFile(e,t,s,r,o){if(!this.reporter?.onFileResult)return;const i={localPath:e.absolutePath,relativePath:e.relativePath,bucket:t,key:s,status:r,...this.cdnBaseUrl&&{accessUrl:u(this.cdnBaseUrl,s)},...o&&{error:o}};await this.reporter.onFileResult(i)}}class d extends f{constructor(e,t,s,r,o){super(e,t,s,r,o)}}class g{constructor(){this.resolver=new l}async uploadDirectory(e){const t=this.resolver.resolve(),s=function(e){if("huawei"===e.type)return new a(e.options);throw new Error(`Unsupported provider type: ${e.type}`)}(t.providerConfig),r=e.reporter??new i;return new d(s,t.basePrefix,t.cdnBaseUrl,r,e.fileProcessor).uploadDirectory({bucket:e.bucket,localDir:e.localDir,env:e.env,pathPrefix:e.pathPrefix,include:e.include,exclude:e.exclude,forceUploadPatterns:m()})}}function m(){try{const e='["/index.html"]';return JSON.parse(e)}catch{return[]}}var w=Object.freeze({__proto__:null,OssService:g,uploadDirectory:async function(e){return(new g).uploadDirectory(e)}});const b="OBS_CREDENTIALS_URL";var y=Object.freeze({__proto__:null,loadObsCredentialsFromUrlIfNeeded:async function(){const e="https://hr-uat.jtexpress.com.cn/hrpt/ast/static/obs-config.yaml",t=process.env.OBS_ACCESS_KEY,s=process.env.OBS_SECRET_KEY;if(t&&s)return;const r=await fetch(e);if(!r.ok)throw new Error(`拉取 OBS 凭证失败 (${r.status}): ${e}`);const i=await r.text(),n=o.load(i);if(!n||"object"!=typeof n)throw new Error(`OBS 凭证 YAML 解析结果无效,请检查 ${b} 返回内容`);const a=n.OBS_ACCESS_KEY,c=n.OBS_SECRET_KEY;if("string"==typeof a&&(process.env.OBS_ACCESS_KEY=a),"string"==typeof c&&(process.env.OBS_SECRET_KEY=c),!process.env.OBS_ACCESS_KEY||!process.env.OBS_SECRET_KEY)throw new Error(`OBS 凭证 YAML 中未找到 OBS_ACCESS_KEY 或 OBS_SECRET_KEY,请检查 ${b} 返回内容`)}});
|
|
@@ -21,5 +21,7 @@ export interface HeadObjectResult {
|
|
|
21
21
|
export interface StorageClient {
|
|
22
22
|
putObject(options: PutObjectOptions): Promise<void>;
|
|
23
23
|
headObject(bucket: string, key: string): Promise<HeadObjectResult>;
|
|
24
|
+
/** 可选:按前缀列举 key,用于一次拉取后内存判断存在,避免每个文件都 head */
|
|
25
|
+
listObjectKeys?(bucket: string, prefix: string): Promise<string[]>;
|
|
24
26
|
}
|
|
25
27
|
//# sourceMappingURL=StorageClient.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"StorageClient.d.ts","sourceRoot":"","sources":["../../src/core/StorageClient.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,oFAAoF;AACpF,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,4BAA4B;IAC5B,IAAI,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC,cAAc,CAAC;IACtC,0DAA0D;IAC1D,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAClC;AAED,iDAAiD;AACjD,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,OAAO,CAAC;CACjB;AAED,qDAAqD;AACrD,MAAM,WAAW,aAAa;IAC5B,SAAS,CAAC,OAAO,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACpD,UAAU,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAAC;CACpE"}
|
|
1
|
+
{"version":3,"file":"StorageClient.d.ts","sourceRoot":"","sources":["../../src/core/StorageClient.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,oFAAoF;AACpF,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,4BAA4B;IAC5B,IAAI,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC,cAAc,CAAC;IACtC,0DAA0D;IAC1D,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAClC;AAED,iDAAiD;AACjD,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,OAAO,CAAC;CACjB;AAED,qDAAqD;AACrD,MAAM,WAAW,aAAa;IAC5B,SAAS,CAAC,OAAO,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACpD,UAAU,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAAC;IACnE,8CAA8C;IAC9C,cAAc,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;CACpE"}
|
|
@@ -9,6 +9,8 @@ export interface UploadDirectoryOptions {
|
|
|
9
9
|
pathPrefix?: string;
|
|
10
10
|
include?: string[];
|
|
11
11
|
exclude?: string[];
|
|
12
|
+
/** 匹配到的 key 强制上传(不跳过),如从 FILE_RE_WHITE_LIST 解析的列表 */
|
|
13
|
+
forceUploadPatterns?: string[];
|
|
12
14
|
}
|
|
13
15
|
export interface LocalFile {
|
|
14
16
|
absolutePath: string;
|
|
@@ -28,8 +30,8 @@ export declare abstract class UploadService {
|
|
|
28
30
|
uploadDirectory(options: UploadDirectoryOptions): Promise<UploadSummary>;
|
|
29
31
|
/** 收集并过滤要上传的文件,子类可覆盖 */
|
|
30
32
|
protected collectFiles(localDir: string, include?: string[], exclude?: string[]): Promise<LocalFile[]>;
|
|
31
|
-
/**
|
|
32
|
-
protected uploadOneFile(file: LocalFile, options: UploadDirectoryOptions): Promise<{
|
|
33
|
+
/** 上传单个文件:有 existingKeysSet 时用 Set 判断存在,否则 head;存在则跳过,否则 put;子类可覆盖 */
|
|
34
|
+
protected uploadOneFile(file: LocalFile, options: UploadDirectoryOptions, existingKeysSet?: Set<string>): Promise<{
|
|
33
35
|
status: 'success' | 'skipped' | 'failed';
|
|
34
36
|
key: string;
|
|
35
37
|
}>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"UploadService.d.ts","sourceRoot":"","sources":["../../src/core/UploadService.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AACvE,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAGhD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAgCrD,gBAAgB;AAChB,MAAM,WAAW,sBAAsB;IACrC,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;
|
|
1
|
+
{"version":3,"file":"UploadService.d.ts","sourceRoot":"","sources":["../../src/core/UploadService.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AACvE,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAGhD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAgCrD,gBAAgB;AAChB,MAAM,WAAW,sBAAsB;IACrC,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,qDAAqD;IACrD,mBAAmB,CAAC,EAAE,MAAM,EAAE,CAAC;CAChC;AAED,MAAM,WAAW,SAAS;IACxB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;CACtB;AAED;;;GAGG;AACH,8BAAsB,aAAa;IAE/B,SAAS,CAAC,QAAQ,CAAC,aAAa,EAAE,aAAa;IAC/C,SAAS,CAAC,QAAQ,CAAC,UAAU,EAAE,MAAM;IACrC,SAAS,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM;IACtC,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,QAAQ;IACtC,SAAS,CAAC,QAAQ,CAAC,aAAa,CAAC,EAAE,aAAa;IALlD,SAAS,aACY,aAAa,EAAE,aAAa,EAC5B,UAAU,EAAE,MAAM,EAClB,UAAU,CAAC,EAAE,MAAM,YAAA,EACnB,QAAQ,CAAC,EAAE,QAAQ,YAAA,EACnB,aAAa,CAAC,EAAE,aAAa,YAAA;IAG5C,eAAe,CAAC,OAAO,EAAE,sBAAsB,GAAG,OAAO,CAAC,aAAa,CAAC;IAgD9E,wBAAwB;cACR,YAAY,CAC1B,QAAQ,EAAE,MAAM,EAChB,OAAO,CAAC,EAAE,MAAM,EAAE,EAClB,OAAO,CAAC,EAAE,MAAM,EAAE,GACjB,OAAO,CAAC,SAAS,EAAE,CAAC;IAKvB,sEAAsE;cACtD,aAAa,CAC3B,IAAI,EAAE,SAAS,EACf,OAAO,EAAE,sBAAsB,EAC/B,eAAe,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,GAC5B,OAAO,CAAC;QAAE,MAAM,EAAE,SAAS,GAAG,SAAS,GAAG,QAAQ,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,CAAC;IAsDrE,SAAS,CAAC,QAAQ,CAChB,UAAU,EAAE,MAAM,EAClB,GAAG,CAAC,EAAE,MAAM,EACZ,UAAU,CAAC,EAAE,MAAM,EACnB,YAAY,CAAC,EAAE,MAAM,GACpB,MAAM;cASO,OAAO,CAAC,OAAO,EAAE,MAAM,EAAE,UAAU,SAAK,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC;cAoB/D,UAAU,CACxB,IAAI,EAAE,SAAS,EACf,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,EACX,MAAM,EAAE,gBAAgB,CAAC,QAAQ,CAAC,EAClC,KAAK,CAAC,EAAE,KAAK,GACZ,OAAO,CAAC,IAAI,CAAC;CAajB"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"reporter.d.ts","sourceRoot":"","sources":["../../src/core/reporter.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAIH,MAAM,MAAM,YAAY,GAAG,SAAS,GAAG,QAAQ,GAAG,SAAS,CAAC;AAE5D,8BAA8B;AAC9B,MAAM,WAAW,gBAAgB;IAC/B,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,YAAY,CAAC;IACrB,KAAK,CAAC,EAAE,KAAK,CAAC;IACd,8CAA8C;IAC9C,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,gBAAgB;AAChB,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,OAAO,EAAE,MAAM,EAAE,CAAC;CACnB;AAED,yCAAyC;AACzC,MAAM,WAAW,QAAQ;IACvB,OAAO,CAAC,CAAC,OAAO,EAAE;QAChB,GAAG,CAAC,EAAE,MAAM,CAAC;QACb,MAAM,EAAE,MAAM,CAAC;QACf,UAAU,EAAE,MAAM,CAAC;QACnB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,QAAQ,EAAE,MAAM,CAAC;QACjB,UAAU,CAAC,EAAE,MAAM,CAAC;KACrB,GAAG,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAEzB,gCAAgC;IAChC,UAAU,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAElE,YAAY,CAAC,CAAC,MAAM,EAAE,gBAAgB,GAAG,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAE9D,UAAU,CAAC,CAAC,OAAO,EAAE,aAAa,GAAG,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC3D;AAED,0BAA0B;AAC1B,qBAAa,eAAgB,YAAW,QAAQ;IAC9C,OAAO,CAAC,WAAW,CAA0B;IAE7C,OAAO,CAAC,OAAO,EAAE;QACf,GAAG,CAAC,EAAE,MAAM,CAAC;QACb,MAAM,EAAE,MAAM,CAAC;QACf,UAAU,EAAE,MAAM,CAAC;QACnB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,QAAQ,EAAE,MAAM,CAAC;QACjB,UAAU,CAAC,EAAE,MAAM,CAAC;KACrB,GAAG,IAAI;IAOR,UAAU,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI;IAoBhD,YAAY,CAAC,OAAO,EAAE,gBAAgB,GAAG,IAAI;IAE7C,UAAU,CAAC,OAAO,EAAE,aAAa,GAAG,IAAI;
|
|
1
|
+
{"version":3,"file":"reporter.d.ts","sourceRoot":"","sources":["../../src/core/reporter.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAIH,MAAM,MAAM,YAAY,GAAG,SAAS,GAAG,QAAQ,GAAG,SAAS,CAAC;AAE5D,8BAA8B;AAC9B,MAAM,WAAW,gBAAgB;IAC/B,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,YAAY,CAAC;IACrB,KAAK,CAAC,EAAE,KAAK,CAAC;IACd,8CAA8C;IAC9C,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,gBAAgB;AAChB,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,OAAO,EAAE,MAAM,EAAE,CAAC;CACnB;AAED,yCAAyC;AACzC,MAAM,WAAW,QAAQ;IACvB,OAAO,CAAC,CAAC,OAAO,EAAE;QAChB,GAAG,CAAC,EAAE,MAAM,CAAC;QACb,MAAM,EAAE,MAAM,CAAC;QACf,UAAU,EAAE,MAAM,CAAC;QACnB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,QAAQ,EAAE,MAAM,CAAC;QACjB,UAAU,CAAC,EAAE,MAAM,CAAC;KACrB,GAAG,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAEzB,gCAAgC;IAChC,UAAU,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAElE,YAAY,CAAC,CAAC,MAAM,EAAE,gBAAgB,GAAG,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAE9D,UAAU,CAAC,CAAC,OAAO,EAAE,aAAa,GAAG,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC3D;AAED,0BAA0B;AAC1B,qBAAa,eAAgB,YAAW,QAAQ;IAC9C,OAAO,CAAC,WAAW,CAA0B;IAE7C,OAAO,CAAC,OAAO,EAAE;QACf,GAAG,CAAC,EAAE,MAAM,CAAC;QACb,MAAM,EAAE,MAAM,CAAC;QACf,UAAU,EAAE,MAAM,CAAC;QACnB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,QAAQ,EAAE,MAAM,CAAC;QACjB,UAAU,CAAC,EAAE,MAAM,CAAC;KACrB,GAAG,IAAI;IAOR,UAAU,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI;IAoBhD,YAAY,CAAC,OAAO,EAAE,gBAAgB,GAAG,IAAI;IAE7C,UAAU,CAAC,OAAO,EAAE,aAAa,GAAG,IAAI;CAQzC"}
|
package/dist/index.cjs
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
"use strict";var e=require("cli-progress"),t=require("fs/promises"),s=require("path"),r=require("micromatch");class o{constructor(){this.progressBar=null}onStart(e){const{cdnBaseUrl:t}=e;t&&console.log(` CDN 根地址: ${t}`)}onProgress(t,s){null===this.progressBar&&(this.progressBar=new e.SingleBar({format:" 上传进度 |{bar}| {percentage}% | {value}/{total} 文件",barCompleteChar:"█",barIncompleteChar:"░",hideCursor:!0},e.Presets.shades_classic),this.progressBar.start(s,0)),this.progressBar.update(t),t>=s&&(this.progressBar.stop(),this.progressBar=null)}onFileResult(e){}onComplete(e){console.log("上传完成:"),console.log(` 总数: ${e.total}`),console.log(
|
|
1
|
+
"use strict";var e=require("cli-progress"),t=require("fs/promises"),s=require("path"),r=require("micromatch");class o{constructor(){this.progressBar=null}onStart(e){const{cdnBaseUrl:t}=e;t&&console.log(` CDN 根地址: ${t}`)}onProgress(t,s){null===this.progressBar&&(this.progressBar=new e.SingleBar({format:" 上传进度 |{bar}| {percentage}% | {value}/{total} 文件",barCompleteChar:"█",barIncompleteChar:"░",hideCursor:!0},e.Presets.shades_classic),this.progressBar.start(s,0)),this.progressBar.update(t),t>=s&&(this.progressBar.stop(),this.progressBar=null)}onFileResult(e){}onComplete(e){console.log("上传完成:"),console.log(` 总数: ${e.total}`),console.log(`\n 跳过: ${e.skipped.length}`),console.log(` 成功:\n ${e.success.join("\n")}`),console.log(`\n\n\n 失败:\n ${e.failed.join("\n")}`)}}const i=require("esdk-obs-nodejs");class a{constructor(e){this.config=e,this.client=new i({access_key_id:e.accessKey,secret_access_key:e.secretKey,server:e.endpoint}),this.whiteList=JSON.parse('["/index.html"]')}async listObjectKeys(e,t){const s=[];let r;do{const o={Bucket:e,MaxKeys:1e3,Prefix:t||void 0};r&&(o.Marker=r);const i=await this.client.listObjects(o);if(i.CommonMsg.Status>300)throw new Error(`OBS listObjects 失败: ${i.CommonMsg.Status} ${i.CommonMsg.Code} ${i.CommonMsg.Message}`);const a=i.InterfaceResult?.Contents??[];for(const e of a)e.Key&&s.push(e.Key);r="true"===i.InterfaceResult?.IsTruncated?i.InterfaceResult?.NextMarker:void 0}while(r);return s}async headObject(e,t){try{if(this.whiteList.some(e=>t.includes(e)))return{exists:!1};const s=await this.client.getObjectMetadata({Bucket:e,Key:t});if(s.CommonMsg.Status<=300)return{exists:!0};if(404===s.CommonMsg.Status||"NoSuchKey"===s.CommonMsg.Code||"NotFound"===s.CommonMsg.Code)return{exists:!1};throw new Error(`OBS getObjectMetadata 失败: ${s.CommonMsg.Status} ${s.CommonMsg.Code} ${s.CommonMsg.Message}`)}catch(e){const t=e&&"string"==typeof e.message?e.message:String(e),s=e?.Code??e?.code;if("NoSuchKey"===s||"NotFound"===s||t.includes("404")||t.includes("NoSuchKey"))return{exists:!1};throw e}}async putObject(e){const{bucket:t,key:s,contentType:r}=e,o={Bucket:t,Key:s,ContentType:r};if(e.sourceFile)o.SourceFile=e.sourceFile;else{if(!e.body)throw new Error("putObject 需要 body 或 sourceFile");o.Body=e.body}const i=await this.client.putObject(o);if(i.CommonMsg.Status>300){const{Status:e,Code:t,Message:s,RequestId:r}=i.CommonMsg,o=[e,t,s,r].filter(Boolean).join(" ");throw new Error(`OBS putObject 失败 (${o||"无详情"}). 请检查: 1) 桶名是否正确且已创建 2) OBS_ENDPOINT 是否与该桶所在区域一致,如 https://obs.xx-xx-1.myhuaweicloud.com`)}}}class n{resolveHuaweiConfig(e){const t=process.env.OBS_ACCESS_KEY,s=process.env.OBS_SECRET_KEY;if(!t||!s)throw new Error("缺少华为 OBS 配置: OBS_ENDPOINT / OBS_ACCESS_KEY / OBS_SECRET_KEY");return{endpoint:"https://obs.cn-east-3.myhuaweicloud.com",accessKey:t,secretKey:s,region:"cn-east-3",basePrefix:"jhr-static",cdnBaseUrl:"https://hruat-cos.jtexpress.com.cn"}}}class c{constructor(){this.huaweiResolver=new n}resolve(){{const e=this.huaweiResolver.resolveHuaweiConfig();return{providerConfig:{type:"huawei",options:e},basePrefix:e.basePrefix,cdnBaseUrl:e.cdnBaseUrl}}}}function l(e,t){const s=e.replace(/\/+$/,""),r=t.replace(/^\/+/,"");return r?`${s}/${r}`:s}const u={png:"image/png",jpg:"image/jpeg",jpeg:"image/jpeg",gif:"image/gif",webp:"image/webp",svg:"image/svg+xml",ico:"image/x-icon",bmp:"image/bmp",tiff:"image/tiff",tif:"image/tiff",js:"application/javascript",mjs:"application/javascript",json:"application/json",css:"text/css",html:"text/html",htm:"text/html",txt:"text/plain",woff:"font/woff",woff2:"font/woff2",ttf:"font/ttf",eot:"application/vnd.ms-fontobject"};function h(e){const t=s.extname(e).slice(1).toLowerCase();return t?u[t]:void 0}class p{constructor(e,t,s,r,o){this.storageClient=e,this.basePrefix=t,this.cdnBaseUrl=s,this.reporter=r,this.fileProcessor=o}async uploadDirectory(e){const t=await this.collectFiles(e.localDir,e.include,e.exclude),s={total:t.length,success:[],failed:[],skipped:[]},r=this.buildKey(this.basePrefix,e.env,e.pathPrefix,"");let o;"function"==typeof this.storageClient.listObjectKeys&&(o=new Set(await this.storageClient.listObjectKeys(e.bucket,r))),this.reporter?.onStart&&await this.reporter.onStart({env:e.env,bucket:e.bucket,basePrefix:this.basePrefix,pathPrefix:e.pathPrefix,localDir:e.localDir,cdnBaseUrl:this.cdnBaseUrl});for(const r of t){const i=await this.uploadOneFile(r,e,o);s[i.status].push(i.key);const a=s.success.length+s.failed.length+s.skipped.length;this.reporter?.onProgress&&await this.reporter.onProgress(a,t.length)}return this.reporter?.onComplete&&await this.reporter.onComplete(s),s}async collectFiles(e,t,s){return(await this.walkDir(e)).filter(e=>function(e,t,s){let o=!0;return t&&t.length>0&&(o=r.isMatch(e,t)),!(!o||s&&s.length>0&&r.isMatch(e,s))}(e.relativePath,t,s))}async uploadOneFile(e,s,r){const o=this.buildKey(this.basePrefix,s.env,s.pathPrefix,e.relativePath);try{const i=s.forceUploadPatterns?.some(e=>o.includes(e));if(void 0!==r){if(!i&&r.has(o))return await this.reportFile(e,s.bucket,o,"skipped"),{status:"skipped",key:o}}else{if((await this.storageClient.headObject(s.bucket,o)).exists)return await this.reportFile(e,s.bucket,o,"skipped"),{status:"skipped",key:o}}if(this.fileProcessor){const r=await t.readFile(e.absolutePath),{buffer:i,contentType:a}=await this.fileProcessor.process({localPath:e.absolutePath,relativePath:e.relativePath,buffer:r});await this.storageClient.putObject({bucket:s.bucket,key:o,body:i,contentType:a})}else await this.storageClient.putObject({bucket:s.bucket,key:o,sourceFile:e.absolutePath,contentType:h(e.relativePath)});return await this.reportFile(e,s.bucket,o,"success"),{status:"success",key:o}}catch(t){const r=t instanceof Error?t:new Error(String(t));return await this.reportFile(e,s.bucket,o,"failed",r),{status:"failed",key:o}}}buildKey(e,t,s,r){const o=[];return e&&o.push(e),t&&o.push(t),s&&o.push(s),o.push((r??"").replace(/\\/g,"/")),o.join("/")}async walkDir(e,r=""){const o=s.join(e,r),i=await t.readdir(o,{withFileTypes:!0}),a=[];for(const t of i){const o=s.join(r,t.name),i=s.join(e,o);t.isDirectory()?a.push(...await this.walkDir(e,o)):t.isFile()&&a.push({absolutePath:i,relativePath:o.replace(/\\/g,"/")})}return a}async reportFile(e,t,s,r,o){if(!this.reporter?.onFileResult)return;const i={localPath:e.absolutePath,relativePath:e.relativePath,bucket:t,key:s,status:r,...this.cdnBaseUrl&&{accessUrl:l(this.cdnBaseUrl,s)},...o&&{error:o}};await this.reporter.onFileResult(i)}}class f extends p{constructor(e,t,s,r,o){super(e,t,s,r,o)}}class d{constructor(){this.resolver=new c}async uploadDirectory(e){const t=this.resolver.resolve(),s=function(e){if("huawei"===e.type)return new a(e.options);throw new Error(`Unsupported provider type: ${e.type}`)}(t.providerConfig),r=e.reporter??new o;return new f(s,t.basePrefix,t.cdnBaseUrl,r,e.fileProcessor).uploadDirectory({bucket:e.bucket,localDir:e.localDir,env:e.env,pathPrefix:e.pathPrefix,include:e.include,exclude:e.exclude,forceUploadPatterns:g()})}}function g(){try{const e='["/index.html"]';return JSON.parse(e)}catch{return[]}}exports.upload=async function(e){return async function(e){return(new d).uploadDirectory(e)}(e)};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"OssService.d.ts","sourceRoot":"","sources":["../../src/middleware/OssService.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAmB,MAAM,kBAAkB,CAAC;AAE5E,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAE3D,OAAO,EAAE,KAAK,sBAAsB,IAAI,WAAW,EAAE,MAAM,uBAAuB,CAAC;AAKnF,sDAAsD;AACtD,MAAM,WAAW,sBAAuB,SAAQ,WAAW;IACzD,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,aAAa,CAAC,EAAE,aAAa,CAAC;CAC/B;AAED;;GAEG;AACH,wBAAsB,eAAe,CACnC,OAAO,EAAE,sBAAsB,GAC9B,OAAO,CAAC,aAAa,CAAC,CAExB;AAID;;GAEG;AACH,qBAAa,UAAU;IACrB,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAoC;IAEvD,eAAe,CAAC,OAAO,EAAE,sBAAsB,GAAG,OAAO,CAAC,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"OssService.d.ts","sourceRoot":"","sources":["../../src/middleware/OssService.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAmB,MAAM,kBAAkB,CAAC;AAE5E,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAE3D,OAAO,EAAE,KAAK,sBAAsB,IAAI,WAAW,EAAE,MAAM,uBAAuB,CAAC;AAKnF,sDAAsD;AACtD,MAAM,WAAW,sBAAuB,SAAQ,WAAW;IACzD,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,aAAa,CAAC,EAAE,aAAa,CAAC;CAC/B;AAED;;GAEG;AACH,wBAAsB,eAAe,CACnC,OAAO,EAAE,sBAAsB,GAC9B,OAAO,CAAC,aAAa,CAAC,CAExB;AAID;;GAEG;AACH,qBAAa,UAAU;IACrB,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAoC;IAEvD,eAAe,CAAC,OAAO,EAAE,sBAAsB,GAAG,OAAO,CAAC,aAAa,CAAC;CAuB/E"}
|
|
@@ -9,8 +9,13 @@ export declare class HuaweiObsClient implements StorageClient {
|
|
|
9
9
|
private readonly client;
|
|
10
10
|
private readonly whiteList;
|
|
11
11
|
constructor(config: HuaweiObsConfig);
|
|
12
|
+
/**
|
|
13
|
+
* 按前缀列举对象 key(分页拉全),用于上传前一次拉取、内存判断存在,避免每个文件都 head。
|
|
14
|
+
*/
|
|
15
|
+
listObjectKeys(bucket: string, prefix: string): Promise<string[]>;
|
|
12
16
|
/**
|
|
13
17
|
* 调用 OBS getObjectMetadata 查元数据,判断对象是否存在(同路径即跳过上传)。
|
|
18
|
+
* 当使用 listObjectKeys 时,上传流程会优先用内存 Set 判断,不再逐文件调用本方法。
|
|
14
19
|
*/
|
|
15
20
|
headObject(bucket: string, key: string): Promise<HeadObjectResult>;
|
|
16
21
|
/** 调用 OBS putObject 上传对象(支持 Body 或 SourceFile 本地路径,SourceFile 由 SDK 直接读文件避免 Buffer 问题) */
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"HuaweiObsClient.d.ts","sourceRoot":"","sources":["../../../src/providers/huawei/HuaweiObsClient.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AAC7F,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAKrD;;;GAGG;AACH,qBAAa,eAAgB,YAAW,aAAa;IAGvC,OAAO,CAAC,QAAQ,CAAC,MAAM;IAFnC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAiC;IACxD,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAW;gBACR,MAAM,EAAE,eAAe;IASpD;;OAEG;IACG,UAAU,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC;IA6BxE,0FAA0F;IACpF,SAAS,CAAC,OAAO,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC;CA0B1D"}
|
|
1
|
+
{"version":3,"file":"HuaweiObsClient.d.ts","sourceRoot":"","sources":["../../../src/providers/huawei/HuaweiObsClient.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AAC7F,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAKrD;;;GAGG;AACH,qBAAa,eAAgB,YAAW,aAAa;IAGvC,OAAO,CAAC,QAAQ,CAAC,MAAM;IAFnC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAiC;IACxD,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAW;gBACR,MAAM,EAAE,eAAe;IASpD;;OAEG;IACG,cAAc,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IA2BvE;;;OAGG;IACG,UAAU,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC;IA6BxE,0FAA0F;IACpF,SAAS,CAAC,OAAO,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC;CA0B1D"}
|
package/package.json
CHANGED
|
@@ -24,5 +24,7 @@ export interface HeadObjectResult {
|
|
|
24
24
|
export interface StorageClient {
|
|
25
25
|
putObject(options: PutObjectOptions): Promise<void>;
|
|
26
26
|
headObject(bucket: string, key: string): Promise<HeadObjectResult>;
|
|
27
|
+
/** 可选:按前缀列举 key,用于一次拉取后内存判断存在,避免每个文件都 head */
|
|
28
|
+
listObjectKeys?(bucket: string, prefix: string): Promise<string[]>;
|
|
27
29
|
}
|
|
28
30
|
|
|
@@ -44,6 +44,8 @@ export interface UploadDirectoryOptions {
|
|
|
44
44
|
pathPrefix?: string;
|
|
45
45
|
include?: string[];
|
|
46
46
|
exclude?: string[];
|
|
47
|
+
/** 匹配到的 key 强制上传(不跳过),如从 FILE_RE_WHITE_LIST 解析的列表 */
|
|
48
|
+
forceUploadPatterns?: string[];
|
|
47
49
|
}
|
|
48
50
|
|
|
49
51
|
export interface LocalFile {
|
|
@@ -78,6 +80,13 @@ export abstract class UploadService {
|
|
|
78
80
|
skipped: [],
|
|
79
81
|
};
|
|
80
82
|
|
|
83
|
+
// 若有 listObjectKeys:一次拉取前缀下已有 key,用 Set 判断存在,避免每个文件都 head
|
|
84
|
+
const prefix = this.buildKey(this.basePrefix, options.env, options.pathPrefix, '');
|
|
85
|
+
let existingKeysSet: Set<string> | undefined;
|
|
86
|
+
if (typeof this.storageClient.listObjectKeys === 'function') {
|
|
87
|
+
existingKeysSet = new Set(await this.storageClient.listObjectKeys(options.bucket, prefix));
|
|
88
|
+
}
|
|
89
|
+
|
|
81
90
|
if (this.reporter?.onStart) {
|
|
82
91
|
await this.reporter.onStart({
|
|
83
92
|
env: options.env,
|
|
@@ -90,7 +99,7 @@ export abstract class UploadService {
|
|
|
90
99
|
}
|
|
91
100
|
|
|
92
101
|
for (const file of files) {
|
|
93
|
-
const result = await this.uploadOneFile(file, options);
|
|
102
|
+
const result = await this.uploadOneFile(file, options, existingKeysSet);
|
|
94
103
|
summary[result.status].push(result.key);
|
|
95
104
|
const current = summary.success.length + summary.failed.length + summary.skipped.length;
|
|
96
105
|
if (this.reporter?.onProgress) {
|
|
@@ -115,10 +124,11 @@ export abstract class UploadService {
|
|
|
115
124
|
return files.filter((f) => shouldInclude(f.relativePath, include, exclude));
|
|
116
125
|
}
|
|
117
126
|
|
|
118
|
-
/**
|
|
127
|
+
/** 上传单个文件:有 existingKeysSet 时用 Set 判断存在,否则 head;存在则跳过,否则 put;子类可覆盖 */
|
|
119
128
|
protected async uploadOneFile(
|
|
120
129
|
file: LocalFile,
|
|
121
130
|
options: UploadDirectoryOptions,
|
|
131
|
+
existingKeysSet?: Set<string>,
|
|
122
132
|
): Promise<{ status: 'success' | 'skipped' | 'failed'; key: string }> {
|
|
123
133
|
const key = this.buildKey(
|
|
124
134
|
this.basePrefix,
|
|
@@ -128,10 +138,18 @@ export abstract class UploadService {
|
|
|
128
138
|
);
|
|
129
139
|
|
|
130
140
|
try {
|
|
131
|
-
const
|
|
132
|
-
if (
|
|
133
|
-
|
|
134
|
-
|
|
141
|
+
const forceUpload = options.forceUploadPatterns?.some((p) => key.includes(p));
|
|
142
|
+
if (existingKeysSet !== undefined) {
|
|
143
|
+
if (!forceUpload && existingKeysSet.has(key)) {
|
|
144
|
+
await this.reportFile(file, options.bucket, key, 'skipped');
|
|
145
|
+
return { status: 'skipped', key };
|
|
146
|
+
}
|
|
147
|
+
} else {
|
|
148
|
+
const head = await this.storageClient.headObject(options.bucket, key);
|
|
149
|
+
if (head.exists) {
|
|
150
|
+
await this.reportFile(file, options.bucket, key, 'skipped');
|
|
151
|
+
return { status: 'skipped', key };
|
|
152
|
+
}
|
|
135
153
|
}
|
|
136
154
|
|
|
137
155
|
if (this.fileProcessor) {
|
package/src/core/reporter.ts
CHANGED
|
@@ -89,9 +89,10 @@ export class ConsoleReporter implements Reporter {
|
|
|
89
89
|
onComplete(summary: UploadSummary): void {
|
|
90
90
|
console.log('上传完成:');
|
|
91
91
|
console.log(` 总数: ${summary.total}`);
|
|
92
|
+
console.log(`\n 跳过: ${summary.skipped.length}`);
|
|
92
93
|
console.log(` 成功:\n ${summary.success.join('\n')}`);
|
|
93
94
|
console.log(`\n\n\n 失败:\n ${summary.failed.join('\n')}`);
|
|
94
|
-
|
|
95
|
+
|
|
95
96
|
}
|
|
96
97
|
}
|
|
97
98
|
|
|
@@ -50,6 +50,17 @@ export class OssService {
|
|
|
50
50
|
pathPrefix: options.pathPrefix,
|
|
51
51
|
include: options.include,
|
|
52
52
|
exclude: options.exclude,
|
|
53
|
+
forceUploadPatterns: getForceUploadPatterns(),
|
|
53
54
|
});
|
|
54
55
|
}
|
|
55
56
|
}
|
|
57
|
+
|
|
58
|
+
function getForceUploadPatterns(): string[] {
|
|
59
|
+
try {
|
|
60
|
+
const raw = process.env.FILE_RE_WHITE_LIST;
|
|
61
|
+
if (!raw) return [];
|
|
62
|
+
return JSON.parse(raw);
|
|
63
|
+
} catch {
|
|
64
|
+
return [];
|
|
65
|
+
}
|
|
66
|
+
}
|
|
@@ -20,8 +20,39 @@ export class HuaweiObsClient implements StorageClient {
|
|
|
20
20
|
this.whiteList = JSON.parse(process.env.FILE_RE_WHITE_LIST as string)
|
|
21
21
|
}
|
|
22
22
|
|
|
23
|
+
/**
|
|
24
|
+
* 按前缀列举对象 key(分页拉全),用于上传前一次拉取、内存判断存在,避免每个文件都 head。
|
|
25
|
+
*/
|
|
26
|
+
async listObjectKeys(bucket: string, prefix: string): Promise<string[]> {
|
|
27
|
+
const keys: string[] = [];
|
|
28
|
+
let marker: string | undefined;
|
|
29
|
+
const pageSize = 1000;
|
|
30
|
+
do {
|
|
31
|
+
const param: Record<string, unknown> = {
|
|
32
|
+
Bucket: bucket,
|
|
33
|
+
MaxKeys: pageSize,
|
|
34
|
+
Prefix: prefix || undefined,
|
|
35
|
+
};
|
|
36
|
+
if (marker) param.Marker = marker;
|
|
37
|
+
const result = await this.client.listObjects(param);
|
|
38
|
+
if (result.CommonMsg.Status > 300) {
|
|
39
|
+
throw new Error(
|
|
40
|
+
`OBS listObjects 失败: ${result.CommonMsg.Status} ${result.CommonMsg.Code} ${result.CommonMsg.Message}`,
|
|
41
|
+
);
|
|
42
|
+
}
|
|
43
|
+
const contents = result.InterfaceResult?.Contents ?? [];
|
|
44
|
+
for (const item of contents) {
|
|
45
|
+
if (item.Key) keys.push(item.Key);
|
|
46
|
+
}
|
|
47
|
+
const truncated = result.InterfaceResult?.IsTruncated === 'true';
|
|
48
|
+
marker = truncated ? result.InterfaceResult?.NextMarker : undefined;
|
|
49
|
+
} while (marker);
|
|
50
|
+
return keys;
|
|
51
|
+
}
|
|
52
|
+
|
|
23
53
|
/**
|
|
24
54
|
* 调用 OBS getObjectMetadata 查元数据,判断对象是否存在(同路径即跳过上传)。
|
|
55
|
+
* 当使用 listObjectKeys 时,上传流程会优先用内存 Set 判断,不再逐文件调用本方法。
|
|
25
56
|
*/
|
|
26
57
|
async headObject(bucket: string, key: string): Promise<HeadObjectResult> {
|
|
27
58
|
try {
|