@soga/uploader 0.2.27 → 0.2.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/host-uploader/base.js +1 -1
- package/dist/main.js +1 -1
- package/dist/uploader.js +1 -1
- package/package.json +3 -3
|
@@ -1 +1 @@
|
|
|
1
|
-
"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.BaseUploader=void 0;const typeorm_1=require("typeorm"),entities_1=require("@soga/entities"),types_1=require("@soga/types"),prepare_1=require("../hooks/prepare"),utils_1=require("@soga/utils");let isDequeueing=!1;class BaseUploader{hostType;onProgress=async()=>{};onSuccess=async()=>{};onError=async()=>{};successRepository;fileRepository;partRepository;isRunning=!1;thread_count=1;maxThreads=10;threads=[];baseParams;getValidThreads(t){return Math.min(Math.max(t??0,0),this.maxThreads)}async onPartProgress(t){const{id:s,data:e,type:i}=t;if("percent"!=i)return;const{part_id:a,percent:o}=e,r=await this.partRepository.findOneBy({id:a});if(!r)return;if(r.upload_status==types_1.UploadStatus.ERROR)return;if(r.upload_status==types_1.UploadStatus.SUCCESS)return;await this.partRepository.update(a,{upload_percent:o});const p=await this.partRepository.findBy({file_id:s,host_type:this.hostType});let d=0,n=0;for(const t of p)d+=t.info.size,n+=t.info.size*t.upload_percent;const u=n/d,l=await this.fileRepository.findOneBy({id:s}),h=types_1.UploadProcessStep[`upload_${this.hostType}`];l.progress[h].percent=u,l.percent=(0,utils_1.getProgressPercent)(l.progress),await this.fileRepository.update(s,{progress:l.progress,percent:l.percent}),await this.onProgress(l)}async onPartSuccess({file_id:t,host_type:s}){const e=await this.fileRepository.findOneBy({id:t}),i=`${s}_upload_status`;if(e[i]==types_1.UploadStatus.ERROR)return;if(await this.partRepository.findOneBy({file_id:e.id,host_type:s,upload_status:(0,typeorm_1.Not)(types_1.UploadStatus.SUCCESS)}))return;const a=await this.partRepository.findBy({file_id:e.id,host_type:s}),o={};if(s==types_1.HostType.BAIDU)for(const t of a)o[t.info.file]={...t.upload_result.baidu,md4:t.info.md4};else if(s==types_1.HostType.ALI)for(const t of a)o[t.info.file]={...t.upload_result.ali,sha1:t.info.sha1};const r=`${s}_upload_result`,p=await this.fileRepository.findOneBy({id:e.id}),d=types_1.UploadProcessStep[`upload_${this.hostType}`];p.progress[d].percent=1,p.percent=(0,utils_1.getProgressPercent)(p.progress),await this.fileRepository.update(e.id,{[i]:types_1.UploadStatus.SUCCESS,[r]:o,progress:p.progress,percent:p.percent});const n=await this.fileRepository.findOneBy({id:e.id});await this.checkAllHostsUploaded(n)&&await this.onSuccess(n)}async checkAllHostsUploaded(t){return(!t.baidu_host_id||t.baidu_upload_status==types_1.UploadStatus.SUCCESS)&&(!t.ali_host_id||t.ali_upload_status==types_1.UploadStatus.SUCCESS)}async onPartError(t,s){console.log("on part error: ",s.id),console.log(t);const e=`${this.hostType}_upload_status`;s[e]!=types_1.UploadStatus.ERROR&&(await this.fileRepository.update(s.id,{[e]:types_1.UploadStatus.ERROR,upload_status:types_1.UploadStatus.ERROR}),await this.onError(t,s))}constructor(t,s){this.hostType=s,this.baseParams=t;const{dataSource:e}=t;this.fileRepository=e.getRepository(entities_1.UploadFile),this.successRepository=e.getRepository(entities_1.UploadSuccess),this.partRepository=e.getRepository(entities_1.UploadPart),this.thread_count=this.getValidThreads(t.threads??this.thread_count),t.onProgress&&(this.onProgress=t.onProgress.bind(this)),t.onSuccess&&(this.onSuccess=t.onSuccess.bind(this)),t.onError&&(this.onError=t.onError.bind(this))}async setThreads(t){const s=this.getValidThreads(t);this.thread_count=s,await this.run()}async start(){await this.run()}async stopFiles(t){const s=this.threads.filter((s=>t.includes(s.file_id)));await Promise.all(s.map((t=>t.stop()))),await this.start()}async stopAll(){await Promise.all(this.threads.map((t=>t.stop())))}async checkProcessingFile(t){if(t.baidu_host_id&&t.baidu_upload_status==types_1.UploadStatus.PROCESS){await this.partRepository.findOneBy({file_id:t.id,host_type:types_1.HostType.BAIDU,upload_status:(0,typeorm_1.Not)(types_1.UploadStatus.SUCCESS)})||await this.onPartSuccess({file_id:t.id,host_type:types_1.HostType.BAIDU})}if(t.ali_host_id&&t.ali_upload_status==types_1.UploadStatus.PROCESS){await this.partRepository.findOneBy({file_id:t.id,host_type:types_1.HostType.ALI,upload_status:(0,typeorm_1.Not)(types_1.UploadStatus.SUCCESS)})||await this.onPartSuccess({file_id:t.id,host_type:types_1.HostType.ALI})}}async run(){for(;this.isRunning;)await new Promise((t=>{setTimeout(t,200)}));this.isRunning=!0;const t=Date.now(),s=await this.fileRepository.findBy({uid:this.baseParams.uid,is_paused:!1,upload_status:types_1.UploadStatus.PROCESS});if(s.length){const t=this.threads.map((t=>t.file_id)),e=s.filter((s=>!t.includes(s.id)));for(const t of e)await this.checkProcessingFile(t)}console.log("exception time1: ",Date.now()-t);const e=Date.now(),i=await this.partRepository.findBy({uid:this.baseParams.uid,is_paused:!1,upload_status:types_1.UploadStatus.PROCESS,host_id:this.baseParams.host_id});if(i.length){const t=this.threads.map((t=>t.part_id)),s=i.filter((s=>!t.includes(s.id)));if(s.length){const t=s.map((t=>t.id));await this.partRepository.update({id:(0,typeorm_1.In)(t),upload_status:types_1.UploadStatus.PROCESS},{upload_status:types_1.UploadStatus.NULL})}}console.log("exception time2: ",Date.now()-e);const a=this.threads.length,o=this.thread_count;if(a<o){const t=o-a,s=async()=>await this.partRepository.findOne({where:{uid:this.baseParams.uid,is_paused:!1,upload_status:types_1.UploadStatus.NULL,host_id:this.baseParams.host_id},order:{created_at:"ASC"}});for(let e=0;e<t;e++){const t=await s();if(!t)break;const e=this.getThread(t);this.threads.push(e),await this.partRepository.update(t.id,{upload_status:types_1.UploadStatus.PROCESS}),e.start()}if(this.threads.length<o){await this.dequeueOneFile();await s()&&(this.isRunning=!1,await this.run())}}else if(a>o){const t=a-o,s=this.threads.slice(0,t);for(const t of s)await t.stop()}this.isRunning=!1}async dequeueOneFile(){for(;isDequeueing;)await new Promise((t=>{setTimeout(t,50)}));isDequeueing=!0;
|
|
1
|
+
"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.BaseUploader=void 0;const typeorm_1=require("typeorm"),entities_1=require("@soga/entities"),types_1=require("@soga/types"),prepare_1=require("../hooks/prepare"),utils_1=require("@soga/utils");let isDequeueing=!1;class BaseUploader{hostType;onProgress=async()=>{};onSuccess=async()=>{};onError=async()=>{};successRepository;fileRepository;partRepository;isRunning=!1;thread_count=1;maxThreads=10;threads=[];baseParams;getValidThreads(t){return Math.min(Math.max(t??0,0),this.maxThreads)}async onPartProgress(t){const{id:s,data:e,type:i}=t;if("percent"!=i)return;const{part_id:a,percent:o}=e,r=await this.partRepository.findOneBy({id:a});if(!r)return;if(r.upload_status==types_1.UploadStatus.ERROR)return;if(r.upload_status==types_1.UploadStatus.SUCCESS)return;await this.partRepository.update(a,{upload_percent:o});const p=await this.partRepository.findBy({file_id:s,host_type:this.hostType});let d=0,n=0;for(const t of p)d+=t.info.size,n+=t.info.size*t.upload_percent;const u=n/d,l=await this.fileRepository.findOneBy({id:s}),h=types_1.UploadProcessStep[`upload_${this.hostType}`];l.progress[h].percent=u,l.percent=(0,utils_1.getProgressPercent)(l.progress),await this.fileRepository.update(s,{progress:l.progress,percent:l.percent}),await this.onProgress(l)}async onPartSuccess({file_id:t,host_type:s}){const e=await this.fileRepository.findOneBy({id:t}),i=`${s}_upload_status`;if(e[i]==types_1.UploadStatus.ERROR)return;if(await this.partRepository.findOneBy({file_id:e.id,host_type:s,upload_status:(0,typeorm_1.Not)(types_1.UploadStatus.SUCCESS)}))return;const a=await this.partRepository.findBy({file_id:e.id,host_type:s}),o={};if(s==types_1.HostType.BAIDU)for(const t of a)o[t.info.file]={...t.upload_result.baidu,md4:t.info.md4};else if(s==types_1.HostType.ALI)for(const t of a)o[t.info.file]={...t.upload_result.ali,sha1:t.info.sha1};const r=`${s}_upload_result`,p=await this.fileRepository.findOneBy({id:e.id}),d=types_1.UploadProcessStep[`upload_${this.hostType}`];p.progress[d].percent=1,p.percent=(0,utils_1.getProgressPercent)(p.progress),await this.fileRepository.update(e.id,{[i]:types_1.UploadStatus.SUCCESS,[r]:o,progress:p.progress,percent:p.percent});const n=await this.fileRepository.findOneBy({id:e.id});await this.checkAllHostsUploaded(n)&&await this.onSuccess(n)}async checkAllHostsUploaded(t){return(!t.baidu_host_id||t.baidu_upload_status==types_1.UploadStatus.SUCCESS)&&(!t.ali_host_id||t.ali_upload_status==types_1.UploadStatus.SUCCESS)}async onPartError(t,s){console.log("on part error: ",s.id),console.log(t);const e=`${this.hostType}_upload_status`;s[e]!=types_1.UploadStatus.ERROR&&(await this.fileRepository.update(s.id,{[e]:types_1.UploadStatus.ERROR,upload_status:types_1.UploadStatus.ERROR}),await this.onError(t,s))}constructor(t,s){this.hostType=s,this.baseParams=t;const{dataSource:e}=t;this.fileRepository=e.getRepository(entities_1.UploadFile),this.successRepository=e.getRepository(entities_1.UploadSuccess),this.partRepository=e.getRepository(entities_1.UploadPart),this.thread_count=this.getValidThreads(t.threads??this.thread_count),t.onProgress&&(this.onProgress=t.onProgress.bind(this)),t.onSuccess&&(this.onSuccess=t.onSuccess.bind(this)),t.onError&&(this.onError=t.onError.bind(this))}async setThreads(t){const s=this.getValidThreads(t);this.thread_count=s,await this.run()}async start(){await this.run()}async stopFiles(t){const s=this.threads.filter((s=>t.includes(s.file_id)));await Promise.all(s.map((t=>t.stop()))),await this.start()}async stopAll(){await Promise.all(this.threads.map((t=>t.stop())))}async checkProcessingFile(t){if(t.baidu_host_id&&t.baidu_upload_status==types_1.UploadStatus.PROCESS){await this.partRepository.findOneBy({file_id:t.id,host_type:types_1.HostType.BAIDU,upload_status:(0,typeorm_1.Not)(types_1.UploadStatus.SUCCESS)})||await this.onPartSuccess({file_id:t.id,host_type:types_1.HostType.BAIDU})}if(t.ali_host_id&&t.ali_upload_status==types_1.UploadStatus.PROCESS){await this.partRepository.findOneBy({file_id:t.id,host_type:types_1.HostType.ALI,upload_status:(0,typeorm_1.Not)(types_1.UploadStatus.SUCCESS)})||await this.onPartSuccess({file_id:t.id,host_type:types_1.HostType.ALI})}}async run(){for(;this.isRunning;)await new Promise((t=>{setTimeout(t,200)}));try{this.isRunning=!0;const t=Date.now(),s=await this.fileRepository.findBy({uid:this.baseParams.uid,is_paused:!1,upload_status:types_1.UploadStatus.PROCESS});if(s.length){const t=this.threads.map((t=>t.file_id)),e=s.filter((s=>!t.includes(s.id)));for(const t of e)await this.checkProcessingFile(t)}console.log("exception time1: ",Date.now()-t);const e=Date.now(),i=await this.partRepository.findBy({uid:this.baseParams.uid,is_paused:!1,upload_status:types_1.UploadStatus.PROCESS,host_id:this.baseParams.host_id});if(i.length){const t=this.threads.map((t=>t.part_id)),s=i.filter((s=>!t.includes(s.id)));if(s.length){const t=s.map((t=>t.id));await this.partRepository.update({id:(0,typeorm_1.In)(t),upload_status:types_1.UploadStatus.PROCESS},{upload_status:types_1.UploadStatus.NULL})}}console.log("exception time2: ",Date.now()-e);const a=this.threads.length,o=this.thread_count;if(a<o){const t=o-a,s=async()=>await this.partRepository.findOne({where:{uid:this.baseParams.uid,is_paused:!1,upload_status:types_1.UploadStatus.NULL,host_id:this.baseParams.host_id},order:{created_at:"ASC"}});for(let e=0;e<t;e++){const t=await s();if(!t)break;const e=this.getThread(t);this.threads.push(e),await this.partRepository.update(t.id,{upload_status:types_1.UploadStatus.PROCESS}),e.start()}if(this.threads.length<o){await this.dequeueOneFile();await s()&&(this.isRunning=!1,await this.run())}}else if(a>o){const t=a-o,s=this.threads.slice(0,t);for(const t of s)await t.stop()}}finally{this.isRunning=!1}}async dequeueOneFile(){for(;isDequeueing;)await new Promise((t=>{setTimeout(t,50)}));try{isDequeueing=!0;const t=`${this.hostType}_upload_status`,s=`${this.hostType}_host_id`,e=await this.fileRepository.findOne({where:{uid:this.baseParams.uid,is_paused:!1,encode_status:types_1.EncodeStatus.SUCCESS,[t]:(0,typeorm_1.IsNull)(),upload_status:(0,typeorm_1.In)([types_1.UploadStatus.NULL,types_1.UploadStatus.PROCESS])},order:{created_at:"ASC"}});if(!e)return;await this.fileRepository.update(e.id,{[t]:types_1.UploadStatus.PROCESS,upload_status:types_1.UploadStatus.PROCESS}),await(0,prepare_1.prepare)({file_id:e.id,sdk_domain:this.baseParams.sdk_domain,fileRepository:this.fileRepository});const{encoded:i}=e,{affix:a,source:o,txt:r,img:p,media:d}=i,n=[...o?.parts??[],...r?.parts??[],...p?.parts??[],...d?.parts??[]];a?.forEach((t=>{t.parts?.forEach((t=>{n.push(t)}))}));const u=e[s],l=n.map((t=>({uid:this.baseParams.uid,file_id:e.id,info:t,output_root:e.output_root,host_id:u,host_type:this.hostType})));return void await this.partRepository.save(l)}finally{isDequeueing=!1}}}exports.BaseUploader=BaseUploader;
|
package/dist/main.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
"use strict";var __createBinding=this&&this.__createBinding||(Object.create?function(e,t,i
|
|
1
|
+
"use strict";var __createBinding=this&&this.__createBinding||(Object.create?function(e,t,r,i){void 0===i&&(i=r);var a=Object.getOwnPropertyDescriptor(t,r);a&&!("get"in a?!t.__esModule:a.writable||a.configurable)||(a={enumerable:!0,get:function(){return t[r]}}),Object.defineProperty(e,i,a)}:function(e,t,r,i){void 0===i&&(i=r),e[i]=t[r]}),__exportStar=this&&this.__exportStar||function(e,t){for(var r in e)"default"===r||Object.prototype.hasOwnProperty.call(t,r)||__createBinding(t,e,r)};Object.defineProperty(exports,"__esModule",{value:!0}),exports.getUploader=void 0,__exportStar(require("./host-uploader/baidu"),exports),__exportStar(require("./host-uploader/ali"),exports);const uploader_1=require("./uploader"),instanceMap=new Map,getUploader=async e=>{for(;instanceMap.get(e.uid)?.initing;)await new Promise((e=>setTimeout(e,100)));try{if(instanceMap.get(e.uid)?.uploader)return instanceMap.get(e.uid).uploader;const t={uploader:new uploader_1.Uploader(e),initing:!0};return instanceMap.set(e.uid,t),await t.uploader.init(),t.initing=!1,t.uploader}finally{instanceMap.get(e.uid)&&(instanceMap.get(e.uid).initing=!1)}};exports.getUploader=getUploader;
|
package/dist/uploader.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.Uploader=void 0;const types_1=require("@soga/types"),typeorm_1=require("typeorm"),entities_1=require("@soga/entities"),baidu_1=require("./host-uploader/baidu"),ali_1=require("./host-uploader/ali"),complete_1=require("./hooks/complete"),trasform_1=require("./hooks/trasform");class Uploader{params;uid;dataSource;fileRepository;partRepository;constructor(t){this.params=t,this.uid=t.uid,this.dataSource=t.dataSource,this.fileRepository=this.dataSource.getRepository(entities_1.UploadFile),this.partRepository=this.dataSource.getRepository(entities_1.UploadPart)}processorMap=new Map;successing=!1;async onSuccess(t){for(;this.successing;)await new Promise((t=>setTimeout(t,100)));this.successing=!0,await(0,complete_1.complete)({file_id:t.id,fileRepository:this.fileRepository,sdk_domain:this.params.sdk_domain}),await(0,trasform_1.transform)({file_id:t.id,dataSource:this.dataSource}),console.log("onSuccess",t)
|
|
1
|
+
"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.Uploader=void 0;const types_1=require("@soga/types"),typeorm_1=require("typeorm"),entities_1=require("@soga/entities"),baidu_1=require("./host-uploader/baidu"),ali_1=require("./host-uploader/ali"),complete_1=require("./hooks/complete"),trasform_1=require("./hooks/trasform");class Uploader{params;uid;dataSource;fileRepository;partRepository;constructor(t){this.params=t,this.uid=t.uid,this.dataSource=t.dataSource,this.fileRepository=this.dataSource.getRepository(entities_1.UploadFile),this.partRepository=this.dataSource.getRepository(entities_1.UploadPart)}processorMap=new Map;successing=!1;async onSuccess(t){for(;this.successing;)await new Promise((t=>setTimeout(t,100)));try{this.successing=!0,await(0,complete_1.complete)({file_id:t.id,fileRepository:this.fileRepository,sdk_domain:this.params.sdk_domain}),await(0,trasform_1.transform)({file_id:t.id,dataSource:this.dataSource}),console.log("onSuccess",t)}finally{this.successing=!1}}async init(){}async start(){await Promise.all([this.startAli(),this.startBaidu()])}async stopAll(){const t=this.processorMap.values();for(const s of t)await s.stopAll()}async stopFiles(t){const s=this.processorMap.values();for(const i of s)await i.stopFiles(t)}async repairAllParts(){await this.partRepository.update({uid:this.uid,upload_status:types_1.UploadStatus.ERROR},{upload_status:types_1.UploadStatus.NULL})}async repairParts(t){await this.partRepository.update({file_id:(0,typeorm_1.In)(t),upload_status:types_1.UploadStatus.ERROR},{upload_status:types_1.UploadStatus.NULL})}async startHost(t,s){if(this.processorMap.has(t)){const s=this.processorMap.get(t);return void await s.start()}const i={uid:this.uid,sdk_domain:this.params.sdk_domain,dataSource:this.dataSource,host_id:t,onProgress:this.params.onProgress,onSuccess:this.onSuccess.bind(this),onError:this.params.onError};if(s==types_1.HostType.BAIDU){const s=await(0,baidu_1.getBaiduUploader)(i);this.processorMap.set(t,s),await s.start()}else if(s==types_1.HostType.ALI){const s=await(0,ali_1.getAliUploader)(i);this.processorMap.set(t,s),await s.start()}}async startAli(){const t=(await this.fileRepository.createQueryBuilder("file").select("DISTINCT file.ali_host_id").where("file.uid = :uid",{uid:this.uid}).getRawMany()).map((t=>t.ali_host_id)).filter((t=>!!t));if(t.length)for(const s of t)await this.startHost(s,types_1.HostType.ALI)}async startBaidu(){const t=(await this.fileRepository.createQueryBuilder("file").select("DISTINCT file.baidu_host_id").where("file.uid = :uid",{uid:this.uid}).getRawMany()).map((t=>t.baidu_host_id)).filter((t=>!!t));if(t.length)for(const s of t)await this.startHost(s,types_1.HostType.BAIDU)}}exports.Uploader=Uploader;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@soga/uploader",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.28",
|
|
4
4
|
"publishConfig": {
|
|
5
5
|
"access": "public"
|
|
6
6
|
},
|
|
@@ -21,7 +21,7 @@
|
|
|
21
21
|
"prepublishOnly": "npm run build"
|
|
22
22
|
},
|
|
23
23
|
"devDependencies": {
|
|
24
|
-
"@soga/encoder": "^0.2.
|
|
24
|
+
"@soga/encoder": "^0.2.28",
|
|
25
25
|
"@types/fs-extra": "^11.0.4",
|
|
26
26
|
"@types/glob": "^8.1.0",
|
|
27
27
|
"@types/jest": "^29.5.14",
|
|
@@ -55,5 +55,5 @@
|
|
|
55
55
|
"@soga/utils": "^0.2.22",
|
|
56
56
|
"piscina": "^4.9.2"
|
|
57
57
|
},
|
|
58
|
-
"gitHead": "
|
|
58
|
+
"gitHead": "9045d969a1b82bd940b3dcfc3a56136208adc236"
|
|
59
59
|
}
|