@soga/uploader 0.2.41 → 0.2.43

package/dist/host-uploader/base.js

@@ -1 +1 @@
-"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.BaseUploader=void 0;const typeorm_1=require("typeorm"),entities_1=require("@soga/entities"),types_1=require("@soga/types"),prepare_1=require("../hooks/prepare"),utils_1=require("@soga/utils");let isDequeueing=!1;class BaseUploader{hostType;onProgress=async()=>{};onSuccess=async()=>{};onError=async()=>{};successRepository;fileRepository;partRepository;isRunning=!1;thread_count=1;maxThreads=10;threads=[];baseParams;getValidThreads(t){return Math.min(Math.max(t??0,0),this.maxThreads)}async onPartProgress(t){const{id:s,data:e,type:i}=t;if("percent"!=i)return;const{part_id:a,percent:o}=e,r=await this.partRepository.findOneBy({id:a});if(!r)return;if(r.upload_status==types_1.UploadStatus.ERROR)return;if(r.upload_status==types_1.UploadStatus.SUCCESS)return;await this.partRepository.update(a,{upload_percent:o});const p=await this.partRepository.findBy({file_id:s,host_type:this.hostType});let d=0,n=0;for(const t of p)d+=t.info.size,n+=t.info.size*t.upload_percent;const u=n/d,l=await this.fileRepository.findOneBy({id:s}),h=types_1.UploadProcessStep[`upload_${this.hostType}`];l.progress[h].percent=u,l.percent=(0,utils_1.getProgressPercent)(l.progress),await this.fileRepository.update(s,{progress:l.progress,percent:l.percent}),await this.onProgress(l)}async onPartSuccess({file_id:t,host_type:s}){const e=await this.fileRepository.findOneBy({id:t}),i=`${s}_upload_status`;if(e[i]==types_1.UploadStatus.ERROR)return;if(await this.partRepository.findOneBy({file_id:e.id,host_type:s,upload_status:(0,typeorm_1.Not)(types_1.UploadStatus.SUCCESS)}))return;const a=await this.partRepository.findBy({file_id:e.id,host_type:s}),o={};if(s==types_1.HostType.BAIDU)for(const t of a)o[t.info.file]={...t.upload_result.baidu,md4:t.info.md4};else if(s==types_1.HostType.ALI)for(const t of a)o[t.info.file]={...t.upload_result.ali,sha1:t.info.sha1};const r=`${s}_upload_result`,p=await this.fileRepository.findOneBy({id:e.id}),d=types_1.UploadProcessStep[`upload_${this.hostType}`];p.progress[d].percent=1,p.percent=(0,utils_1.getProgressPercent)(p.progress),await this.fileRepository.update(e.id,{[i]:types_1.UploadStatus.SUCCESS,[r]:o,progress:p.progress,percent:p.percent});const n=await this.fileRepository.findOneBy({id:e.id});await this.checkAllHostsUploaded(n)&&await this.onSuccess(n)}async checkAllHostsUploaded(t){return(!t.baidu_host_id||t.baidu_upload_status==types_1.UploadStatus.SUCCESS)&&(!t.ali_host_id||t.ali_upload_status==types_1.UploadStatus.SUCCESS)}async onPartError(t,s){const e=`${this.hostType}_upload_status`;s[e]!=types_1.UploadStatus.ERROR&&(await this.fileRepository.update(s.id,{[e]:types_1.UploadStatus.ERROR,upload_status:types_1.UploadStatus.ERROR}),await this.onError(t,s))}constructor(t,s){this.hostType=s,this.baseParams=t;const{dataSource:e}=t;this.fileRepository=e.getRepository(entities_1.UploadFile),this.successRepository=e.getRepository(entities_1.UploadSuccess),this.partRepository=e.getRepository(entities_1.UploadPart),this.thread_count=this.getValidThreads(t.threads??this.thread_count),t.onProgress&&(this.onProgress=t.onProgress.bind(this)),t.onSuccess&&(this.onSuccess=t.onSuccess.bind(this)),t.onError&&(this.onError=t.onError.bind(this))}async setThreads(t){const s=this.getValidThreads(t);this.thread_count=s,await this.run()}async start(){await this.run()}async stopFiles(t){const s=this.threads.filter((s=>t.includes(s.file_id)));await Promise.all(s.map((t=>t.stop()))),await this.start()}async deleteFiles(t){const s=this.threads.filter((s=>t.includes(s.file_id)));await Promise.all(s.map((t=>t.stop())))}async stopAll(){await Promise.all(this.threads.map((t=>t.stop())))}async checkProcessingFile(t){if(t.baidu_host_id&&t.baidu_upload_status==types_1.UploadStatus.PROCESS){await this.partRepository.findOneBy({file_id:t.id,host_type:types_1.HostType.BAIDU,upload_status:(0,typeorm_1.Not)(types_1.UploadStatus.SUCCESS)})||await this.onPartSuccess({file_id:t.id,host_type:types_1.HostType.BAIDU})}if(t.ali_host_id&&t.ali_upload_status==types_1.UploadStatus.PROCESS){await this.partRepository.findOneBy({file_id:t.id,host_type:types_1.HostType.ALI,upload_status:(0,typeorm_1.Not)(types_1.UploadStatus.SUCCESS)})||await this.onPartSuccess({file_id:t.id,host_type:types_1.HostType.ALI})}}async run(){for(;this.isRunning;)await new Promise((t=>{setTimeout(t,200)}));try{this.isRunning=!0;const t=Date.now(),s=await this.fileRepository.findBy({uid:this.baseParams.uid,is_paused:!1,upload_status:types_1.UploadStatus.PROCESS});if(s.length){const t=this.threads.map((t=>t.file_id)),e=s.filter((s=>!t.includes(s.id)));for(const t of e)await this.checkProcessingFile(t)}console.log("exception time1: ",Date.now()-t);const e=Date.now(),i=await this.partRepository.findBy({uid:this.baseParams.uid,is_paused:!1,upload_status:types_1.UploadStatus.PROCESS,host_id:this.baseParams.host_id});if(i.length){const t=this.threads.map((t=>t.part_id)),s=i.filter((s=>!t.includes(s.id)));if(s.length){const t=s.map((t=>t.id));console.log("exception ids: ",t),await this.partRepository.update({id:(0,typeorm_1.In)(t),upload_status:types_1.UploadStatus.PROCESS},{upload_status:types_1.UploadStatus.NULL})}}console.log("exception time2: ",Date.now()-e);const a=this.threads.length,o=this.thread_count;if(a<o){const t=o-a,s=async()=>{const t=await this.partRepository.findOne({where:{uid:this.baseParams.uid,is_paused:!1,upload_status:types_1.UploadStatus.NULL,host_id:this.baseParams.host_id},order:{created_at:"ASC"}});if(!t)return null;const{file_id:e,host_id:i,part_name:a}=t,o=await this.partRepository.findBy({file_id:e,host_id:i,part_name:a});if(o.length>1){const t=await this.partRepository.findOneBy({file_id:e,host_id:i,part_name:a,upload_status:types_1.UploadStatus.SUCCESS});if(t){await this.partRepository.delete({file_id:e,host_id:i,part_name:a,id:(0,typeorm_1.Not)(t.id)});const o=await this.fileRepository.findOneBy({id:e});return await this.onPartSuccess({file_id:o.id,host_type:this.hostType}),await s()}{const t=o.sort(((t,s)=>s.upload_percent-t.upload_percent)),s=t[0],e=t.slice(1).map((t=>t.id));return await this.partRepository.delete({id:(0,typeorm_1.In)(e)}),s}}return t};for(let e=0;e<t;e++){const t=await s();if(!t)break;const e=this.getThread(t);this.threads.push(e),await this.partRepository.update(t.id,{upload_status:types_1.UploadStatus.PROCESS}),e.start()}if(this.threads.length<o){await this.dequeueOneFile();await s()&&(this.isRunning=!1,await this.run())}}else if(a>o){const t=a-o,s=this.threads.slice(0,t);for(const t of s)await t.stop()}}finally{this.isRunning=!1}}async dequeueOneFile(){for(;isDequeueing;)await new Promise((t=>{setTimeout(t,50)}));try{isDequeueing=!0;const t=`${this.hostType}_upload_status`,s=`${this.hostType}_host_id`,e=await this.fileRepository.findOne({where:{uid:this.baseParams.uid,is_paused:!1,encode_status:types_1.EncodeStatus.SUCCESS,[t]:(0,typeorm_1.IsNull)(),upload_status:(0,typeorm_1.In)([types_1.UploadStatus.NULL,types_1.UploadStatus.PROCESS])},order:{created_at:"ASC"}});if(!e)return;await this.fileRepository.update(e.id,{[t]:types_1.UploadStatus.PROCESS,upload_status:types_1.UploadStatus.PROCESS});if(this.partRepository.findOneBy({file_id:e.id,host_type:this.hostType}))return void await this.partRepository.update({file_id:e.id,host_type:this.hostType,upload_status:(0,typeorm_1.Not)(types_1.UploadStatus.SUCCESS)},{upload_status:types_1.UploadStatus.PROCESS});await(0,prepare_1.prepare)({file_id:e.id,sdk_domain:this.baseParams.sdk_domain,fileRepository:this.fileRepository});const{encoded:i}=e,{affix:a,source:o,txt:r,img:p,media:d}=i,n=[...o?.parts??[],...r?.parts??[],...p?.parts??[],...d?.parts??[]];a?.forEach((t=>{t.parts?.forEach((t=>{n.push(t)}))}));const u=e[s],l=[],{length:h}=n;for(let t=0;t<h;t++){const s=n[t];await this.partRepository.findOneBy({file_id:e.id,part_name:s.file,host_id:u})||l.push({uid:this.baseParams.uid,file_id:e.id,part_name:s.file,info:s,output_root:e.output_root,host_id:u,host_type:this.hostType})}return void await this.partRepository.save(l)}finally{isDequeueing=!1}}}exports.BaseUploader=BaseUploader;
+"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.BaseUploader=void 0;const typeorm_1=require("typeorm"),entities_1=require("@soga/entities"),types_1=require("@soga/types"),prepare_1=require("../hooks/prepare"),utils_1=require("@soga/utils");let isDequeueing=!1;class BaseUploader{hostType;onProgress=async()=>{};onSuccess=async()=>{};onError=async()=>{};successRepository;fileRepository;partRepository;isRunning=!1;thread_count=1;maxThreads=10;threads=[];baseParams;getValidThreads(t){return Math.min(Math.max(t??0,0),this.maxThreads)}async onPartProgress(t){const{id:s,data:e,type:i}=t;if("percent"!=i)return;const{part_id:a,percent:o}=e,r=await this.partRepository.findOneBy({id:a});if(!r)return;if(r.upload_status==types_1.UploadStatus.ERROR)return;if(r.upload_status==types_1.UploadStatus.SUCCESS)return;await this.partRepository.update(a,{upload_percent:o});const p=await this.partRepository.findBy({file_id:s,host_type:this.hostType});let d=0,n=0;for(const t of p)d+=t.info.size,n+=t.info.size*t.upload_percent;const u=n/d,l=await this.fileRepository.findOneBy({id:s}),h=types_1.UploadProcessStep[`upload_${this.hostType}`];l.progress[h].percent=u,l.percent=(0,utils_1.getProgressPercent)(l.progress),await this.fileRepository.update(s,{progress:l.progress,percent:l.percent}),await this.onProgress(l)}async onPartSuccess({file_id:t,host_type:s}){const e=await this.fileRepository.findOneBy({id:t}),i=`${s}_upload_status`;if(e[i]==types_1.UploadStatus.ERROR)return;if(await this.partRepository.findOneBy({file_id:e.id,host_type:s,upload_status:(0,typeorm_1.Not)(types_1.UploadStatus.SUCCESS)}))return;const a=await this.partRepository.findBy({file_id:e.id,host_type:s}),o={};if(s==types_1.HostType.BAIDU)for(const t of a)o[t.info.file]={...t.upload_result.baidu,md4:t.info.md4};else if(s==types_1.HostType.ALI)for(const t of a)o[t.info.file]={...t.upload_result.ali,sha1:t.info.sha1};const r=`${s}_upload_result`,p=await this.fileRepository.findOneBy({id:e.id}),d=types_1.UploadProcessStep[`upload_${this.hostType}`];p.progress[d].percent=1,p.percent=(0,utils_1.getProgressPercent)(p.progress),await this.fileRepository.update(e.id,{[i]:types_1.UploadStatus.SUCCESS,[r]:o,progress:p.progress,percent:p.percent});const n=await this.fileRepository.findOneBy({id:e.id});await this.checkAllHostsUploaded(n)&&await this.onSuccess(n)}async checkAllHostsUploaded(t){return(!t.baidu_host_id||t.baidu_upload_status==types_1.UploadStatus.SUCCESS)&&(!t.ali_host_id||t.ali_upload_status==types_1.UploadStatus.SUCCESS)}async onPartError(t,s){const e=`${this.hostType}_upload_status`;s[e]!=types_1.UploadStatus.ERROR&&(await this.fileRepository.update(s.id,{[e]:types_1.UploadStatus.ERROR,upload_status:types_1.UploadStatus.ERROR}),await this.onError(t,s))}constructor(t,s){this.hostType=s,this.baseParams=t;const{dataSource:e}=t;this.fileRepository=e.getRepository(entities_1.UploadFile),this.successRepository=e.getRepository(entities_1.UploadSuccess),this.partRepository=e.getRepository(entities_1.UploadPart),this.thread_count=this.getValidThreads(t.threads??this.thread_count),t.onProgress&&(this.onProgress=t.onProgress.bind(this)),t.onSuccess&&(this.onSuccess=t.onSuccess.bind(this)),t.onError&&(this.onError=t.onError.bind(this))}async setThreads(t){const s=this.getValidThreads(t);this.thread_count=s,await this.run()}async start(){await this.run()}async stopFiles(t){const s=this.threads.filter((s=>t.includes(s.file_id)));await Promise.all(s.map((t=>t.stop()))),await this.start()}async deleteFiles(t){const s=this.threads.filter((s=>t.includes(s.file_id)));await Promise.all(s.map((t=>t.stop())))}async stopAll(){await Promise.all(this.threads.map((t=>t.stop())))}async checkProcessingFile(t){if(t.baidu_host_id&&t.baidu_upload_status==types_1.UploadStatus.PROCESS){await this.partRepository.findOneBy({file_id:t.id,host_type:types_1.HostType.BAIDU,upload_status:(0,typeorm_1.Not)(types_1.UploadStatus.SUCCESS)})||await this.onPartSuccess({file_id:t.id,host_type:types_1.HostType.BAIDU})}if(t.ali_host_id&&t.ali_upload_status==types_1.UploadStatus.PROCESS){await this.partRepository.findOneBy({file_id:t.id,host_type:types_1.HostType.ALI,upload_status:(0,typeorm_1.Not)(types_1.UploadStatus.SUCCESS)})||await this.onPartSuccess({file_id:t.id,host_type:types_1.HostType.ALI})}}async run(){for(;this.isRunning;)await new Promise((t=>{setTimeout(t,200)}));try{this.isRunning=!0;const t=Date.now(),s=await this.fileRepository.findBy({uid:this.baseParams.uid,is_paused:!1,upload_status:types_1.UploadStatus.PROCESS});if(s.length){const t=this.threads.map((t=>t.file_id)),e=s.filter((s=>!t.includes(s.id)));for(const t of e)await this.checkProcessingFile(t)}console.log("exception time1: ",Date.now()-t);const e=Date.now(),i=await this.partRepository.findBy({uid:this.baseParams.uid,is_paused:!1,upload_status:types_1.UploadStatus.PROCESS,host_id:this.baseParams.host_id});if(i.length){const t=this.threads.map((t=>t.part_id)),s=i.filter((s=>!t.includes(s.id)));if(s.length){const t=s.map((t=>t.id));console.log("exception ids: ",t),await this.partRepository.update({id:(0,typeorm_1.In)(t),upload_status:types_1.UploadStatus.PROCESS},{upload_status:types_1.UploadStatus.NULL})}}console.log("exception time2: ",Date.now()-e);const a=this.threads.length,o=this.thread_count;if(a<o){const t=o-a,s=async()=>await this.partRepository.findOne({where:{uid:this.baseParams.uid,is_paused:!1,upload_status:types_1.UploadStatus.NULL,host_id:this.baseParams.host_id},order:{created_at:"ASC"}});for(let e=0;e<t;e++){const t=await s();if(!t)break;const e=this.getThread(t);this.threads.push(e),await this.partRepository.update(t.id,{upload_status:types_1.UploadStatus.PROCESS}),e.start()}if(this.threads.length<o){await this.dequeueOneFile();await s()&&(this.isRunning=!1,await this.run())}}else if(a>o){const t=a-o,s=this.threads.slice(0,t);for(const t of s)await t.stop()}}finally{this.isRunning=!1}}async dequeueOneFile(){for(;isDequeueing;)await new Promise((t=>{setTimeout(t,50)}));try{isDequeueing=!0;const t=`${this.hostType}_upload_status`,s=`${this.hostType}_host_id`,e=await this.fileRepository.findOne({where:{uid:this.baseParams.uid,is_paused:!1,encode_status:types_1.EncodeStatus.SUCCESS,[t]:(0,typeorm_1.IsNull)(),upload_status:(0,typeorm_1.In)([types_1.UploadStatus.NULL,types_1.UploadStatus.PROCESS])},order:{created_at:"ASC"}});if(!e)return;await this.fileRepository.update(e.id,{[t]:types_1.UploadStatus.PROCESS,upload_status:types_1.UploadStatus.PROCESS}),await(0,prepare_1.prepare)({file_id:e.id,sdk_domain:this.baseParams.sdk_domain,fileRepository:this.fileRepository});const{encoded:i}=e,{affix:a,source:o,txt:r,img:p,media:d}=i,n=[...o?.parts??[],...r?.parts??[],...p?.parts??[],...d?.parts??[]];a?.forEach((t=>{t.parts?.forEach((t=>{n.push(t)}))}));const u=e[s],l=[],{length:h}=n;for(let t=0;t<h;t++){const s=n[t];await this.partRepository.findOneBy({file_id:e.id,part_name:s.file,host_id:u})||l.push({uid:this.baseParams.uid,file_id:e.id,part_name:s.file,info:s,output_root:e.output_root,host_id:u,host_type:this.hostType})}return void await this.partRepository.save(l)}finally{isDequeueing=!1}}}exports.BaseUploader=BaseUploader;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@soga/uploader",
-  "version": "0.2.41",
+  "version": "0.2.43",
   "publishConfig": {
     "access": "public"
   },
@@ -55,5 +55,5 @@
     "@soga/utils": "^0.2.33",
     "piscina": "^4.9.2"
   },
-  "gitHead": "b92fb54c5ef456202704dc8eec228e5c5c15b98f"
+  "gitHead": "973f8ae35fcd223ef6ebe0f00163383dac80455b"
 }