@soga/uploader 0.1.4 → 0.1.6

package/dist/processor/base.js

@@ -1 +1 @@
-"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.BaseProcessor=void 0;const typeorm_1=require("typeorm"),entities_1=require("@soga/entities"),types_1=require("@soga/types"),prepare_1=require("../hooks/prepare"),utils_1=require("@soga/utils");let isDequeueing=!1;class BaseProcessor{hostType;onProgress=async()=>{};onSuccess=async()=>{};onError=async()=>{};fileRepository;partRepository;isRunning=!1;thread_count=3;maxThreads=10;threads=[];baseParams;getValidThreads(t){return Math.min(Math.max(t??0,0),this.maxThreads)}async onPartProgress(t){const{file_id:s,part_id:i,host_type:e,percent:a,type:r}=t,o=`${e}_percent`;await this.partRepository.update(i,{[o]:a});const p=await this.partRepository.findBy({file_id:s});let d=0,n=0;for(const t of p)d+=t.size,n+=t.size*t[o];const u=n/d,h=await this.fileRepository.findOneBy({id:s});h.progress[r].percent=u,h.percent=(0,utils_1.getProgressPercent)(h.progress),await this.fileRepository.update(s,{progress:h.progress,percent:h.percent}),await this.onProgress(h)}async onPartSuccess(t){const{ali_host_id:s,baidu_host_id:i}=t;if(s){if((await this.partRepository.findBy({file_id:t.id,ali_status:(0,typeorm_1.Not)(types_1.UploadStatus.SUCCESS)})).length)return}if(i){if((await this.partRepository.findBy({file_id:t.id,baidu_status:(0,typeorm_1.Not)(types_1.UploadStatus.SUCCESS)})).length)return}const e={ali:{},baidu:{}},a=await this.partRepository.findBy({file_id:t.id});for(const t of a)t.ali_status===types_1.UploadStatus.SUCCESS&&(e.ali[t.info.file]=t.ali_result),t.baidu_status===types_1.UploadStatus.SUCCESS&&(e.baidu[t.info.file]=t.baidu_result);await this.fileRepository.update(t.id,{upload_data:e}),t.progress.END.percent=1,t.percent=1,await this.fileRepository.update(t.id,{progress:t.progress,percent:t.percent,upload_status:types_1.UploadStatus.SUCCESS}),await this.partRepository.delete({file_id:t.id});const r=await this.fileRepository.findOneBy({id:t.id});await this.onSuccess(r)}async onPartError(t,s){s.upload_status!==types_1.UploadStatus.ERROR&&(await this.fileRepository.update(s.id,{upload_status:types_1.UploadStatus.ERROR}),await this.onError(t,s))}constructor(t){this.baseParams=t,this.fileRepository=t.dataSource.getRepository(entities_1.UploadFile),this.partRepository=t.dataSource.getRepository(entities_1.UploadPart),this.thread_count=this.getValidThreads(t.threads??this.thread_count),t.onProgress&&(this.onProgress=t.onProgress.bind(this)),t.onSuccess&&(this.onSuccess=t.onSuccess.bind(this)),t.onError&&(this.onError=t.onError.bind(this))}async setThreads(t){const s=this.getValidThreads(t);this.thread_count=s,await this.run()}is_initing=!1;async init(){this.is_initing=!0;const t=`${this.hostType}_status`;this.partRepository.update({uid:this.baseParams.uid,[t]:types_1.UploadStatus.PROCESS},{[t]:types_1.UploadStatus.NULL}),this.is_initing=!1}async start(){await this.run()}async stopFiles(t,s=!1){s&&(await this.partRepository.update({file_id:(0,typeorm_1.In)(t),is_paused:!1},{is_paused:!0}),await this.fileRepository.update({id:(0,typeorm_1.In)(t),is_paused:!1},{is_paused:!0}));const i=this.threads.filter((s=>t.includes(s.file_id)));await Promise.all(i.map((t=>t.stop()))),await this.start()}async stopAll(t=!1){t&&(await this.partRepository.update({uid:this.baseParams.uid},{is_paused:!0}),await this.fileRepository.update({uid:this.baseParams.uid,is_folder:!1},{is_paused:!0})),await Promise.all(this.threads.map((t=>t.stop())))}async run(){for(;this.isRunning;)await new Promise((t=>{setTimeout(t,200)}));this.isRunning=!0;const t=`${this.hostType}_status`,s=`${this.hostType}_host_id`,i=this.threads.length,e=this.thread_count;if(i<e){const a=e-i;for(let i=0;i<a;i++){const i=await this.partRepository.findOne({where:{uid:this.baseParams.uid,is_paused:!1,[t]:types_1.UploadStatus.NULL,[s]:this.baseParams.host_id},order:{created_at:"ASC"}});if(!i)break;const e=this.getThread(i);this.threads.push(e),await this.partRepository.update(i.id,{[t]:types_1.UploadStatus.PROCESS}),e.start()}if(this.threads.length<e){await this.dequeueOneFile();const i=await this.partRepository.findOneBy({uid:this.baseParams.uid,is_paused:!1,[t]:types_1.UploadStatus.NULL,[s]:this.baseParams.host_id});this.isRunning=!1,i&&await this.run()}}else if(i>e){const t=i-e,s=this.threads.slice(0,t);for(const t of s)await t.stop()}this.isRunning=!1}async dequeueOneFile(){for(;isDequeueing;)await new Promise((t=>{setTimeout(t,50)}));isDequeueing=!0;try{const t=await this.fileRepository.findOne({where:{uid:this.baseParams.uid,is_paused:!1,is_folder:!1,encode_status:types_1.EncodeStatus.SUCCESS,upload_status:types_1.UploadStatus.NULL},order:{created_at:"ASC"}});if(!t)return;await this.fileRepository.update(t.id,{upload_status:types_1.UploadStatus.PROCESS}),await(0,prepare_1.prepare)({file_id:t.id,fileRepository:this.fileRepository,createCloudRecord:this.baseParams.createCloudRecord});const{source_data:s,txt_data:i,img_data:e,media_data:a}=t,r=[...s.parts??[],...i.parts??[],...e.parts??[],...a.parts??[]].map((s=>({uid:this.baseParams.uid,file_id:t.id,info:s,filepath:t.path,output_root:t.output_root,size:s.size,ali_host_id:t.ali_host_id||null,baidu_host_id:t.baidu_host_id||null})));return void await this.partRepository.save(r)}finally{isDequeueing=!1}}}exports.BaseProcessor=BaseProcessor;
+"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.BaseProcessor=void 0;const typeorm_1=require("typeorm"),entities_1=require("@soga/entities"),types_1=require("@soga/types"),prepare_1=require("../hooks/prepare"),utils_1=require("@soga/utils"),complete_1=require("../hooks/complete");let isDequeueing=!1;class BaseProcessor{hostType;onProgress=async()=>{};onSuccess=async()=>{};onError=async()=>{};fileRepository;partRepository;isRunning=!1;thread_count=3;maxThreads=10;threads=[];baseParams;getValidThreads(t){return Math.min(Math.max(t??0,0),this.maxThreads)}async onPartProgress(t){const{file_id:s,part_id:i,host_type:e,percent:a,type:o}=t,r=`${e}_percent`;await this.partRepository.update(i,{[r]:a});const p=await this.partRepository.findBy({file_id:s});let d=0,u=0;for(const t of p)d+=t.size,u+=t.size*t[r];const n=u/d,h=await this.fileRepository.findOneBy({id:s});h.progress[o].percent=n,h.percent=(0,utils_1.getProgressPercent)(h.progress),await this.fileRepository.update(s,{progress:h.progress,percent:h.percent}),await this.onProgress(h)}async onPartSuccess(t){const{ali_host_id:s,baidu_host_id:i}=t;if(s){if((await this.partRepository.findBy({file_id:t.id,ali_status:(0,typeorm_1.Not)(types_1.UploadStatus.SUCCESS)})).length)return}if(i){if((await this.partRepository.findBy({file_id:t.id,baidu_status:(0,typeorm_1.Not)(types_1.UploadStatus.SUCCESS)})).length)return}const e={ali:{},baidu:{}},a=await this.partRepository.findBy({file_id:t.id});for(const t of a)t.ali_status===types_1.UploadStatus.SUCCESS&&(e.ali[t.info.file]=t.ali_result),t.baidu_status===types_1.UploadStatus.SUCCESS&&(e.baidu[t.info.file]=t.baidu_result);await this.fileRepository.update(t.id,{upload_data:e}),await(0,complete_1.complete)({file_id:t.id,fileRepository:this.fileRepository,updateCloudRecord:this.baseParams.updateCloudRecord}),t.progress.END.percent=1,t.percent=1,await this.fileRepository.update(t.id,{progress:t.progress,percent:t.percent,upload_status:types_1.UploadStatus.SUCCESS}),await this.partRepository.delete({file_id:t.id});const o=await this.fileRepository.findOneBy({id:t.id});await this.onSuccess(o)}async onPartError(t,s){s.upload_status!==types_1.UploadStatus.ERROR&&(await this.fileRepository.update(s.id,{upload_status:types_1.UploadStatus.ERROR}),await this.onError(t,s))}constructor(t){this.baseParams=t,this.fileRepository=t.dataSource.getRepository(entities_1.UploadFile),this.partRepository=t.dataSource.getRepository(entities_1.UploadPart),this.thread_count=this.getValidThreads(t.threads??this.thread_count),t.onProgress&&(this.onProgress=t.onProgress.bind(this)),t.onSuccess&&(this.onSuccess=t.onSuccess.bind(this)),t.onError&&(this.onError=t.onError.bind(this))}async setThreads(t){const s=this.getValidThreads(t);this.thread_count=s,await this.run()}is_initing=!1;async init(){this.is_initing=!0;const t=`${this.hostType}_status`;this.partRepository.update({uid:this.baseParams.uid,[t]:types_1.UploadStatus.PROCESS},{[t]:types_1.UploadStatus.NULL}),this.is_initing=!1}async start(){await this.run()}async stopFiles(t,s=!1){s&&(await this.partRepository.update({file_id:(0,typeorm_1.In)(t),is_paused:!1},{is_paused:!0}),await this.fileRepository.update({id:(0,typeorm_1.In)(t),is_paused:!1},{is_paused:!0}));const i=this.threads.filter((s=>t.includes(s.file_id)));await Promise.all(i.map((t=>t.stop()))),await this.start()}async stopAll(t=!1){t&&(await this.partRepository.update({uid:this.baseParams.uid},{is_paused:!0}),await this.fileRepository.update({uid:this.baseParams.uid,is_folder:!1},{is_paused:!0})),await Promise.all(this.threads.map((t=>t.stop())))}async run(){for(;this.isRunning;)await new Promise((t=>{setTimeout(t,200)}));this.isRunning=!0;const t=`${this.hostType}_status`,s=`${this.hostType}_host_id`,i=this.threads.length,e=this.thread_count;if(i<e){const a=e-i;for(let i=0;i<a;i++){const i=await this.partRepository.findOne({where:{uid:this.baseParams.uid,is_paused:!1,[t]:types_1.UploadStatus.NULL,[s]:this.baseParams.host_id},order:{created_at:"ASC"}});if(!i)break;const e=this.getThread(i);this.threads.push(e),await this.partRepository.update(i.id,{[t]:types_1.UploadStatus.PROCESS}),e.start()}if(this.threads.length<e){await this.dequeueOneFile();const i=await this.partRepository.findOneBy({uid:this.baseParams.uid,is_paused:!1,[t]:types_1.UploadStatus.NULL,[s]:this.baseParams.host_id});this.isRunning=!1,i&&await this.run()}}else if(i>e){const t=i-e,s=this.threads.slice(0,t);for(const t of s)await t.stop()}this.isRunning=!1}async dequeueOneFile(){for(;isDequeueing;)await new Promise((t=>{setTimeout(t,50)}));isDequeueing=!0;try{const t=await this.fileRepository.findOne({where:{uid:this.baseParams.uid,is_paused:!1,is_folder:!1,encode_status:types_1.EncodeStatus.SUCCESS,upload_status:types_1.UploadStatus.NULL},order:{created_at:"ASC"}});if(!t)return;await this.fileRepository.update(t.id,{upload_status:types_1.UploadStatus.PROCESS}),await(0,prepare_1.prepare)({file_id:t.id,fileRepository:this.fileRepository,createCloudRecord:this.baseParams.createCloudRecord});const{source_data:s,txt_data:i,img_data:e,media_data:a}=t,o=[...s.parts??[],...i.parts??[],...e.parts??[],...a.parts??[]].map((s=>({uid:this.baseParams.uid,file_id:t.id,info:s,filepath:t.path,output_root:t.output_root,size:s.size,ali_host_id:t.ali_host_id||null,baidu_host_id:t.baidu_host_id||null})));return void await this.partRepository.save(o)}finally{isDequeueing=!1}}}exports.BaseProcessor=BaseProcessor;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@soga/uploader",
-  "version": "0.1.4",
+  "version": "0.1.6",
   "publishConfig": {
     "access": "public"
   },
@@ -58,5 +58,5 @@
     "level": "*",
     "typeorm": "*"
   },
-  "gitHead": "00dc069c76994ff3763659209610666bc3de15fb"
+  "gitHead": "7692cde3d5e97cd466df5a989b1b5b3e42f3a6c0"
 }