@soga/uploader 0.2.6 → 0.2.7

package/dist/host-uploader/ali.js

@@ -1 +1 @@
-"use strict";var __importDefault=this&&this.__importDefault||function(i){return i&&i.__esModule?i:{default:i}};Object.defineProperty(exports,"__esModule",{value:!0}),exports.getAliUploader=exports.AliUploader=void 0;const base_1=require("./base"),types_1=require("@soga/types"),piscina_1=__importDefault(require("piscina")),piscina=new piscina_1.default({filename:require.resolve("@soga/part-uploader")}),uploaders=new Map;class AliUploader extends base_1.BaseUploader{params;constructor(i){super(i,types_1.HostType.ALI),this.params=i}getThread(i){return{file_id:i.file_id,part_id:i.id,uid:i.uid,start:async()=>{try{const t=new AbortController,{file_id:e,host_id:a,id:s,info:r}=i,o=await this.fileRepository.findOneBy({id:e}),{output_root:d}=o;piscina.on("message",(async i=>{await this.onPartProgress(i)}));const l={file_id:e,host_id:a,part_id:s,output_root:d,part:r,cloud_folder_id:o.cloud_info.hosts.ali?.file_id,sdk_domain:this.params.sdk_domain},p=await piscina.run(l,{name:"uploadAli",signal:t.signal});if(p){await this.partRepository.update(i.id,{upload_result:{ali:p},upload_status:types_1.UploadStatus.SUCCESS});const t=await this.partRepository.findOneBy({id:s}),a=await this.fileRepository.findOneBy({id:e});await this.onPartSuccess(t,a)}}catch(t){await this.onPartError(t,await this.fileRepository.findOneBy({id:i.file_id}))}finally{this.threads=this.threads.filter((t=>t.part_id!=i.id)),await this.start()}},stop:async()=>{this.threads=this.threads.filter((t=>t.part_id!==i.id))}}}}exports.AliUploader=AliUploader;const getAliUploader=async i=>{const{uid:t,host_id:e}=i,a=`${t}_${e}`;let s=uploaders.get(a);if(s)for(;s.is_initing;)await new Promise((i=>setTimeout(i,20)));else s=new AliUploader(i),s.setThreads(3),uploaders.set(a,s),await s.init();return s};exports.getAliUploader=getAliUploader;
+"use strict";var __importDefault=this&&this.__importDefault||function(t){return t&&t.__esModule?t:{default:t}};Object.defineProperty(exports,"__esModule",{value:!0}),exports.getAliUploader=exports.AliUploader=void 0;const base_1=require("./base"),types_1=require("@soga/types"),piscina_1=__importDefault(require("piscina")),piscina=new piscina_1.default({filename:require.resolve("@soga/part-uploader")}),uploaders=new Map;class AliUploader extends base_1.BaseUploader{params;constructor(t){super(t,types_1.HostType.ALI),this.params=t}getThread(t){const i=new AbortController;return{file_id:t.file_id,part_id:t.id,uid:t.uid,start:async()=>{try{const{file_id:e,host_id:a,id:s,info:r}=t,o=await this.fileRepository.findOneBy({id:e}),{output_root:d}=o;piscina.on("message",(async t=>{await this.onPartProgress(t)}));const l={file_id:e,host_id:a,part_id:s,output_root:d,part:r,cloud_folder_id:o.cloud_info.hosts.ali?.file_id,sdk_domain:this.params.sdk_domain},p=await piscina.run(l,{name:"uploadAli",signal:i.signal});if(p){await this.partRepository.update(t.id,{upload_result:{ali:p},upload_status:types_1.UploadStatus.SUCCESS});const i=await this.partRepository.findOneBy({id:s}),a=await this.fileRepository.findOneBy({id:e});await this.onPartSuccess(i,a)}}catch(i){"AbortError"==i.name?this.partRepository.update(t.id,{upload_status:types_1.UploadStatus.NULL}):await this.onPartError(i,await this.fileRepository.findOneBy({id:t.file_id}))}finally{this.threads=this.threads.filter((i=>i.part_id!=t.id)),await this.start()}},stop:async()=>{i.abort(),this.threads=this.threads.filter((i=>i.part_id!==t.id))}}}}exports.AliUploader=AliUploader;const getAliUploader=async t=>{const{uid:i,host_id:e}=t,a=`${i}_${e}`;let s=uploaders.get(a);if(s)for(;s.is_initing;)await new Promise((t=>setTimeout(t,20)));else s=new AliUploader(t),s.setThreads(3),uploaders.set(a,s),await s.init();return s};exports.getAliUploader=getAliUploader;

package/dist/host-uploader/baidu.js

@@ -1 +1 @@
-"use strict";var __importDefault=this&&this.__importDefault||function(t){return t&&t.__esModule?t:{default:t}};Object.defineProperty(exports,"__esModule",{value:!0}),exports.getBaiduUploader=exports.BaiduUploader=void 0;const base_1=require("./base"),types_1=require("@soga/types"),piscina_1=__importDefault(require("piscina")),piscina=new piscina_1.default({filename:require.resolve("@soga/part-uploader")}),uploaders=new Map;class BaiduUploader extends base_1.BaseUploader{params;constructor(t){super(t,types_1.HostType.BAIDU),this.params=t}getThread(t){return{file_id:t.file_id,part_id:t.id,uid:t.uid,start:async()=>{try{const i=new AbortController,{file_id:a,host_id:e,id:s,info:r}=t,o=await this.fileRepository.findOneBy({id:a}),{output_root:d}=o;piscina.on("message",(async t=>{await this.onPartProgress(t)}));const p={file_id:a,host_id:e,part_id:s,output_root:d,part:r,cloud_folder_path:o.cloud_info.hosts.baidu?.path,sdk_domain:this.params.sdk_domain},n=await piscina.run(p,{name:"uploadBaidu",signal:i.signal});if(n){await this.partRepository.update(t.id,{upload_result:{baidu:n},upload_status:types_1.UploadStatus.SUCCESS});const i=await this.partRepository.findOneBy({id:s}),e=await this.fileRepository.findOneBy({id:a});await this.onPartSuccess(i,e)}}catch(i){await this.onPartError(i,await this.fileRepository.findOneBy({id:t.file_id}))}finally{this.threads=this.threads.filter((i=>i.part_id!=t.id)),await this.start()}},stop:async()=>{this.threads=this.threads.filter((i=>i.part_id!==t.id))}}}}exports.BaiduUploader=BaiduUploader;const getBaiduUploader=async t=>{const{uid:i,host_id:a}=t,e=`${i}_${a}`;let s=uploaders.get(e);if(s)for(;s.is_initing;)await new Promise((t=>setTimeout(t,20)));else s=new BaiduUploader(t),s.setThreads(3),uploaders.set(e,s),await s.init();return s};exports.getBaiduUploader=getBaiduUploader;
+"use strict";var __importDefault=this&&this.__importDefault||function(t){return t&&t.__esModule?t:{default:t}};Object.defineProperty(exports,"__esModule",{value:!0}),exports.getBaiduUploader=exports.BaiduUploader=void 0;const base_1=require("./base"),types_1=require("@soga/types"),piscina_1=__importDefault(require("piscina")),piscina=new piscina_1.default({filename:require.resolve("@soga/part-uploader")}),uploaders=new Map;class BaiduUploader extends base_1.BaseUploader{params;constructor(t){super(t,types_1.HostType.BAIDU),this.params=t}getThread(t){const i=new AbortController;return{file_id:t.file_id,part_id:t.id,uid:t.uid,start:async()=>{try{const{file_id:a,host_id:e,id:s,info:r}=t,o=await this.fileRepository.findOneBy({id:a}),{output_root:d}=o;piscina.on("message",(async t=>{await this.onPartProgress(t)}));const p={file_id:a,host_id:e,part_id:s,output_root:d,part:r,cloud_folder_path:o.cloud_info.hosts.baidu?.path,sdk_domain:this.params.sdk_domain},u=await piscina.run(p,{name:"uploadBaidu",signal:i.signal});if(u){await this.partRepository.update(t.id,{upload_result:{baidu:u},upload_status:types_1.UploadStatus.SUCCESS});const i=await this.partRepository.findOneBy({id:s}),e=await this.fileRepository.findOneBy({id:a});await this.onPartSuccess(i,e)}}catch(i){"AbortError"==i.name?this.partRepository.update(t.id,{upload_status:types_1.UploadStatus.NULL}):await this.onPartError(i,await this.fileRepository.findOneBy({id:t.file_id}))}finally{this.threads=this.threads.filter((i=>i.part_id!=t.id)),await this.start()}},stop:async()=>{i.abort(),this.threads=this.threads.filter((i=>i.part_id!==t.id))}}}}exports.BaiduUploader=BaiduUploader;const getBaiduUploader=async t=>{const{uid:i,host_id:a}=t,e=`${i}_${a}`;let s=uploaders.get(e);if(s)for(;s.is_initing;)await new Promise((t=>setTimeout(t,20)));else s=new BaiduUploader(t),s.setThreads(3),uploaders.set(e,s),await s.init();return s};exports.getBaiduUploader=getBaiduUploader;

package/dist/host-uploader/base.js

@@ -1 +1 @@
-"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.BaseUploader=void 0;const typeorm_1=require("typeorm"),entities_1=require("@soga/entities"),types_1=require("@soga/types"),prepare_1=require("../hooks/prepare"),complete_1=require("../hooks/complete"),utils_1=require("@soga/utils");let isDequeueing=!1;class BaseUploader{hostType;onProgress=async()=>{};onSuccess=async()=>{};onError=async()=>{};fileRepository;partRepository;isRunning=!1;thread_count=1;maxThreads=10;threads=[];baseParams;getValidThreads(t){return Math.min(Math.max(t??0,0),this.maxThreads)}async onPartProgress(t){const{id:s,data:e,type:i}=t;if("percent"!=i)return;const{part_id:a,percent:o}=e;await this.partRepository.update(a,{upload_percent:o});const r=await this.partRepository.findBy({file_id:s});let p=0,d=0;for(const t of r)p+=t.info.size,d+=t.info.size*t.upload_percent;const n=d/p,u=await this.fileRepository.findOneBy({id:s}),h=types_1.UploadProcessStep[`upload_${this.hostType}`];u.progress[h].percent=n,u.percent=(0,utils_1.getProgressPercent)(u.progress),await this.fileRepository.update(s,{progress:u.progress,percent:u.percent}),await this.onProgress(u)}async onSomePartSuccess(t){const{ali_host_id:s,baidu_host_id:e,ali_upload_status:i,baidu_upload_status:a}=t;if(e&&a!=types_1.UploadStatus.SUCCESS)return;if(s&&i!=types_1.UploadStatus.SUCCESS)return;const o={ali:{},baidu:{}},r=await this.partRepository.findBy({file_id:t.id});if(!r.length)return;for(const t of r)t.host_type==types_1.HostType.BAIDU&&(o.baidu[t.info.file]=t.upload_result.baidu),t.host_type==types_1.HostType.ALI&&(o.ali[t.info.file]=t.upload_result.ali);await this.fileRepository.update(t.id,{uploaded:o}),await(0,complete_1.complete)({file_id:t.id,sdk_domain:this.baseParams.sdk_domain,fileRepository:this.fileRepository});const p=await this.fileRepository.findOneBy({id:t.id});p.progress.END.percent=1,p.percent=1,await this.fileRepository.update(t.id,{progress:p.progress,percent:p.percent,upload_status:types_1.UploadStatus.SUCCESS}),await this.partRepository.delete({file_id:t.id}),await this.onSuccess(await this.fileRepository.findOneBy({id:t.id}))}async onPartSuccess(t,s){const{host_type:e}=t,i=`${e}_upload_status`;if(s[i]==types_1.UploadStatus.ERROR)return;if(await this.partRepository.findOneBy({id:(0,typeorm_1.Not)(t.id),file_id:s.id,host_type:e,upload_status:(0,typeorm_1.Not)(types_1.UploadStatus.SUCCESS)}))return;await this.fileRepository.update(s.id,{[i]:types_1.UploadStatus.SUCCESS,upload_status:types_1.UploadStatus.SOME_PART_SUCCESS});const a=await this.fileRepository.findOneBy({id:s.id});await this.onSomePartSuccess(a)}async onPartError(t,s){const e=`${this.hostType}_upload_status`;s[e]!=types_1.UploadStatus.ERROR&&(await this.fileRepository.update(s.id,{[e]:types_1.UploadStatus.ERROR}),await this.onError(t,s))}constructor(t,s){this.hostType=s,this.baseParams=t;const{dataSource:e}=t;this.fileRepository=e.getRepository(entities_1.UploadFile),this.partRepository=e.getRepository(entities_1.UploadPart),this.thread_count=this.getValidThreads(t.threads??this.thread_count),t.onProgress&&(this.onProgress=t.onProgress.bind(this)),t.onSuccess&&(this.onSuccess=t.onSuccess.bind(this)),t.onError&&(this.onError=t.onError.bind(this))}async setThreads(t){const s=this.getValidThreads(t);this.thread_count=s,await this.run()}is_initing=!1;async init(){this.is_initing=!0,await this.partRepository.update({uid:this.baseParams.uid,upload_status:types_1.UploadStatus.PROCESS},{upload_status:types_1.UploadStatus.NULL});const t=await this.fileRepository.findBy({uid:this.baseParams.uid,upload_status:types_1.UploadStatus.SOME_PART_SUCCESS});for(const s of t)await this.onSomePartSuccess(s);this.is_initing=!1}async start(){await this.run()}async stopFiles(t,s=!1){s&&(await this.partRepository.update({file_id:(0,typeorm_1.In)(t),is_paused:!1},{is_paused:!0}),await this.fileRepository.update({id:(0,typeorm_1.In)(t),is_paused:!1},{is_paused:!0}));const e=this.threads.filter((s=>t.includes(s.file_id)));await Promise.all(e.map((t=>t.stop()))),await this.start()}async stopAll(t=!1){t&&(await this.partRepository.update({uid:this.baseParams.uid},{is_paused:!0}),await this.fileRepository.update({uid:this.baseParams.uid,type:(0,typeorm_1.Not)(types_1.RecordType.FOLDER)},{is_paused:!0})),await Promise.all(this.threads.map((t=>t.stop())))}async run(){for(;this.isRunning;)await new Promise((t=>{setTimeout(t,200)}));this.isRunning=!0;const t=this.threads.length,s=this.thread_count;if(t<s){const e=s-t,i=async()=>await this.partRepository.findOne({where:{uid:this.baseParams.uid,is_paused:!1,upload_status:types_1.UploadStatus.NULL,host_id:this.baseParams.host_id},order:{created_at:"ASC"}});for(let t=0;t<e;t++){const t=await i();if(!t)break;const s=this.getThread(t);this.threads.push(s),await this.partRepository.update(t.id,{upload_status:types_1.UploadStatus.PROCESS}),s.start()}if(this.threads.length<s){await this.dequeueOneFile();await i()&&(this.isRunning=!1,await this.run())}}else if(t>s){const e=t-s,i=this.threads.slice(0,e);for(const t of i)await t.stop()}this.isRunning=!1}async dequeueOneFile(){for(;isDequeueing;)await new Promise((t=>{setTimeout(t,50)}));isDequeueing=!0;try{const t=`${this.hostType}_upload_status`,s=`${this.hostType}_host_id`,e=await this.fileRepository.findOne({where:{uid:this.baseParams.uid,is_paused:!1,encode_status:types_1.EncodeStatus.SUCCESS,[t]:(0,typeorm_1.IsNull)(),upload_status:(0,typeorm_1.In)([types_1.UploadStatus.NULL,types_1.UploadStatus.PROCESS])},order:{created_at:"ASC"}});if(!e)return;await this.fileRepository.update(e.id,{[t]:types_1.UploadStatus.PROCESS,upload_status:types_1.UploadStatus.PROCESS}),await(0,prepare_1.prepare)({file_id:e.id,sdk_domain:this.baseParams.sdk_domain,fileRepository:this.fileRepository});const{encoded:i}=e,{affix:a,source:o,txt:r,img:p,media:d}=i,n=[...o?.parts??[],...r?.parts??[],...p?.parts??[],...d?.parts??[]];a?.forEach((t=>{t.parts?.forEach((t=>{n.push(t)}))}));const u=e[s],h=n.map((t=>({uid:this.baseParams.uid,file_id:e.id,info:t,output_root:e.output_root,host_id:u,host_type:this.hostType})));return void await this.partRepository.save(h)}finally{isDequeueing=!1}}}exports.BaseUploader=BaseUploader;
+"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.BaseUploader=void 0;const typeorm_1=require("typeorm"),entities_1=require("@soga/entities"),types_1=require("@soga/types"),prepare_1=require("../hooks/prepare"),complete_1=require("../hooks/complete"),utils_1=require("@soga/utils");let isDequeueing=!1;class BaseUploader{hostType;onProgress=async()=>{};onSuccess=async()=>{};onError=async()=>{};fileRepository;partRepository;isRunning=!1;thread_count=1;maxThreads=10;threads=[];baseParams;getValidThreads(t){return Math.min(Math.max(t??0,0),this.maxThreads)}async onPartProgress(t){const{id:s,data:e,type:i}=t;if("percent"!=i)return;const{part_id:a,percent:o}=e;await this.partRepository.update(a,{upload_percent:o});const r=await this.partRepository.findBy({file_id:s});let p=0,d=0;for(const t of r)p+=t.info.size,d+=t.info.size*t.upload_percent;const n=d/p,u=await this.fileRepository.findOneBy({id:s}),h=types_1.UploadProcessStep[`upload_${this.hostType}`];u.progress[h].percent=n,u.percent=(0,utils_1.getProgressPercent)(u.progress),await this.fileRepository.update(s,{progress:u.progress,percent:u.percent}),await this.onProgress(u)}async onSomePartSuccess(t){const{ali_host_id:s,baidu_host_id:e,ali_upload_status:i,baidu_upload_status:a}=t;if(e&&a!=types_1.UploadStatus.SUCCESS)return;if(s&&i!=types_1.UploadStatus.SUCCESS)return;const o={ali:{},baidu:{}},r=await this.partRepository.findBy({file_id:t.id});if(!r.length)return;for(const t of r)t.host_type==types_1.HostType.BAIDU&&(o.baidu[t.info.file]=t.upload_result.baidu),t.host_type==types_1.HostType.ALI&&(o.ali[t.info.file]=t.upload_result.ali);await this.fileRepository.update(t.id,{uploaded:o}),await(0,complete_1.complete)({file_id:t.id,sdk_domain:this.baseParams.sdk_domain,fileRepository:this.fileRepository});const p=await this.fileRepository.findOneBy({id:t.id});p.progress.END.percent=1,p.percent=1,await this.fileRepository.update(t.id,{progress:p.progress,percent:p.percent,upload_status:types_1.UploadStatus.SUCCESS}),await this.partRepository.delete({file_id:t.id}),await this.onSuccess(await this.fileRepository.findOneBy({id:t.id}))}async onPartSuccess(t,s){const{host_type:e}=t,i=`${e}_upload_status`;if(s[i]==types_1.UploadStatus.ERROR)return;if(await this.partRepository.findOneBy({id:(0,typeorm_1.Not)(t.id),file_id:s.id,host_type:e,upload_status:(0,typeorm_1.Not)(types_1.UploadStatus.SUCCESS)}))return;await this.fileRepository.update(s.id,{[i]:types_1.UploadStatus.SUCCESS,some_parts_uploaded:!0});const a=await this.fileRepository.findOneBy({id:s.id});await this.onSomePartSuccess(a)}async onPartError(t,s){const e=`${this.hostType}_upload_status`;s[e]!=types_1.UploadStatus.ERROR&&(await this.fileRepository.update(s.id,{[e]:types_1.UploadStatus.ERROR}),await this.onError(t,s))}constructor(t,s){this.hostType=s,this.baseParams=t;const{dataSource:e}=t;this.fileRepository=e.getRepository(entities_1.UploadFile),this.partRepository=e.getRepository(entities_1.UploadPart),this.thread_count=this.getValidThreads(t.threads??this.thread_count),t.onProgress&&(this.onProgress=t.onProgress.bind(this)),t.onSuccess&&(this.onSuccess=t.onSuccess.bind(this)),t.onError&&(this.onError=t.onError.bind(this))}async setThreads(t){const s=this.getValidThreads(t);this.thread_count=s,await this.run()}is_initing=!1;async init(){this.is_initing=!0,await this.partRepository.update({uid:this.baseParams.uid,upload_status:types_1.UploadStatus.PROCESS},{upload_status:types_1.UploadStatus.NULL});const t=await this.fileRepository.findBy({uid:this.baseParams.uid,some_parts_uploaded:!0});for(const s of t)await this.onSomePartSuccess(s);this.is_initing=!1}async start(){await this.run()}async stopFiles(t,s=!1){s&&(await this.partRepository.update({file_id:(0,typeorm_1.In)(t),is_paused:!1},{is_paused:!0}),await this.fileRepository.update({id:(0,typeorm_1.In)(t),is_paused:!1},{is_paused:!0}));const e=this.threads.filter((s=>t.includes(s.file_id)));await Promise.all(e.map((t=>t.stop()))),await this.start()}async stopAll(t=!1){t&&(await this.partRepository.update({uid:this.baseParams.uid},{is_paused:!0}),await this.fileRepository.update({uid:this.baseParams.uid,type:(0,typeorm_1.Not)(types_1.RecordType.FOLDER)},{is_paused:!0})),await Promise.all(this.threads.map((t=>t.stop())))}async run(){for(;this.isRunning;)await new Promise((t=>{setTimeout(t,200)}));this.isRunning=!0;const t=this.threads.length,s=this.thread_count;if(t<s){const e=s-t,i=async()=>await this.partRepository.findOne({where:{uid:this.baseParams.uid,is_paused:!1,upload_status:types_1.UploadStatus.NULL,host_id:this.baseParams.host_id},order:{created_at:"ASC"}});for(let t=0;t<e;t++){const t=await i();if(!t)break;const s=this.getThread(t);this.threads.push(s),await this.partRepository.update(t.id,{upload_status:types_1.UploadStatus.PROCESS}),s.start()}if(this.threads.length<s){await this.dequeueOneFile();await i()&&(this.isRunning=!1,await this.run())}}else if(t>s){const e=t-s,i=this.threads.slice(0,e);for(const t of i)await t.stop()}this.isRunning=!1}async dequeueOneFile(){for(;isDequeueing;)await new Promise((t=>{setTimeout(t,50)}));isDequeueing=!0;try{const t=`${this.hostType}_upload_status`,s=`${this.hostType}_host_id`,e=await this.fileRepository.findOne({where:{uid:this.baseParams.uid,is_paused:!1,encode_status:types_1.EncodeStatus.SUCCESS,[t]:(0,typeorm_1.IsNull)(),upload_status:(0,typeorm_1.In)([types_1.UploadStatus.NULL,types_1.UploadStatus.PROCESS])},order:{created_at:"ASC"}});if(!e)return;await this.fileRepository.update(e.id,{[t]:types_1.UploadStatus.PROCESS,upload_status:types_1.UploadStatus.PROCESS}),await(0,prepare_1.prepare)({file_id:e.id,sdk_domain:this.baseParams.sdk_domain,fileRepository:this.fileRepository});const{encoded:i}=e,{affix:a,source:o,txt:r,img:p,media:d}=i,n=[...o?.parts??[],...r?.parts??[],...p?.parts??[],...d?.parts??[]];a?.forEach((t=>{t.parts?.forEach((t=>{n.push(t)}))}));const u=e[s],h=n.map((t=>({uid:this.baseParams.uid,file_id:e.id,info:t,output_root:e.output_root,host_id:u,host_type:this.hostType})));return void await this.partRepository.save(h)}finally{isDequeueing=!1}}}exports.BaseUploader=BaseUploader;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@soga/uploader",
-  "version": "0.2.6",
+  "version": "0.2.7",
   "publishConfig": {
     "access": "public"
   },
@@ -54,12 +54,12 @@
     "typeorm": "*"
   },
   "dependencies": {
-    "@soga/entities": "^0.2.6",
+    "@soga/entities": "^0.2.7",
     "@soga/part-uploader": "^0.2.1",
-    "@soga/sdk": "^0.2.1",
-    "@soga/types": "^0.2.1",
+    "@soga/sdk": "^0.2.7",
+    "@soga/types": "^0.2.7",
     "@soga/utils": "^0.2.1",
     "piscina": "^4.9.2"
   },
-  "gitHead": "67c0fe7916946b87e7f9743e412441470856eb3a"
+  "gitHead": "527e923683c11ace899c5111d7dcfb38115d4d48"
 }