@soga/uploader 0.2.22 → 0.2.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -1 +1 @@
|
|
|
1
|
-
"use strict";var __importDefault=this&&this.__importDefault||function(
|
|
1
|
+
"use strict";var __importDefault=this&&this.__importDefault||function(e){return e&&e.__esModule?e:{default:e}};Object.defineProperty(exports,"__esModule",{value:!0}),exports.getAliUploader=exports.AliUploader=void 0;const base_1=require("./base"),types_1=require("@soga/types"),piscina_1=__importDefault(require("piscina")),node_worker_threads_1=require("node:worker_threads"),piscina=new piscina_1.default({filename:require.resolve("@soga/part-uploader")}),uploaders=new Map;class AliUploader extends base_1.BaseUploader{params;constructor(e){super(e,types_1.HostType.ALI),this.params=e}getThread(e){const t=new AbortController,{port1:i,port2:a}=new node_worker_threads_1.MessageChannel;return{file_id:e.file_id,part_id:e.id,uid:e.uid,start:async()=>{try{const{file_id:r,host_id:s,id:o,info:l}=e,d=await this.fileRepository.findOneBy({id:r}),{output_root:p}=d;a.on("message",(async e=>{"percent"==e.type&&1==e.data.percent&&console.log(`ali_${r}-${o}=====================`),await this.onPartProgress(e)})),a.on("error",(e=>{console.log("ali upload part error evt: "),console.log(e)}));const n={file_id:r,host_id:s,part_id:o,output_root:p,part:l,cloud_folder_id:d.cloud_info.hosts.ali?.file_id,sdk_domain:this.params.sdk_domain,port:i},_=await piscina.run(n,{name:"uploadAli",signal:t.signal,transferList:[i]});_&&(await this.partRepository.update(e.id,{upload_result:{ali:_},upload_status:types_1.UploadStatus.SUCCESS}),console.log(`ali_${r}-${o}----------------------------------`),await this.onPartSuccess({file_id:r,part_id:o,host_type:types_1.HostType.ALI}))}catch(t){"AbortError"==t.name?this.partRepository.update(e.id,{upload_status:types_1.UploadStatus.NULL}):await this.onPartError(t,await this.fileRepository.findOneBy({id:e.file_id}))}finally{this.threads=this.threads.filter((t=>t.part_id!=e.id)),a.close(),await this.start()}},stop:async()=>{t.abort(),this.threads=this.threads.filter((t=>t.part_id!==e.id))}}}}exports.AliUploader=AliUploader;const getAliUploader=async e=>{const{uid:t,host_id:i}=e,a=`${t}_${i}`;let r=uploaders.get(a);if(r)for(;r.is_initing;)await new Promise((e=>setTimeout(e,20)));else r=new AliUploader(e),r.setThreads(3),uploaders.set(a,r),await r.init();return r};exports.getAliUploader=getAliUploader;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
"use strict";var __importDefault=this&&this.__importDefault||function(t){return t&&t.__esModule?t:{default:t}};Object.defineProperty(exports,"__esModule",{value:!0}),exports.getBaiduUploader=exports.BaiduUploader=void 0;const base_1=require("./base"),types_1=require("@soga/types"),piscina_1=__importDefault(require("piscina")),node_worker_threads_1=require("node:worker_threads"),piscina=new piscina_1.default({filename:require.resolve("@soga/part-uploader")}),uploaders=new Map;class BaiduUploader extends base_1.BaseUploader{params;constructor(t){super(t,types_1.HostType.BAIDU),this.params=t}getThread(t){const e=new AbortController,{port1:a,port2:i}=new node_worker_threads_1.MessageChannel;return{file_id:t.file_id,part_id:t.id,uid:t.uid,start:async()=>{try{const{file_id:
|
|
1
|
+
"use strict";var __importDefault=this&&this.__importDefault||function(t){return t&&t.__esModule?t:{default:t}};Object.defineProperty(exports,"__esModule",{value:!0}),exports.getBaiduUploader=exports.BaiduUploader=void 0;const base_1=require("./base"),types_1=require("@soga/types"),piscina_1=__importDefault(require("piscina")),node_worker_threads_1=require("node:worker_threads"),piscina=new piscina_1.default({filename:require.resolve("@soga/part-uploader")}),uploaders=new Map;class BaiduUploader extends base_1.BaseUploader{params;constructor(t){super(t,types_1.HostType.BAIDU),this.params=t}getThread(t){const e=new AbortController,{port1:a,port2:i}=new node_worker_threads_1.MessageChannel;return{file_id:t.file_id,part_id:t.id,uid:t.uid,start:async()=>{try{const{file_id:r,host_id:s,id:o,info:d}=t,p=await this.fileRepository.findOneBy({id:r}),{output_root:n}=p;i.on("message",(async t=>{await this.onPartProgress(t)})),i.on("error",(t=>{console.log("baidu upload part error evt: "),console.log(t)}));const u={file_id:r,host_id:s,part_id:o,output_root:n,part:d,cloud_folder_path:p.cloud_info.hosts.baidu?.path,sdk_domain:this.params.sdk_domain,port:a},l=await piscina.run(u,{name:"uploadBaidu",signal:e.signal,transferList:[a]});l&&(await this.partRepository.update(t.id,{upload_result:{baidu:l},upload_status:types_1.UploadStatus.SUCCESS}),await this.onPartSuccess({file_id:r,part_id:o,host_type:types_1.HostType.BAIDU}))}catch(e){"AbortError"==e.name?this.partRepository.update(t.id,{upload_status:types_1.UploadStatus.NULL}):await this.onPartError(e,await this.fileRepository.findOneBy({id:t.file_id}))}finally{this.threads=this.threads.filter((e=>e.part_id!=t.id)),i.close(),await this.start()}},stop:async()=>{e.abort(),this.threads=this.threads.filter((e=>e.part_id!==t.id))}}}}exports.BaiduUploader=BaiduUploader;const getBaiduUploader=async t=>{const{uid:e,host_id:a}=t,i=`${e}_${a}`;let r=uploaders.get(i);if(r)for(;r.is_initing;)await new Promise((t=>setTimeout(t,20)));else r=new BaiduUploader(t),r.setThreads(3),uploaders.set(i,r),await r.init();return r};exports.getBaiduUploader=getBaiduUploader;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.BaseUploader=void 0;const typeorm_1=require("typeorm"),entities_1=require("@soga/entities"),types_1=require("@soga/types"),prepare_1=require("../hooks/prepare"),complete_1=require("../hooks/complete"),utils_1=require("@soga/utils"),trasform_1=require("../hooks/trasform");let isDequeueing=!1;class BaseUploader{hostType;onProgress=async()=>{};onSuccess=async()=>{};onError=async()=>{};successRepository;fileRepository;partRepository;isRunning=!1;thread_count=1;maxThreads=10;threads=[];baseParams;getValidThreads(t){return Math.min(Math.max(t??0,0),this.maxThreads)}async onPartProgress(t){try{const{id:s,data:e,type:i}=t;if(this.processing_file_map.get(s))return;if(this.processing_file_map.set(s,!0),"percent"!=i)return;const{part_id:a,percent:o}=e;await this.partRepository.update(a,{upload_percent:o});const r=await this.partRepository.findBy({file_id:s});let p=0,d=0;for(const t of r)p+=t.info.size,d+=t.info.size*t.upload_percent;const n=d/p,u=await this.fileRepository.findOneBy({id:s}),
|
|
1
|
+
"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.BaseUploader=void 0;const typeorm_1=require("typeorm"),entities_1=require("@soga/entities"),types_1=require("@soga/types"),prepare_1=require("../hooks/prepare"),complete_1=require("../hooks/complete"),utils_1=require("@soga/utils"),trasform_1=require("../hooks/trasform");let isDequeueing=!1;class BaseUploader{hostType;onProgress=async()=>{};onSuccess=async()=>{};onError=async()=>{};successRepository;fileRepository;partRepository;isRunning=!1;thread_count=1;maxThreads=10;threads=[];baseParams;getValidThreads(t){return Math.min(Math.max(t??0,0),this.maxThreads)}async onPartProgress(t){try{const{id:s,data:e,type:i}=t;if(this.processing_file_map.get(s))return;if(this.processing_file_map.set(s,!0),"percent"!=i)return;const{part_id:a,percent:o}=e;await this.partRepository.update(a,{upload_percent:o});const r=await this.partRepository.findBy({file_id:s});let p=0,d=0;for(const t of r)p+=t.info.size,d+=t.info.size*t.upload_percent;const n=d/p,u=await this.fileRepository.findOneBy({id:s}),l=types_1.UploadProcessStep[`upload_${this.hostType}`];u.progress[l].percent=n,u.percent=(0,utils_1.getProgressPercent)(u.progress),await this.fileRepository.update(s,{progress:u.progress,percent:u.percent}),await this.onProgress(u)}finally{this.processing_file_map.delete(t.id)}}async moveToCompleted(t){await this.partRepository.delete({file_id:t}),await(0,trasform_1.transform)({file_id:t,dataSource:this.baseParams.dataSource}),await this.onSuccess(await this.fileRepository.findOneBy({id:t}))}async checkAllUploaded(t){const{ali_host_id:s,baidu_host_id:e,ali_upload_status:i,baidu_upload_status:a}=t;if(e){if(a==types_1.UploadStatus.NULL||a==types_1.UploadStatus.ERROR)return;if(a==types_1.UploadStatus.PROCESS){if(await this.partRepository.findOneBy({file_id:t.id,host_type:types_1.HostType.BAIDU,upload_status:(0,typeorm_1.Not)(types_1.UploadStatus.SUCCESS)}))return;await this.fileRepository.update(t.id,{baidu_upload_status:types_1.UploadStatus.SUCCESS})}}if(s){if(i==types_1.UploadStatus.NULL||i==types_1.UploadStatus.ERROR)return;if(i==types_1.UploadStatus.PROCESS){if(await this.partRepository.findOneBy({file_id:t.id,host_type:types_1.HostType.ALI,upload_status:(0,typeorm_1.Not)(types_1.UploadStatus.SUCCESS)}))return;await this.fileRepository.update(t.id,{ali_upload_status:types_1.UploadStatus.SUCCESS})}}const o={ali:{},baidu:{}},r=await this.partRepository.findBy({file_id:t.id});if(r.length){for(const t of r)t.host_type==types_1.HostType.BAIDU&&(o.baidu[t.info.file]={...t.upload_result.baidu,md4:t.info.md4}),t.host_type==types_1.HostType.ALI&&(o.ali[t.info.file]={...t.upload_result.ali,sha1:t.info.sha1});await this.fileRepository.update(t.id,{uploaded:o}),await(0,complete_1.complete)({file_id:t.id,sdk_domain:this.baseParams.sdk_domain,fileRepository:this.fileRepository}),await this.moveToCompleted(t.id)}}processing_file_map=new Map;async onPartSuccess({part_id:t,file_id:s,host_type:e}){try{for(;this.processing_file_map.get(s);)await new Promise((t=>setTimeout(t,50)));this.processing_file_map.set(s,!0);const i=await this.fileRepository.findOneBy({id:s}),a=`${e}_upload_status`;if(i[a]==types_1.UploadStatus.ERROR)return void this.processing_file_map.delete(i.id);if(await this.partRepository.findOneBy({id:(0,typeorm_1.Not)(t),file_id:i.id,host_type:e,upload_status:(0,typeorm_1.Not)(types_1.UploadStatus.SUCCESS)}))return void this.processing_file_map.delete(i.id);await this.fileRepository.update(i.id,{[a]:types_1.UploadStatus.SUCCESS,some_host_uploaded:!0});const o=await this.fileRepository.findOneBy({id:i.id});await this.checkAllUploaded(o)}finally{this.processing_file_map.delete(s)}}async onPartError(t,s){console.log("on part error: ",s.id),console.log(t);const e=`${this.hostType}_upload_status`;s[e]!=types_1.UploadStatus.ERROR&&(await this.fileRepository.update(s.id,{[e]:types_1.UploadStatus.ERROR}),await this.onError(t,s))}constructor(t,s){this.hostType=s,this.baseParams=t;const{dataSource:e}=t;this.fileRepository=e.getRepository(entities_1.UploadFile),this.successRepository=e.getRepository(entities_1.UploadSuccess),this.partRepository=e.getRepository(entities_1.UploadPart),this.thread_count=this.getValidThreads(t.threads??this.thread_count),t.onProgress&&(this.onProgress=t.onProgress.bind(this)),t.onSuccess&&(this.onSuccess=t.onSuccess.bind(this)),t.onError&&(this.onError=t.onError.bind(this))}async setThreads(t){const s=this.getValidThreads(t);this.thread_count=s,await this.run()}is_initing=!1;async init(){this.is_initing=!0,await this.partRepository.update({uid:this.baseParams.uid,upload_status:types_1.UploadStatus.PROCESS},{upload_status:types_1.UploadStatus.NULL});const t=await this.fileRepository.findBy({uid:this.baseParams.uid,some_host_uploaded:!0,upload_status:(0,typeorm_1.Not)(types_1.UploadStatus.SUCCESS)});console.log("files:"),console.log(t.map((t=>t.id)));for(const s of t)await this.checkAllUploaded(s);this.is_initing=!1}async start(){await this.run()}async stopFiles(t,s=!1){s&&(await this.partRepository.update({file_id:(0,typeorm_1.In)(t),is_paused:!1},{is_paused:!0}),await this.fileRepository.update({id:(0,typeorm_1.In)(t),is_paused:!1},{is_paused:!0}));const e=this.threads.filter((s=>t.includes(s.file_id)));await Promise.all(e.map((t=>t.stop()))),await this.start()}async stopAll(t=!1){t&&(await this.partRepository.update({uid:this.baseParams.uid},{is_paused:!0}),await this.fileRepository.update({uid:this.baseParams.uid,type:(0,typeorm_1.Not)(types_1.RecordType.FOLDER)},{is_paused:!0})),await Promise.all(this.threads.map((t=>t.stop())))}async run(){for(;this.isRunning;)await new Promise((t=>{setTimeout(t,200)}));this.isRunning=!0;const t=this.threads.length,s=this.thread_count;if(t<s){const e=s-t,i=async()=>await this.partRepository.findOne({where:{uid:this.baseParams.uid,is_paused:!1,upload_status:types_1.UploadStatus.NULL,host_id:this.baseParams.host_id},order:{created_at:"ASC"}});for(let t=0;t<e;t++){const t=await i();if(!t)break;const s=this.getThread(t);this.threads.push(s),await this.partRepository.update(t.id,{upload_status:types_1.UploadStatus.PROCESS}),s.start()}if(this.threads.length<s){await this.dequeueOneFile();await i()&&(this.isRunning=!1,await this.run())}}else if(t>s){const e=t-s,i=this.threads.slice(0,e);for(const t of i)await t.stop()}this.isRunning=!1}async dequeueOneFile(){for(;isDequeueing;)await new Promise((t=>{setTimeout(t,50)}));isDequeueing=!0;try{const t=`${this.hostType}_upload_status`,s=`${this.hostType}_host_id`,e=await this.fileRepository.findOne({where:{uid:this.baseParams.uid,is_paused:!1,encode_status:types_1.EncodeStatus.SUCCESS,[t]:(0,typeorm_1.IsNull)(),upload_status:(0,typeorm_1.In)([types_1.UploadStatus.NULL,types_1.UploadStatus.PROCESS])},order:{created_at:"ASC"}});if(!e)return;await this.fileRepository.update(e.id,{[t]:types_1.UploadStatus.PROCESS,upload_status:types_1.UploadStatus.PROCESS}),await(0,prepare_1.prepare)({file_id:e.id,sdk_domain:this.baseParams.sdk_domain,fileRepository:this.fileRepository});const{encoded:i}=e,{affix:a,source:o,txt:r,img:p,media:d}=i,n=[...o?.parts??[],...r?.parts??[],...p?.parts??[],...d?.parts??[]];a?.forEach((t=>{t.parts?.forEach((t=>{n.push(t)}))}));const u=e[s],l=n.map((t=>({uid:this.baseParams.uid,file_id:e.id,info:t,output_root:e.output_root,host_id:u,host_type:this.hostType})));return void await this.partRepository.save(l)}finally{isDequeueing=!1}}}exports.BaseUploader=BaseUploader;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@soga/uploader",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.23",
|
|
4
4
|
"publishConfig": {
|
|
5
5
|
"access": "public"
|
|
6
6
|
},
|
|
@@ -48,11 +48,11 @@
|
|
|
48
48
|
},
|
|
49
49
|
"dependencies": {
|
|
50
50
|
"@soga/entities": "^0.2.22",
|
|
51
|
-
"@soga/part-uploader": "^0.2.
|
|
51
|
+
"@soga/part-uploader": "^0.2.23",
|
|
52
52
|
"@soga/sdk": "^0.2.22",
|
|
53
53
|
"@soga/types": "^0.2.22",
|
|
54
54
|
"@soga/utils": "^0.2.22",
|
|
55
55
|
"piscina": "^4.9.2"
|
|
56
56
|
},
|
|
57
|
-
"gitHead": "
|
|
57
|
+
"gitHead": "f232aca1cb2b1e9c86d3142c2845d11c87b065ba"
|
|
58
58
|
}
|