@soga/uploader 0.0.20 → 0.0.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/main.d.ts
CHANGED
|
@@ -2,3 +2,4 @@ import { UploaderParams } from './types/main';
|
|
|
2
2
|
import { Uploader } from './uploader/uploader';
|
|
3
3
|
export declare const getUploader: (params: UploaderParams) => Promise<Uploader>;
|
|
4
4
|
export declare const getUploaderByUid: (uid: number) => Promise<Uploader>;
|
|
5
|
+
export { UploaderParams } from './types/main';
|
package/dist/processor/ali.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.getAliProcessor=exports.AliProcessor=void 0;const single_uploader_1=require("@soga/single-uploader"),base_1=require("./base"),types_1=require("@soga/types"),processors=new Map;class AliProcessor extends base_1.BaseProcessor{hostType=types_1.HostType.ALI;params;constructor(s){super(s),this.params=s}getThread(s){let i,
|
|
1
|
+
"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.getAliProcessor=exports.AliProcessor=void 0;const single_uploader_1=require("@soga/single-uploader"),base_1=require("./base"),types_1=require("@soga/types"),processors=new Map;class AliProcessor extends base_1.BaseProcessor{hostType=types_1.HostType.ALI;params;constructor(s){super(s),this.params=s}getThread(s){let i,t=null;return{file_id:s.file_id,part_id:s.id,uid:s.uid,start:async()=>{try{const{filepath:e,output_root:r,info:o}=s,a=await this.fileRepository.findOneBy({id:s.file_id}),d={file_id:s.file_id,filePath:e,part:o,uid:s.uid,part_id:s.id,outputRoot:r,drive_id:a.cloud_info.hosts.ali.drive_id,cloud_parent_id:a.cloud_info.hosts.ali.file_id,getAccessToken:async()=>await this.params.getAccessToken(a.ali_host_id),onProgress:async s=>{await this.onPartProgress(s)}};i=(0,single_uploader_1.getAliUploader)(d),t=await i.start(),t&&(await this.partRepository.update(s.id,{ali_result:t,ali_status:types_1.UploadStatus.SUCCESS}),await this.onPartSuccess(await this.fileRepository.findOneBy({id:s.file_id})))}catch(i){await this.onPartError(i,await this.fileRepository.findOneBy({id:s.file_id}))}finally{t&&(this.threads=this.threads.filter((i=>i.part_id!==s.id))),await this.start()}},stop:async()=>{i&&(await i.stop(),this.threads=this.threads.filter((i=>i.part_id!==s.id)))}}}}exports.AliProcessor=AliProcessor;const getAliProcessor=async s=>{const{uid:i,host_id:t}=s,e=`${i}_${t}`;let r=processors.get(e);if(r)for(;r.is_initing;)await new Promise((s=>setTimeout(s,20)));else r=new AliProcessor(s),r.setThreads(3),processors.set(e,r),await r.init();return r};exports.getAliProcessor=getAliProcessor;
|
package/dist/processor/baidu.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.getBaiduProcessor=exports.BaiduProcessor=void 0;const single_uploader_1=require("@soga/single-uploader"),base_1=require("./base"),types_1=require("@soga/types"),processors=new Map;class BaiduProcessor extends base_1.BaseProcessor{hostType=types_1.HostType.BAIDU;params;constructor(s){super(s),this.params=s}getThread(s){let t,e=null;return{file_id:s.file_id,part_id:s.id,uid:s.uid,start:async()=>{try{const{filepath:i,output_root:a,info:r}=s,o=await this.fileRepository.findOneBy({id:s.file_id}),d={file_id:s.file_id,filePath:i,part:r,uid:s.uid,part_id:s.id,outputRoot:a,cloudRoot:o.cloud_info.hosts.baidu.path,getAccessToken:async()=>await this.params.getAccessToken(o.baidu_host_id),onProgress:async s=>{await this.onPartProgress(s)}};t=(0,single_uploader_1.getBaiduUploader)(d),e=await t.start(),e&&(await this.partRepository.update(s.id,{baidu_result:e,baidu_status:types_1.UploadStatus.SUCCESS}),await this.onPartSuccess(await this.fileRepository.findOneBy({id:s.file_id})))}catch(t){await this.onPartError(t,await this.fileRepository.findOneBy({id:s.file_id}))}finally{e&&(this.threads=this.threads.filter((t=>t.part_id!==s.id))),await this.start()}},stop:async()=>{t&&(await t.stop(),this.threads=this.threads.filter((t=>t.part_id!==s.id)))}}}}exports.BaiduProcessor=BaiduProcessor;const getBaiduProcessor=async s=>{const{uid:t,
|
|
1
|
+
"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.getBaiduProcessor=exports.BaiduProcessor=void 0;const single_uploader_1=require("@soga/single-uploader"),base_1=require("./base"),types_1=require("@soga/types"),processors=new Map;class BaiduProcessor extends base_1.BaseProcessor{hostType=types_1.HostType.BAIDU;params;constructor(s){super(s),this.params=s}getThread(s){let t,e=null;return{file_id:s.file_id,part_id:s.id,uid:s.uid,start:async()=>{try{const{filepath:i,output_root:a,info:r}=s,o=await this.fileRepository.findOneBy({id:s.file_id}),d={file_id:s.file_id,filePath:i,part:r,uid:s.uid,part_id:s.id,outputRoot:a,cloudRoot:o.cloud_info.hosts.baidu.path,getAccessToken:async()=>await this.params.getAccessToken(o.baidu_host_id),onProgress:async s=>{await this.onPartProgress(s)}};t=(0,single_uploader_1.getBaiduUploader)(d),e=await t.start(),e&&(await this.partRepository.update(s.id,{baidu_result:e,baidu_status:types_1.UploadStatus.SUCCESS}),await this.onPartSuccess(await this.fileRepository.findOneBy({id:s.file_id})))}catch(t){await this.onPartError(t,await this.fileRepository.findOneBy({id:s.file_id}))}finally{e&&(this.threads=this.threads.filter((t=>t.part_id!==s.id))),await this.start()}},stop:async()=>{t&&(await t.stop(),this.threads=this.threads.filter((t=>t.part_id!==s.id)))}}}}exports.BaiduProcessor=BaiduProcessor;const getBaiduProcessor=async s=>{const{uid:t,host_id:e}=s,i=`${t}_${e}`;let a=processors.get(i);if(a)for(;a.is_initing;)await new Promise((s=>setTimeout(s,20)));else a=new BaiduProcessor(s),a.setThreads(5),processors.set(i,a),await a.init();return a};exports.getBaiduProcessor=getBaiduProcessor;
|
package/dist/processor/base.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.BaseProcessor=void 0;const typeorm_1=require("typeorm"),entities_1=require("@soga/entities"),types_1=require("@soga/types"),prepare_1=require("../prepare/prepare");let isDequeueing=!1;class BaseProcessor{hostType;onProgress=async()=>{};onSuccess=async()=>{};onError=async()=>{};fileRepository;partRepository;isRunning=!1;thread_count=3;maxThreads=10;threads=[];baseParams;getValidThreads(t){return Math.min(Math.max(t??0,0),this.maxThreads)}async onPartProgress(t){const{file_id:s,part_id:i,host_type:
|
|
1
|
+
"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.BaseProcessor=void 0;const typeorm_1=require("typeorm"),entities_1=require("@soga/entities"),types_1=require("@soga/types"),prepare_1=require("../prepare/prepare");let isDequeueing=!1;class BaseProcessor{hostType;onProgress=async()=>{};onSuccess=async()=>{};onError=async()=>{};fileRepository;partRepository;isRunning=!1;thread_count=3;maxThreads=10;threads=[];baseParams;getValidThreads(t){return Math.min(Math.max(t??0,0),this.maxThreads)}async onPartProgress(t){const{file_id:s,part_id:i,host_type:a,percent:e,type:o}=t,r=`${a}_percent`;await this.partRepository.update(i,{[r]:e});const d=await this.partRepository.findBy({file_id:s});let p=0,n=0;for(const t of d)p+=t.size,n+=t.size*t[r];const u=n/p,h=await this.fileRepository.findOneBy({id:s});h.progress[o].percent=u,await this.fileRepository.update(s,{progress:h.progress}),await this.onProgress(h)}async onPartSuccess(t){const{ali_host_id:s,baidu_host_id:i}=t;if(s){if((await this.partRepository.findBy({file_id:t.id,ali_status:(0,typeorm_1.Not)(types_1.UploadStatus.SUCCESS)})).length)return}if(i){if((await this.partRepository.findBy({file_id:t.id,baidu_status:(0,typeorm_1.Not)(types_1.UploadStatus.SUCCESS)})).length)return}const a={ali:{},baidu:{}},e=await this.partRepository.findBy({file_id:t.id});for(const t of e)t.ali_status===types_1.UploadStatus.SUCCESS&&(a.ali[t.info.file]=t.ali_result),t.baidu_status===types_1.UploadStatus.SUCCESS&&(a.baidu[t.info.file]=t.baidu_result);await this.fileRepository.update(t.id,{upload_status:types_1.UploadStatus.SUCCESS,upload_data:a}),await this.partRepository.delete({file_id:t.id});const o=await this.fileRepository.findOneBy({id:t.id});await this.onSuccess(o)}async onPartError(t,s){s.upload_status!==types_1.UploadStatus.ERROR&&(await this.fileRepository.update(s.id,{upload_status:types_1.UploadStatus.ERROR}),await this.onError(t,s))}constructor(t){this.baseParams=t,this.fileRepository=t.dataSource.getRepository(entities_1.UploadFile),this.partRepository=t.dataSource.getRepository(entities_1.UploadPart),this.thread_count=this.getValidThreads(t.threads??this.thread_count),t.onProgress&&(this.onProgress=t.onProgress.bind(this)),t.onSuccess&&(this.onSuccess=t.onSuccess.bind(this)),t.onError&&(this.onError=t.onError.bind(this))}async setThreads(t){const s=this.getValidThreads(t);this.thread_count=s,await this.run()}is_initing=!1;async init(){this.is_initing=!0;const t=`${this.hostType}_status`;this.partRepository.update({uid:this.baseParams.uid,[t]:types_1.UploadStatus.PROCESS},{[t]:types_1.UploadStatus.NULL}),this.is_initing=!1}async start(){await this.run()}async stopFiles(t,s=!1){s&&(await this.partRepository.update({file_id:(0,typeorm_1.In)(t),is_paused:!1},{is_paused:!0}),await this.fileRepository.update({id:(0,typeorm_1.In)(t),is_paused:!1},{is_paused:!0}));const i=this.threads.filter((s=>t.includes(s.file_id)));await Promise.all(i.map((t=>t.stop()))),await this.start()}async stopAll(t=!1){t&&(await this.partRepository.update({uid:this.baseParams.uid},{is_paused:!0}),await this.fileRepository.update({uid:this.baseParams.uid,is_folder:!1},{is_paused:!0})),await Promise.all(this.threads.map((t=>t.stop())))}async run(){for(;this.isRunning;)await new Promise((t=>{setTimeout(t,200)}));this.isRunning=!0;const t=`${this.hostType}_status`,s=`${this.hostType}_host_id`,i=this.threads.length,a=this.thread_count;if(i<a){const e=a-i;for(let i=0;i<e;i++){const i=await this.partRepository.findOne({where:{uid:this.baseParams.uid,is_paused:!1,[t]:types_1.UploadStatus.NULL,[s]:this.baseParams.host_id},order:{created_at:"ASC"}});if(this.hostType,!i)break;const a=this.getThread(i);this.threads.push(a),await this.partRepository.update(i.id,{[t]:types_1.UploadStatus.PROCESS}),a.start()}if(this.threads.length<a){await this.dequeueOneFile();const i=await this.partRepository.findOneBy({uid:this.baseParams.uid,is_paused:!1,[t]:types_1.UploadStatus.NULL,[s]:this.baseParams.host_id});this.isRunning=!1,i&&await this.run()}}else if(i>a){const t=i-a,s=this.threads.slice(0,t);for(const t of s)await t.stop()}this.isRunning=!1}async dequeueOneFile(){for(;isDequeueing;)await new Promise((t=>{setTimeout(t,50)}));isDequeueing=!0;try{const t=await this.fileRepository.findOne({where:{uid:this.baseParams.uid,is_paused:!1,is_folder:!1,encode_status:types_1.EncodeStatus.SUCCESS,upload_status:types_1.UploadStatus.NULL},order:{created_at:"ASC"}});if(!t)return;await this.fileRepository.update(t.id,{upload_status:types_1.UploadStatus.PROCESS}),await(0,prepare_1.prepare)({file_id:t.id,fileRepository:this.fileRepository,createCloudRecord:this.baseParams.createCloudRecord});const{source_data:s,txt_data:i,img_data:a,media_data:e}=t,o=[...s.parts??[],...i.parts??[],...a.parts??[],...e.parts??[]].map((s=>({uid:this.baseParams.uid,file_id:t.id,info:s,filepath:t.path,output_root:t.output_root,size:s.size,ali_host_id:t.ali_host_id||null,baidu_host_id:t.baidu_host_id||null})));return void await this.partRepository.save(o)}finally{isDequeueing=!1}}}exports.BaseProcessor=BaseProcessor;
|
package/dist/types/main.d.ts
CHANGED
|
@@ -20,6 +20,7 @@ export interface UploaderParams {
|
|
|
20
20
|
}
|
|
21
21
|
export interface ProcessorParams {
|
|
22
22
|
uid: number;
|
|
23
|
+
host_id: number;
|
|
23
24
|
dataSource: DataSource;
|
|
24
25
|
threads?: number;
|
|
25
26
|
debug?: boolean;
|
|
@@ -39,13 +40,11 @@ export interface BaiduProcessorParams extends ProcessorParams {
|
|
|
39
40
|
getAccessToken: (baidu_host_id: number) => Promise<string>;
|
|
40
41
|
}
|
|
41
42
|
export interface GetBaiduProcessorParams extends BaiduProcessorParams {
|
|
42
|
-
unique: string | number;
|
|
43
43
|
}
|
|
44
44
|
export interface AliProcessorParams extends ProcessorParams {
|
|
45
45
|
getAccessToken: (ali_host_id: number) => Promise<string>;
|
|
46
46
|
}
|
|
47
47
|
export interface GetAliProcessorParams extends AliProcessorParams {
|
|
48
|
-
unique: string | number;
|
|
49
48
|
}
|
|
50
49
|
export type ThreadType = {
|
|
51
50
|
file_id: number;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.Uploader=void 0;const types_1=require("@soga/types"),typeorm_1=require("typeorm"),entities_1=require("@soga/entities"),baidu_1=require("../processor/baidu"),ali_1=require("../processor/ali");class Uploader{params;uid;dataSource;fileRepository;partRepository;constructor(t){this.params=t,this.uid=t.uid,this.dataSource=t.dataSource,this.fileRepository=this.dataSource.getRepository(entities_1.UploadFile),this.partRepository=this.dataSource.getRepository(entities_1.UploadPart)}processorMap=new Map;async init(){}async start(){await Promise.all([this.startAli(),this.startBaidu()])}async stopAll(t=!1){const s=this.processorMap.values();for(const a of s)await a.stopAll(t)}async stopFiles(t,s=!1){const a=this.processorMap.values();for(const i of a)await i.stopFiles(t,s)}async repairAllParts(){await Promise.all([this.partRepository.update({uid:this.uid,baidu_status:types_1.UploadStatus.ERROR},{baidu_status:types_1.UploadStatus.NULL}),this.partRepository.update({uid:this.uid,ali_status:types_1.UploadStatus.ERROR},{ali_status:types_1.UploadStatus.NULL})])}async repairParts(t){await Promise.all([this.partRepository.update({file_id:(0,typeorm_1.In)(t),baidu_status:types_1.UploadStatus.ERROR},{baidu_status:types_1.UploadStatus.NULL}),this.partRepository.update({file_id:(0,typeorm_1.In)(t),ali_status:types_1.UploadStatus.ERROR},{ali_status:types_1.UploadStatus.NULL})])}async startHost(t,s){if(this.processorMap.has(t)){const s=this.processorMap.get(t);return void await s.start()}const a={uid:this.uid,dataSource:this.dataSource,
|
|
1
|
+
"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.Uploader=void 0;const types_1=require("@soga/types"),typeorm_1=require("typeorm"),entities_1=require("@soga/entities"),baidu_1=require("../processor/baidu"),ali_1=require("../processor/ali");class Uploader{params;uid;dataSource;fileRepository;partRepository;constructor(t){this.params=t,this.uid=t.uid,this.dataSource=t.dataSource,this.fileRepository=this.dataSource.getRepository(entities_1.UploadFile),this.partRepository=this.dataSource.getRepository(entities_1.UploadPart)}processorMap=new Map;async init(){}async start(){await Promise.all([this.startAli(),this.startBaidu()])}async stopAll(t=!1){const s=this.processorMap.values();for(const a of s)await a.stopAll(t)}async stopFiles(t,s=!1){const a=this.processorMap.values();for(const i of a)await i.stopFiles(t,s)}async repairAllParts(){await Promise.all([this.partRepository.update({uid:this.uid,baidu_status:types_1.UploadStatus.ERROR},{baidu_status:types_1.UploadStatus.NULL}),this.partRepository.update({uid:this.uid,ali_status:types_1.UploadStatus.ERROR},{ali_status:types_1.UploadStatus.NULL})])}async repairParts(t){await Promise.all([this.partRepository.update({file_id:(0,typeorm_1.In)(t),baidu_status:types_1.UploadStatus.ERROR},{baidu_status:types_1.UploadStatus.NULL}),this.partRepository.update({file_id:(0,typeorm_1.In)(t),ali_status:types_1.UploadStatus.ERROR},{ali_status:types_1.UploadStatus.NULL})])}async startHost(t,s){if(this.processorMap.has(t)){const s=this.processorMap.get(t);return void await s.start()}const a={uid:this.uid,dataSource:this.dataSource,host_id:t,createCloudRecord:this.params.createCloudRecord.bind(this),onProgress:this.params.onProgress?.bind(this),onSuccess:this.params.onSuccess?.bind(this),onError:this.params.onError?.bind(this)};if(s==types_1.HostType.BAIDU){const s=await(0,baidu_1.getBaiduProcessor)({...a,getAccessToken:this.params.getBaiduAccessToken.bind(this)});this.processorMap.set(t,s),await s.start()}else if(s==types_1.HostType.ALI){const s=await(0,ali_1.getAliProcessor)({...a,getAccessToken:this.params.getAliAccessToken.bind(this)});this.processorMap.set(t,s),await s.start()}}async startAli(){const t=(await this.fileRepository.createQueryBuilder("file").select("DISTINCT file.ali_host_id").where("file.uid = :uid",{uid:this.uid}).getRawMany()).map((t=>t.ali_host_id));if(t.length)for(const s of t)await this.startHost(s,types_1.HostType.ALI)}async startBaidu(){const t=(await this.fileRepository.createQueryBuilder("file").select("DISTINCT file.baidu_host_id").where("file.uid = :uid",{uid:this.uid}).getRawMany()).map((t=>t.baidu_host_id));if(t.length)for(const s of t)await this.startHost(s,types_1.HostType.BAIDU)}}exports.Uploader=Uploader;
|