@soga/uploader 1.3.14 → 1.3.15

package/dist/main.js

@@ -1 +1 @@
-var t,i=Object.create,e=Object.defineProperty,a=Object.getOwnPropertyDescriptor,s=Object.getOwnPropertyNames,o=Object.getPrototypeOf,r=Object.prototype.hasOwnProperty,d=(t,i,o,d)=>{if(i&&"object"==typeof i||"function"==typeof i)for(let n of s(i))r.call(t,n)||n===o||e(t,n,{get:()=>i[n],enumerable:!(d=a(i,n))||d.enumerable});return t},n=(t,a,s)=>(s=null!=t?i(o(t)):{},d(!a&&t&&t.__esModule?s:e(s,"default",{value:t,enumerable:!0}),t)),u={};((t,i)=>{for(var a in i)e(t,a,{get:i[a],enumerable:!0})})(u,{Uploader:()=>Q,getUploader:()=>G}),module.exports=(t=u,d(e({},"__esModule",{value:!0}),t));var p=require("@soga/types"),l=require("typeorm"),c=require("@soga/entities"),f=require("@soga/sdk"),_=require("@soga/types"),h=require("uuid"),y="1.0.0",m=require("@soga/utils"),w=new Map;var g=require("@soga/sdk"),R=require("@soga/types"),S=require("fs-extra"),b=require("typeorm");var P=require("@soga/types"),O=require("@soga/sdk");async function U({file_id:t,sdk_domain:i,fileRepository:e}){const a=await e.findOneBy({id:t});if(a.is_attachment){const{task_record_id:t}=a,s=(0,O.getSdk)(i),r=await s.getRecordInfo({space_id:a.space_id,record_id:t,refresh:!0}),d=await o(r);await e.update(a.id,{cloud_info:d})}else await async function(){const i=await e.findOneBy({id:t});if(!i)return;const a=await s(i.pid,[]);for(const t of a)await r(t);return a}(),await r(a);async function s(t,i=[]){try{if(0===t)return i;const a=await e.findOneBy({id:t});return a?a.cloud_info?.id?i:(i.unshift(a),0!==a.pid?await s(a.pid,i).catch(t=>{throw t}):i):i}catch(t){throw t}}async function o(t){const{id:i,name:s,cloud_info:o}=t,r={id:i,name:s,hosts:{}};if(o.ali&&(r.hosts.ali={id:o.ali.id,name:o.ali.name,drive_id:o.ali.drive_id,file_id:o.ali.file_id}),o.baidu&&(r.hosts.baidu={id:o.baidu.id,name:o.baidu.name,fs_id:o.baidu.fs_id,path:o.baidu.path},!r.hosts.baidu.path)){const t=await e.findOneBy({id:a.pid}),{path:i}=t.cloud_info.hosts.baidu;i&&(r.hosts.baidu.path=`${i}/${o.baidu.name}`)}return r}async function r(t){const a=await e.findOneBy({id:t.id});if(a.cloud_info?.id)return;const{input:s,space_id:r,task_record_id:d,type:n}=a,u=n===P.RecordType.FOLDER;let p=d;if(0!==a.pid){p=(await e.findOneBy({id:a.pid})).cloud_info.id}const l=(0,O.getSdk)(i),c={space_id:r,name:s.filename,parent_id:p,type:a.type,ftype:a.ftype},f={};a.baidu_host_id&&(f.baidu=0),a.ali_host_id&&(f.ali=0);const _=await l.createRecord({...c,manifest:JSON.stringify({version:y,complete:!!u,hosts:f,metas:{file:{btime:a.input.local_btime,ctime:a.input.local_ctime,mtime:a.input.local_mtime,file:a.input.filename}}})}),h=await o(_);await e.update({id:a.id},{cloud_info:h})}}var E=require("@soga/entities"),T=require("@soga/types"),A=require("@soga/utils"),B=require("@soga/node-types"),D=class{hostType;onProgress=async()=>{};fileRepository;partRepository;runningRepository;isRunning=!1;thread_count=2;maxThreads=8;threads=[];baseParams;getValidThreads(t){return Math.min(Math.max(t??0,0),this.maxThreads)}async onPartProgress(t){const{id:i,data:e,type:a}=t;if("percent"!=a)return;const{part_id:s,percent:o}=e,r=await this.partRepository.findOneBy({id:s});if(!r)return;if(r.upload_percent>=o)return;if(r.upload_status==T.NormalStatus.ERROR)return;if(r.upload_status==T.NormalStatus.SUCCESS)return;await this.partRepository.update(s,{upload_percent:o});const d=await this.partRepository.findBy({file_id:i,host_type:this.hostType});let n=0,u=0;for(const t of d)n+=t.info.size,u+=t.info.size*t.upload_percent;const p=u/n,l=await this.fileRepository.findOneBy({id:i}),c=B.UploadProgress[`upload_${this.hostType}`];l.progress[c].percent=p,l.percent=(0,A.getProgressPercent)(l.progress),await this.fileRepository.update(i,{progress:l.progress,percent:l.percent}),await this.onProgress(l)}async onPartError({error:t,part_id:i,file_id:e}){const a=await this.fileRepository.findOneBy({id:e});a&&(await this.fileRepository.update(e,{upload_status:T.UploadStatus.UPLOAD_ERROR,error:{type:"upload",message:t.message,details:t.details,cause:t.cause,stack:t.stack}}),await this.partRepository.update(i,{upload_status:T.NormalStatus.ERROR}),await this.fileRepository.update(a.id,{upload_status:T.UploadStatus.UPLOAD_ERROR}),await this.baseParams.onError(t,a.id))}constructor(t,i){this.hostType=i,this.baseParams=t;const{dataSource:e}=t;this.fileRepository=e.getRepository(E.UploadFile),this.partRepository=e.getRepository(E.UploadPart),this.runningRepository=e.getRepository(E.UploadRunning),t.onProgress&&(this.onProgress=t.onProgress.bind(this))}async setThreads(t){const i=this.getValidThreads(t);this.thread_count=i,await this.start()}async getParts(t){const i={uid:this.baseParams.uid,is_paused:!1,upload_status:T.NormalStatus.NULL,host_id:this.baseParams.host_id};for(let e=0;e<t;e++){const e=await this.partRepository.countBy(i);if(e>=t)break;if(e<t){await this.baseParams.onDequeue(this.hostType);if(await this.partRepository.countBy(i)==e)break}}return await this.partRepository.find({where:i,order:{id:"ASC"},take:t})}async start(){for(;this.isRunning;)await new Promise(t=>{setTimeout(t,50)});try{this.isRunning=!0;const t=this.threads.length,i=this.thread_count;if(t<i){const e=i-t,a=await this.getParts(e),s=a.length;for(let t=0;t<s;t++){const i=a[t],e=this.getThread(i);(await this.fileRepository.findOneBy({id:i.file_id})).is_paused||(this.threads.push(e),await this.partRepository.update(i.id,{upload_status:T.NormalStatus.PROCESS}),e.start().catch(async t=>{}))}}else if(t>i){const e=t-i,a=this.threads.slice(0,e);for(const t of a)await t.stop()}}finally{this.isRunning=!1}}async stopFiles(t){const i=this.threads.filter(i=>t.includes(i.file_id));await Promise.all(i.map(t=>t.stop()))}async stopAll(){await Promise.all(this.threads.map(t=>t.stop()))}async addRunning({file_id:t,part_id:i,root_id:e}){await this.runningRepository.save(this.runningRepository.create({type:E.UploadRunningType.UPLOAD,uid:this.baseParams.uid,root_id:e,file_id:t,part_id:i}))}async deleteRunning({file_id:t,part_id:i}){await this.runningRepository.delete({uid:this.baseParams.uid,file_id:t,part_id:i})}},k=require("@soga/types"),C=n(require("piscina")),q=require("worker_threads"),v=require("@soga/error"),L=new C.default({filename:require.resolve("@soga/part-uploader")}),N=class extends D{params;constructor(t){super(t,k.HostType.BAIDU),this.params=t}getThread(t){const i=new AbortController,{port1:e,port2:a}=new q.MessageChannel;return{file_id:t.file_id,part_id:t.id,uid:t.uid,start:async()=>{let s=null;try{if(s=await this.fileRepository.findOneBy({id:t.file_id}),!s)return;const{file_id:o,host_id:r,id:d,info:n,upload_status:u,root_id:p}=t;if(u==k.NormalStatus.SUCCESS)return;await this.addRunning({file_id:o,part_id:d,root_id:p});const{output_root:l}=s;a.on("message",async t=>{await this.onPartProgress(t)});const c={file_id:o,host_id:r,part_id:d,output_root:l,part:n,cloud_folder_path:s.cloud_info.hosts.baidu?.path,sdk_domain:this.params.sdk_domain,port:e,appid:30045106},f=await L.run(c,{name:"uploadBaidu",signal:i.signal,transferList:[e]});if(!f?.fs_id)throw new Error("Upload to Baidu failed, result is null");await this.partRepository.update(t.id,{upload_result:{baidu:f},upload_status:k.NormalStatus.SUCCESS}),await this.baseParams.onPartSuccess(o)}catch(e){if(i.abort(),"AbortError"==e.name)await this.partRepository.update(t.id,{upload_status:k.NormalStatus.NULL});else{const i=s?s.input.filepath:"unknown filepath",a=(0,v.buildDError)(e,{message:"Error occurred during baidu uploading",detail:`Error occurred during baidu uploading: ${i}`});await this.fileRepository.update(t.file_id,{upload_status:k.UploadStatus.UPLOAD_ERROR,error:{type:"upload",message:a.message,details:a.details,cause:a.cause,stack:a.stack}}),await this.onPartError({error:e,part_id:t.id,file_id:t.file_id})}}finally{this.threads=this.threads.filter(i=>i.part_id!=t.id),a.close(),await this.deleteRunning({file_id:t.file_id,part_id:t.id}),await this.start()}},stop:async()=>{i.abort(),this.threads=this.threads.filter(i=>i.part_id!==t.id)}}}},I=require("@soga/types"),M=n(require("piscina")),H=require("worker_threads"),z=require("@soga/error"),F=new M.default({filename:require.resolve("@soga/part-uploader")}),x=class extends D{params;constructor(t){super(t,I.HostType.ALI),this.params=t}getThread(t){const i=new AbortController,{port1:e,port2:a}=new H.MessageChannel;return{file_id:t.file_id,part_id:t.id,uid:t.uid,start:async()=>{const s=await this.fileRepository.findOneBy({id:t.file_id});if(s)try{const{file_id:o,host_id:r,id:d,info:n,upload_status:u,root_id:p}=t;if(u==I.NormalStatus.SUCCESS)return;await this.addRunning({file_id:o,part_id:d,root_id:p});const{output_root:l}=s;a.on("message",async t=>{await this.onPartProgress(t)});const c={file_id:o,host_id:r,part_id:d,output_root:l,part:n,cloud_folder_id:s.cloud_info.hosts.ali?.file_id,sdk_domain:this.params.sdk_domain,port:e},f=await F.run(c,{name:"uploadAli",signal:i.signal,transferList:[e]});if(!f?.file_id)throw new Error("Upload to Ali failed, result is null");await this.partRepository.update(t.id,{upload_result:{ali:f},upload_status:I.NormalStatus.SUCCESS}),await this.baseParams.onPartSuccess(o)}catch(e){if(i.abort(),"AbortError"==e.name)this.partRepository.update(t.id,{upload_status:I.NormalStatus.NULL});else{const i=(0,z.buildDError)(e,{message:"Error occurred during ali uploading",detail:`Error occurred during ali uploading: ${s.input.filepath}`});await this.fileRepository.update(t.file_id,{upload_status:I.UploadStatus.UPLOAD_ERROR,error:{type:"upload",message:i.message,details:i.details,cause:i.cause,stack:i.stack}}),await this.onPartError({error:e,part_id:t.id,file_id:t.file_id})}}finally{this.threads=this.threads.filter(i=>i.part_id!=t.id),a.close(),await this.deleteRunning({file_id:t.file_id,part_id:t.id}),await this.start()}},stop:async()=>{i.abort(),this.threads=this.threads.filter(i=>i.part_id!==t.id)}}}},j=require("@soga/error"),$=require("@soga/utils"),Y=require("@soga/node-types"),J=new Map,Q=class{params;uid;dataSource;fileRepository;partRepository;runningRepository;successRepository;processorMap=new Map;isDequeueing=!1;threadConfig={baidu:4,ali:2};constructor(t){this.params=t,this.uid=t.uid,this.dataSource=t.dataSource,this.fileRepository=this.dataSource.getRepository(c.UploadFile),this.partRepository=this.dataSource.getRepository(c.UploadPart),this.runningRepository=this.dataSource.getRepository(c.UploadRunning),this.successRepository=this.dataSource.getRepository(c.UploadSuccess)}async init(){await this.runningRepository.delete({uid:this.uid,type:c.UploadRunningType.UPLOAD}),await this.fileRepository.update({uid:this.uid,upload_status:p.UploadStatus.UPLOAD_ERROR},{upload_status:p.UploadStatus.UPLOAD_READY}),await this.partRepository.update({uid:this.uid,upload_status:p.NormalStatus.PROCESS},{upload_status:p.NormalStatus.NULL})}async start(){await this.startCreates(),await Promise.all([this.startBaidu(),this.startAli()]).catch(t=>{console.error("Error starting uploader:",t)})}async stopFiles(t){const i=this.processorMap.values();for(const e of i)await e.stopFiles(t)}async stopAll(){const t=this.processorMap.values();for(const i of t)await i.stopAll()}async setThreadCount(t,i){this.threadConfig[t]=i;const e=this.processorMap.values();for(const a of e)a.hostType==t&&await a.setThreads(i)}isCreating=!1;async startCreates(){for(;this.isCreating;)await new Promise(t=>setTimeout(t,100));try{this.isCreating=!0;const t=await this.fileRepository.findBy({uid:this.uid,upload_status:p.UploadStatus.SUBMIT_READY});if(!t.length)return;for(const i of t)await this.onPartSuccess({file_id:i.id})}finally{this.isCreating=!1}}async startHost(t,i){if(this.processorMap.has(t)){const i=this.processorMap.get(t),e=this.getThreadCount(i.hostType);return void await i.setThreads(e)}const e={uid:this.uid,sdk_domain:this.params.sdk_domain,dataSource:this.dataSource,host_id:t,onProgress:async t=>{await this.onFileProgress(t)},onPartSuccess:async t=>{await this.onPartSuccess({file_id:t})},onError:async(t,i)=>{await this.onFileError(t,i)},onDequeue:async t=>{await this.dequeueOneFile(t)}};if(i==p.HostType.BAIDU){const i=new N(e);this.processorMap.set(t,i);const a=this.getThreadCount(p.HostType.BAIDU);await i.setThreads(a)}else if(i==p.HostType.ALI){const i=new x(e);this.processorMap.set(t,i);const a=this.getThreadCount(p.HostType.ALI);await i.setThreads(a)}}async startBaidu(){const t=(await this.fileRepository.createQueryBuilder("file").select("DISTINCT file.baidu_host_id").where("file.uid = :uid",{uid:this.uid}).getRawMany()).map(t=>t.baidu_host_id).filter(t=>!!t);if(t.length)for(const i of t)await this.startHost(i,p.HostType.BAIDU)}async startAli(){const t=(await this.fileRepository.createQueryBuilder("file").select("DISTINCT file.ali_host_id").where("file.uid = :uid",{uid:this.uid}).getRawMany()).map(t=>t.ali_host_id).filter(t=>!!t);if(t.length)for(const i of t)await this.startHost(i,p.HostType.ALI)}async onPartSuccess({file_id:t}){await this.partRepository.findOneBy({file_id:t,upload_status:(0,l.Not)(p.NormalStatus.SUCCESS)})||await this.onFileSuccess(t)}async onFileSuccess(t){try{const{uid:i}=this;for(;J.get(i);)await new Promise(t=>setTimeout(t,100));J.set(i,!0);const e=await this.fileRepository.findOneBy({id:t});if(!e)return;try{const i=await this.partRepository.findBy({file_id:e.id}),a={},s=e.progress;if(e.baidu_host_id){a.baidu_upload_result={};for(const t of i)t.host_type==p.HostType.BAIDU&&(a.baidu_upload_result[t.info.file]={...t.upload_result.baidu,md4:t.info.md4});s[Y.UploadProgress.upload_baidu].percent=1}if(e.ali_host_id){a.ali_upload_result={};for(const t of i)t.host_type==p.HostType.ALI&&(a.ali_upload_result[t.info.file]={...t.upload_result.ali,sha1:t.info.sha1});s[Y.UploadProgress.upload_ali].percent=1}const o=(0,$.getProgressPercent)(s);await this.fileRepository.update(e.id,{...a,progress:s,percent:o,upload_status:p.UploadStatus.SUBMIT_READY}),await async function({file_id:t,sdk_domain:i,fileRepository:e}){try{for(;w.get(t);)await new Promise(t=>setTimeout(t,100));w.set(t,!0);const a=await e.findOneBy({id:t});let s=a.task_record_id;const o=(0,f.getSdk)(i),{encoded:r,baidu_host_id:d,ali_host_id:n}=a,{ali:u,baidu:p,meta:l}=r,c=[];if(d&&c.push(_.HostType.BAIDU),n&&c.push(_.HostType.ALI),!c.length)return;const g=t=>{const i=r[t];if(!i.media?.subtitles?.length)return;const e=[];return i.media.subtitles.forEach(t=>{const{uuid:i,lang:a,name:s,title:o,builtin:r}=t;e.push({uuid:i,lang:a,name:s,title:o,builtin:r})}),e},R=t=>{if(!t)return;if(!t.subtitles)return t;const{subtitles:i,...e}=t,a={};return i.forEach(t=>{const{uuid:i,file:e,start:s,end:o,size:r,source:d}=t;a[i]={file:e,start:s,end:o,size:r,source:d}}),{...e,subtitles:a}};if(a.is_attachment){const t=await o.getRecordInfo({space_id:a.space_id,record_id:a.task_record_id,refresh:!0}),i=Object.assign({version:y,metas:{},hosts:{},attachments:[]},t.manifest);t.type,_.RecordType.FOLDER;const e={metas:l,uuid:(0,h.v4)(),type:a.type,ftype:a.ftype,title:a.input.filename,subtitles:g(c[0])};if(d){const{parts:t,max_size:i,media:s,img:o,txt:r,source:n}=p;e.baidu={host_id:d,media:R(s),max_size:i,img:o,txt:r,source:n,files:{}};const u=a.baidu_upload_result;t.forEach(t=>{const{file:i,md4:a,preview:s,source:o,size:r,md5:d}=t,{fs_id:n}=u[i];e.baidu.files[i]={fs_id:n,md5:d,md4:a,size:r,preview:s,source:o}})}if(n){const{parts:t,max_size:i,media:s,img:o,txt:r,source:d}=u,{drive_id:p}=a.cloud_info.hosts.ali;e.ali={host_id:n,drive_id:p,media:R(s),max_size:i,img:o,txt:r,source:d,files:{}};const l=a.ali_upload_result;t.forEach(t=>{const{file:i,preview:a,source:s,size:o,md5:r}=t,{file_id:d,sha1:n}=l[i];e.ali.files[i]={file_id:d,md5:r,sha1:n,size:o,preview:a,source:s}})}i.attachments=i.attachments||[],i.attachments.push(e),i.hosts||(i.hosts={}),d&&(i.hosts.baidu=(0,m.calculateManifestHostSize)(i,_.HostType.BAIDU)),n&&(i.hosts.ali=(0,m.calculateManifestHostSize)(i,_.HostType.ALI)),await o.updateRecord({space_id:t.space_id,record_id:t.id,parent_id:t.parent_id,manifest:JSON.stringify(i)})}else{0!==a.pid&&(s=(await e.findOneBy({id:a.pid})).cloud_info.id);const t={version:y,hosts:{},complete:!0,metas:l,subtitles:g(c[0])};if(d){const{parts:i,media:e,...s}=p,o=R(e),r=a.baidu_upload_result;t.baidu={host_id:d,...s,files:{},media:o},i.forEach(i=>{const{file:e,md4:a,preview:s,source:o,size:d,md5:n}=i,{fs_id:u}=r[e];t.baidu.files[e]={fs_id:u,md5:n,md4:a,size:d,preview:s,source:o}})}if(n){const{parts:i,media:e,...s}=u,o=R(e),r=a.ali_upload_result;t.ali={host_id:n,drive_id:"",...s,media:o,files:{}},i.forEach(i=>{const{file:e,preview:a,source:s,size:o,md5:d}=i,{file_id:n,drive_id:u,sha1:p}=r[e];t.ali.drive_id||(t.ali.drive_id=u),t.ali.files[e]={file_id:n,sha1:p,md5:d,size:o,preview:a,source:s}});const d=Object.values(t.ali.files).reduce((t,i)=>t+i.size,0);t.hosts.ali=d}if(d&&(t.hosts.baidu=(0,m.calculateManifestHostSize)(t,_.HostType.BAIDU)),n&&(t.hosts.ali=(0,m.calculateManifestHostSize)(t,_.HostType.ALI)),a.cloud_info?.id){const t=await o.getRecordInfo({space_id:a.space_id,record_id:a.cloud_info.id,refresh:!0});if(t.manifest?.complete)throw new Error("the file has been modified!")}await o.updateRecord({space_id:a.space_id,record_id:a.cloud_info.id,parent_id:s,manifest:JSON.stringify(t)})}}catch(t){throw t}finally{w.delete(t)}}({file_id:e.id,fileRepository:this.fileRepository,sdk_domain:this.params.sdk_domain}),this.params.onSuccess&&await this.params.onSuccess(t),await async function(t){const{fileRepository:i,partRepository:e,successRepository:a}=t,s=await i.findOneBy({id:t.file_id}),o={is_attachment:s.is_attachment,root_id:s.root_id,pid:0,space_id:s.space_id,space_name:s.space_name,uid:s.uid,input:s.input,cloud_id:s.cloud_info.id,cloud_name:s.cloud_info.name,type:s.type},r=0==s.root_id;if(!r){const e=await a.findOneBy({root_id:s.root_id,pid:0});if(s.is_attachment){const r=(0,g.getSdk)(t.sdk_domain);if(e)o.pid=e.id;else{const t=await r.getRecordInfo({space_id:s.space_id,record_id:s.task_record_id,refresh:!1}),e=await i.findOneBy({id:s.root_id}),d=await a.save(a.create({is_attachment:!0,root_id:s.root_id,pid:0,space_id:t.space_id,space_name:s.space_name,uid:s.uid,input:e.input,cloud_id:t.id,cloud_name:t.name,type:t.type}));o.pid=d.id}}else if(e)o.pid=e.id;else{const t=await i.findOneBy({id:s.root_id}),e=await a.save(a.create({is_attachment:!1,root_id:s.root_id,pid:0,space_id:t.space_id,space_name:t.space_name,uid:t.uid,input:t.input,cloud_id:t.cloud_info.id,cloud_name:t.cloud_info.name,type:t.type}));o.pid=e.id}await a.update({id:o.pid},{updated_at:new Date})}await a.save(a.create(o)),await i.delete({id:t.file_id}),await e.delete({file_id:t.file_id}),await(0,S.remove)(s.output_root),r||(await i.findOneBy({root_id:s.root_id,type:(0,b.Not)(R.RecordType.FOLDER)})?await i.increment({id:s.root_id},"completed_count",1):(await i.delete({root_id:s.root_id}),await i.delete({id:s.root_id})))}({file_id:e.id,sdk_domain:this.params.sdk_domain,fileRepository:this.fileRepository,partRepository:this.partRepository,successRepository:this.successRepository})}catch(t){await this.fileRepository.update(e.id,{upload_status:p.UploadStatus.UPLOAD_ERROR,error:{type:"upload",message:t.message,details:t.details,cause:t.cause,stack:t.stack}})}finally{J.delete(i)}}catch(t){}}async onFileError(t,i){try{this.params.onError&&await this.params.onError(t,i)}catch(t){}}async onFileProgress(t){try{this.params.onProgress&&await this.params.onProgress(t)}catch(t){}}async dequeueOneFile(t){for(;this.isDequeueing;)await new Promise(t=>{setTimeout(t,50)});let i=null;try{this.isDequeueing=!0;const e=`${t}_host_id`;if(i=await this.fileRepository.findOne({where:{uid:this.params.uid,is_paused:!1,upload_status:p.UploadStatus.UPLOAD_READY,[e]:(0,l.Not)((0,l.IsNull)())},order:{id:"ASC"}}),!i)return;if(i.cloud_info?.id||(await U({file_id:i.id,sdk_domain:this.params.sdk_domain,fileRepository:this.fileRepository}),i=await this.fileRepository.findOneBy({id:i.id})),!i.cloud_info?.id)throw new Error("Created cloud folder or file failed");const{encoded:a}=i,s=[],o=[p.HostType.BAIDU,p.HostType.ALI];for await(const t of o){const e=`${t}_host_id`;if(!i[e])continue;const o=i[e],r=a[t].parts||[],{length:d}=r;for(let e=0;e<d;e++){const a=r[e];await this.partRepository.findOneBy({file_id:i.id,part_name:a.file,host_id:o})||s.push({uid:this.params.uid,file_id:i.id,root_id:i.root_id??i.id,part_name:a.file,info:a,output_root:i.output_root,host_id:o,host_type:t})}}if(s.length)await this.partRepository.save(s),await this.fileRepository.update(i.id,{upload_status:p.UploadStatus.UPLOAD_PROCESS});else{await this.fileRepository.update(i.id,{upload_status:p.UploadStatus.UPLOAD_PROCESS});const e=await this.partRepository.findOneBy({file_id:i.id,upload_status:(0,l.Not)(p.NormalStatus.SUCCESS)});await this.partRepository.findOneBy({file_id:i.id,upload_status:p.NormalStatus.SUCCESS});e||await this.onPartSuccess({file_id:i.id});await this.partRepository.findOneBy({file_id:i.id,upload_status:p.NormalStatus.NULL})||(this.isDequeueing=!1,await this.dequeueOneFile(t))}}catch(e){if(!i)throw e;{const a=(0,j.buildDError)(e,{message:"Error occurred during baidu preparing",detail:`Error occurred during baidu preparing: ${i.input.filepath}`});await this.fileRepository.update(i.id,{upload_status:p.UploadStatus.UPLOAD_ERROR,error:{type:"upload",message:a.message,details:a.details,cause:a.cause,stack:a.stack}}),this.isDequeueing=!1,await this.dequeueOneFile(t)}}finally{this.isDequeueing=!1}}getThreadCount(t){return this.threadConfig[t]??2}},V=new Map,G=async t=>{for(;V.get(t.uid)?.initing;)await new Promise(t=>setTimeout(t,100));try{if(V.get(t.uid)?.uploader)return V.get(t.uid).uploader;const i={uploader:new Q(t),initing:!0};return V.set(t.uid,i),await i.uploader.init(),i.uploader}finally{V.get(t.uid)&&(V.get(t.uid).initing=!1)}};
+var t,i=Object.create,e=Object.defineProperty,a=Object.getOwnPropertyDescriptor,s=Object.getOwnPropertyNames,o=Object.getPrototypeOf,r=Object.prototype.hasOwnProperty,d=(t,i,o,d)=>{if(i&&"object"==typeof i||"function"==typeof i)for(let n of s(i))r.call(t,n)||n===o||e(t,n,{get:()=>i[n],enumerable:!(d=a(i,n))||d.enumerable});return t},n=(t,a,s)=>(s=null!=t?i(o(t)):{},d(!a&&t&&t.__esModule?s:e(s,"default",{value:t,enumerable:!0}),t)),u={};((t,i)=>{for(var a in i)e(t,a,{get:i[a],enumerable:!0})})(u,{Uploader:()=>Q,getUploader:()=>G}),module.exports=(t=u,d(e({},"__esModule",{value:!0}),t));var p=require("@soga/types"),l=require("typeorm"),c=require("@soga/entities"),f=require("@soga/sdk"),_=require("@soga/types"),h=require("uuid"),y="1.0.0",m=require("@soga/utils"),w=new Map;var g=require("@soga/sdk"),R=require("@soga/types"),S=require("fs-extra"),b=require("typeorm");var P=require("@soga/types"),O=require("@soga/sdk");async function U({file_id:t,sdk_domain:i,fileRepository:e}){const a=await e.findOneBy({id:t});if(a.is_attachment){const{task_record_id:t}=a,s=(0,O.getSdk)(i),r=await s.getRecordInfo({space_id:a.space_id,record_id:t,refresh:!0}),d=await o(r);await e.update(a.id,{cloud_info:d})}else await async function(){const i=await e.findOneBy({id:t});if(!i)return;const a=await s(i.pid,[]);for(const t of a)await r(t);return a}(),await r(a);async function s(t,i=[]){try{if(0===t)return i;const a=await e.findOneBy({id:t});return a?a.cloud_info?.id?i:(i.unshift(a),0!==a.pid?await s(a.pid,i).catch(t=>{throw t}):i):i}catch(t){throw t}}async function o(t){const{id:i,name:s,cloud_info:o}=t,r={id:i,name:s,hosts:{}};if(o.ali&&(r.hosts.ali={id:o.ali.id,name:o.ali.name,drive_id:o.ali.drive_id,file_id:o.ali.file_id}),o.baidu&&(r.hosts.baidu={id:o.baidu.id,name:o.baidu.name,fs_id:o.baidu.fs_id,path:o.baidu.path},!r.hosts.baidu.path)){const t=await e.findOneBy({id:a.pid}),{path:i}=t.cloud_info.hosts.baidu;i&&(r.hosts.baidu.path=`${i}/${o.baidu.name}`)}return r}async function r(t){const a=await e.findOneBy({id:t.id});if(a.cloud_info?.id)return;const{input:s,space_id:r,task_record_id:d,type:n}=a,u=n===P.RecordType.FOLDER;let p=d;if(0!==a.pid){p=(await e.findOneBy({id:a.pid})).cloud_info.id}const l=(0,O.getSdk)(i),c={space_id:r,name:s.filename,parent_id:p,type:a.type,ftype:a.ftype},f={};a.baidu_host_id&&(f.baidu=0),a.ali_host_id&&(f.ali=0);const _=await l.createRecord({...c,manifest:JSON.stringify({version:y,complete:!!u,hosts:f,metas:{file:{btime:a.input.local_btime,ctime:a.input.local_ctime,mtime:a.input.local_mtime,file:a.input.filename}}})}),h=await o(_);await e.update({id:a.id},{cloud_info:h})}}var E=require("@soga/entities"),T=require("@soga/types"),A=require("@soga/utils"),B=require("@soga/node-types"),D=class{hostType;onProgress=async()=>{};fileRepository;partRepository;runningRepository;isRunning=!1;thread_count=2;maxThreads=8;threads=[];baseParams;getValidThreads(t){return Math.min(Math.max(t??0,0),this.maxThreads)}async onPartProgress(t){const{id:i,data:e,type:a}=t;if("percent"!=a)return;const{part_id:s,percent:o}=e,r=await this.partRepository.findOneBy({id:s});if(!r)return;if(r.upload_percent>=o)return;if(r.upload_status==T.NormalStatus.ERROR)return;if(r.upload_status==T.NormalStatus.SUCCESS)return;await this.partRepository.update(s,{upload_percent:o});const d=await this.partRepository.findBy({file_id:i,host_type:this.hostType});let n=0,u=0;for(const t of d)n+=t.info.size,u+=t.info.size*t.upload_percent;const p=u/n,l=await this.fileRepository.findOneBy({id:i}),c=B.UploadProgress[`upload_${this.hostType}`];l.progress[c].percent=p,l.percent=(0,A.getProgressPercent)(l.progress),await this.fileRepository.update(i,{progress:l.progress,percent:l.percent}),await this.onProgress(l)}async onPartError({error:t,part_id:i,file_id:e}){const a=await this.fileRepository.findOneBy({id:e});a&&(await this.fileRepository.update(e,{upload_status:T.UploadStatus.UPLOAD_ERROR,error:{type:"upload",message:t.message,details:t.details,cause:t.cause,stack:t.stack}}),await this.partRepository.update(i,{upload_status:T.NormalStatus.ERROR}),await this.fileRepository.update(a.id,{upload_status:T.UploadStatus.UPLOAD_ERROR}),await this.baseParams.onError(t,a.id))}constructor(t,i){this.hostType=i,this.baseParams=t;const{dataSource:e}=t;this.fileRepository=e.getRepository(E.UploadFile),this.partRepository=e.getRepository(E.UploadPart),this.runningRepository=e.getRepository(E.UploadRunning),t.onProgress&&(this.onProgress=t.onProgress.bind(this))}async setThreads(t){const i=this.getValidThreads(t);this.thread_count=i,await this.start()}async getParts(t){const i={uid:this.baseParams.uid,is_paused:!1,upload_status:T.NormalStatus.NULL,host_id:this.baseParams.host_id};for(let e=0;e<t;e++){const e=await this.partRepository.countBy(i);if(e>=t)break;if(e<t){await this.baseParams.onDequeue(this.hostType);if(await this.partRepository.countBy(i)==e)break}}return await this.partRepository.find({where:i,order:{id:"ASC"},take:t})}async start(){for(;this.isRunning;)await new Promise(t=>{setTimeout(t,50)});try{this.isRunning=!0;const t=this.threads.length,i=this.thread_count;if(t<i){const e=i-t,a=await this.getParts(e);console.log("parts: ",a);const s=a.length;for(let t=0;t<s;t++){const i=a[t],e=this.getThread(i);(await this.fileRepository.findOneBy({id:i.file_id})).is_paused||(this.threads.push(e),await this.partRepository.update(i.id,{upload_status:T.NormalStatus.PROCESS}),e.start().catch(async t=>{}))}}else if(t>i){const e=t-i,a=this.threads.slice(0,e);for(const t of a)await t.stop()}}finally{this.isRunning=!1}}async stopFiles(t){const i=this.threads.filter(i=>t.includes(i.file_id));await Promise.all(i.map(t=>t.stop()))}async stopAll(){await Promise.all(this.threads.map(t=>t.stop()))}async addRunning({file_id:t,part_id:i,root_id:e}){await this.runningRepository.save(this.runningRepository.create({type:E.UploadRunningType.UPLOAD,uid:this.baseParams.uid,root_id:e,file_id:t,part_id:i}))}async deleteRunning({file_id:t,part_id:i}){await this.runningRepository.delete({uid:this.baseParams.uid,file_id:t,part_id:i})}},k=require("@soga/types"),C=n(require("piscina")),q=require("worker_threads"),v=require("@soga/error"),L=new C.default({filename:require.resolve("@soga/part-uploader")}),N=class extends D{params;constructor(t){super(t,k.HostType.BAIDU),this.params=t}getThread(t){const i=new AbortController,{port1:e,port2:a}=new q.MessageChannel;return{file_id:t.file_id,part_id:t.id,uid:t.uid,start:async()=>{let s=null;try{if(s=await this.fileRepository.findOneBy({id:t.file_id}),!s)return;const{file_id:o,host_id:r,id:d,info:n,upload_status:u,root_id:p}=t;if(u==k.NormalStatus.SUCCESS)return;await this.addRunning({file_id:o,part_id:d,root_id:p});const{output_root:l}=s;a.on("message",async t=>{await this.onPartProgress(t)});const c={file_id:o,host_id:r,part_id:d,output_root:l,part:n,cloud_folder_path:s.cloud_info.hosts.baidu?.path,sdk_domain:this.params.sdk_domain,port:e,appid:30045106},f=await L.run(c,{name:"uploadBaidu",signal:i.signal,transferList:[e]});if(!f?.fs_id)throw new Error("Upload to Baidu failed, result is null");await this.partRepository.update(t.id,{upload_result:{baidu:f},upload_status:k.NormalStatus.SUCCESS}),await this.baseParams.onPartSuccess(o)}catch(e){if(i.abort(),"AbortError"==e.name)await this.partRepository.update(t.id,{upload_status:k.NormalStatus.NULL});else{const i=s?s.input.filepath:"unknown filepath",a=(0,v.buildDError)(e,{message:"Error occurred during baidu uploading",detail:`Error occurred during baidu uploading: ${i}`});await this.fileRepository.update(t.file_id,{upload_status:k.UploadStatus.UPLOAD_ERROR,error:{type:"upload",message:a.message,details:a.details,cause:a.cause,stack:a.stack}}),await this.onPartError({error:e,part_id:t.id,file_id:t.file_id})}}finally{this.threads=this.threads.filter(i=>i.part_id!=t.id),a.close(),await this.deleteRunning({file_id:t.file_id,part_id:t.id}),await this.start()}},stop:async()=>{i.abort(),this.threads=this.threads.filter(i=>i.part_id!==t.id)}}}},I=require("@soga/types"),M=n(require("piscina")),H=require("worker_threads"),z=require("@soga/error"),F=new M.default({filename:require.resolve("@soga/part-uploader")}),x=class extends D{params;constructor(t){super(t,I.HostType.ALI),this.params=t}getThread(t){const i=new AbortController,{port1:e,port2:a}=new H.MessageChannel;return{file_id:t.file_id,part_id:t.id,uid:t.uid,start:async()=>{const s=await this.fileRepository.findOneBy({id:t.file_id});if(s)try{const{file_id:o,host_id:r,id:d,info:n,upload_status:u,root_id:p}=t;if(u==I.NormalStatus.SUCCESS)return;await this.addRunning({file_id:o,part_id:d,root_id:p});const{output_root:l}=s;a.on("message",async t=>{await this.onPartProgress(t)});const c={file_id:o,host_id:r,part_id:d,output_root:l,part:n,cloud_folder_id:s.cloud_info.hosts.ali?.file_id,sdk_domain:this.params.sdk_domain,port:e},f=await F.run(c,{name:"uploadAli",signal:i.signal,transferList:[e]});if(!f?.file_id)throw new Error("Upload to Ali failed, result is null");await this.partRepository.update(t.id,{upload_result:{ali:f},upload_status:I.NormalStatus.SUCCESS}),await this.baseParams.onPartSuccess(o)}catch(e){if(i.abort(),"AbortError"==e.name)this.partRepository.update(t.id,{upload_status:I.NormalStatus.NULL});else{const i=(0,z.buildDError)(e,{message:"Error occurred during ali uploading",detail:`Error occurred during ali uploading: ${s.input.filepath}`});await this.fileRepository.update(t.file_id,{upload_status:I.UploadStatus.UPLOAD_ERROR,error:{type:"upload",message:i.message,details:i.details,cause:i.cause,stack:i.stack}}),await this.onPartError({error:e,part_id:t.id,file_id:t.file_id})}}finally{this.threads=this.threads.filter(i=>i.part_id!=t.id),a.close(),await this.deleteRunning({file_id:t.file_id,part_id:t.id}),await this.start()}},stop:async()=>{i.abort(),this.threads=this.threads.filter(i=>i.part_id!==t.id)}}}},j=require("@soga/error"),$=require("@soga/utils"),Y=require("@soga/node-types"),J=new Map,Q=class{params;uid;dataSource;fileRepository;partRepository;runningRepository;successRepository;processorMap=new Map;isDequeueing=!1;threadConfig={baidu:4,ali:2};constructor(t){this.params=t,this.uid=t.uid,this.dataSource=t.dataSource,this.fileRepository=this.dataSource.getRepository(c.UploadFile),this.partRepository=this.dataSource.getRepository(c.UploadPart),this.runningRepository=this.dataSource.getRepository(c.UploadRunning),this.successRepository=this.dataSource.getRepository(c.UploadSuccess)}async init(){await this.runningRepository.delete({uid:this.uid,type:c.UploadRunningType.UPLOAD}),await this.fileRepository.update({uid:this.uid,upload_status:p.UploadStatus.UPLOAD_ERROR},{upload_status:p.UploadStatus.UPLOAD_READY}),await this.partRepository.update({uid:this.uid,upload_status:p.NormalStatus.PROCESS},{upload_status:p.NormalStatus.NULL})}async start(){await this.startCreates(),await Promise.all([this.startBaidu(),this.startAli()]).catch(t=>{console.error("Error starting uploader:",t)})}async stopFiles(t){const i=this.processorMap.values();for(const e of i)await e.stopFiles(t)}async stopAll(){const t=this.processorMap.values();for(const i of t)await i.stopAll()}async setThreadCount(t,i){this.threadConfig[t]=i;const e=this.processorMap.values();for(const a of e)a.hostType==t&&await a.setThreads(i)}isCreating=!1;async startCreates(){for(;this.isCreating;)await new Promise(t=>setTimeout(t,100));try{this.isCreating=!0;const t=await this.fileRepository.findBy({uid:this.uid,upload_status:p.UploadStatus.SUBMIT_READY});if(!t.length)return;for(const i of t)await this.onPartSuccess({file_id:i.id})}finally{this.isCreating=!1}}async startHost(t,i){if(this.processorMap.has(t)){const i=this.processorMap.get(t),e=this.getThreadCount(i.hostType);return void await i.setThreads(e)}const e={uid:this.uid,sdk_domain:this.params.sdk_domain,dataSource:this.dataSource,host_id:t,onProgress:async t=>{await this.onFileProgress(t)},onPartSuccess:async t=>{await this.onPartSuccess({file_id:t})},onError:async(t,i)=>{await this.onFileError(t,i)},onDequeue:async t=>{await this.dequeueOneFile(t)}};if(i==p.HostType.BAIDU){const i=new N(e);this.processorMap.set(t,i);const a=this.getThreadCount(p.HostType.BAIDU);await i.setThreads(a)}else if(i==p.HostType.ALI){const i=new x(e);this.processorMap.set(t,i);const a=this.getThreadCount(p.HostType.ALI);await i.setThreads(a)}}async startBaidu(){const t=(await this.fileRepository.createQueryBuilder("file").select("DISTINCT file.baidu_host_id").where("file.uid = :uid",{uid:this.uid}).getRawMany()).map(t=>t.baidu_host_id).filter(t=>!!t);if(t.length)for(const i of t)await this.startHost(i,p.HostType.BAIDU)}async startAli(){const t=(await this.fileRepository.createQueryBuilder("file").select("DISTINCT file.ali_host_id").where("file.uid = :uid",{uid:this.uid}).getRawMany()).map(t=>t.ali_host_id).filter(t=>!!t);if(t.length)for(const i of t)await this.startHost(i,p.HostType.ALI)}async onPartSuccess({file_id:t}){await this.partRepository.findOneBy({file_id:t,upload_status:(0,l.Not)(p.NormalStatus.SUCCESS)})||(await this.fileRepository.update(t,{upload_status:p.UploadStatus.SUBMIT_READY}),await this.onFileSuccess(t))}async onFileSuccess(t){const i=await this.fileRepository.findOneBy({id:t});if(i.upload_status!=p.UploadStatus.SUBMIT_READY)return;if(!i)return;const{uid:e}=this;for(;J.get(e);)await new Promise(t=>setTimeout(t,100));try{J.set(e,!0);const a=await this.partRepository.findBy({file_id:i.id}),s={},o=i.progress;if(i.baidu_host_id){s.baidu_upload_result={};for(const t of a)t.host_type==p.HostType.BAIDU&&(s.baidu_upload_result[t.info.file]={...t.upload_result.baidu,md4:t.info.md4});o[Y.UploadProgress.upload_baidu].percent=1}if(i.ali_host_id){s.ali_upload_result={};for(const t of a)t.host_type==p.HostType.ALI&&(s.ali_upload_result[t.info.file]={...t.upload_result.ali,sha1:t.info.sha1});o[Y.UploadProgress.upload_ali].percent=1}const r=(0,$.getProgressPercent)(o);await this.fileRepository.update(i.id,{...s,progress:o,percent:r,upload_status:p.UploadStatus.SUBMIT_READY}),await async function({file_id:t,sdk_domain:i,fileRepository:e}){try{for(;w.get(t);)await new Promise(t=>setTimeout(t,100));w.set(t,!0);const a=await e.findOneBy({id:t});let s=a.task_record_id;const o=(0,f.getSdk)(i),{encoded:r,baidu_host_id:d,ali_host_id:n}=a,{ali:u,baidu:p,meta:l}=r,c=[];if(d&&c.push(_.HostType.BAIDU),n&&c.push(_.HostType.ALI),!c.length)return;const g=t=>{const i=r[t];if(!i.media?.subtitles?.length)return;const e=[];return i.media.subtitles.forEach(t=>{const{uuid:i,lang:a,name:s,title:o,builtin:r}=t;e.push({uuid:i,lang:a,name:s,title:o,builtin:r})}),e},R=t=>{if(!t)return;if(!t.subtitles)return t;const{subtitles:i,...e}=t,a={};return i.forEach(t=>{const{uuid:i,file:e,start:s,end:o,size:r,source:d}=t;a[i]={file:e,start:s,end:o,size:r,source:d}}),{...e,subtitles:a}};if(a.is_attachment){const t=await o.getRecordInfo({space_id:a.space_id,record_id:a.task_record_id,refresh:!0}),i=Object.assign({version:y,metas:{},hosts:{},attachments:[]},t.manifest);t.type,_.RecordType.FOLDER;const e={metas:l,uuid:(0,h.v4)(),type:a.type,ftype:a.ftype,title:a.input.filename,subtitles:g(c[0])};if(d){const{parts:t,max_size:i,media:s,img:o,txt:r,source:n}=p;e.baidu={host_id:d,media:R(s),max_size:i,img:o,txt:r,source:n,files:{}};const u=a.baidu_upload_result;t.forEach(t=>{const{file:i,md4:a,preview:s,source:o,size:r,md5:d}=t,{fs_id:n}=u[i];e.baidu.files[i]={fs_id:n,md5:d,md4:a,size:r,preview:s,source:o}})}if(n){const{parts:t,max_size:i,media:s,img:o,txt:r,source:d}=u,{drive_id:p}=a.cloud_info.hosts.ali;e.ali={host_id:n,drive_id:p,media:R(s),max_size:i,img:o,txt:r,source:d,files:{}};const l=a.ali_upload_result;t.forEach(t=>{const{file:i,preview:a,source:s,size:o,md5:r}=t,{file_id:d,sha1:n}=l[i];e.ali.files[i]={file_id:d,md5:r,sha1:n,size:o,preview:a,source:s}})}i.attachments=i.attachments||[],i.attachments.push(e),i.hosts||(i.hosts={}),d&&(i.hosts.baidu=(0,m.calculateManifestHostSize)(i,_.HostType.BAIDU)),n&&(i.hosts.ali=(0,m.calculateManifestHostSize)(i,_.HostType.ALI)),await o.updateRecord({space_id:t.space_id,record_id:t.id,parent_id:t.parent_id,manifest:JSON.stringify(i)})}else{0!==a.pid&&(s=(await e.findOneBy({id:a.pid})).cloud_info.id);const t={version:y,hosts:{},complete:!0,metas:l,subtitles:g(c[0])};if(d){const{parts:i,media:e,...s}=p,o=R(e),r=a.baidu_upload_result;t.baidu={host_id:d,...s,files:{},media:o},i.forEach(i=>{const{file:e,md4:a,preview:s,source:o,size:d,md5:n}=i,{fs_id:u}=r[e];t.baidu.files[e]={fs_id:u,md5:n,md4:a,size:d,preview:s,source:o}})}if(n){const{parts:i,media:e,...s}=u,o=R(e),r=a.ali_upload_result;t.ali={host_id:n,drive_id:"",...s,media:o,files:{}},i.forEach(i=>{const{file:e,preview:a,source:s,size:o,md5:d}=i,{file_id:n,drive_id:u,sha1:p}=r[e];t.ali.drive_id||(t.ali.drive_id=u),t.ali.files[e]={file_id:n,sha1:p,md5:d,size:o,preview:a,source:s}});const d=Object.values(t.ali.files).reduce((t,i)=>t+i.size,0);t.hosts.ali=d}if(d&&(t.hosts.baidu=(0,m.calculateManifestHostSize)(t,_.HostType.BAIDU)),n&&(t.hosts.ali=(0,m.calculateManifestHostSize)(t,_.HostType.ALI)),a.cloud_info?.id){const t=await o.getRecordInfo({space_id:a.space_id,record_id:a.cloud_info.id,refresh:!0});if(t.manifest?.complete)throw new Error("the file has been modified!")}await o.updateRecord({space_id:a.space_id,record_id:a.cloud_info.id,parent_id:s,manifest:JSON.stringify(t)})}}catch(t){throw t}finally{w.delete(t)}}({file_id:i.id,fileRepository:this.fileRepository,sdk_domain:this.params.sdk_domain}),this.params.onSuccess&&await this.params.onSuccess(t),await async function(t){const{fileRepository:i,partRepository:e,successRepository:a}=t,s=await i.findOneBy({id:t.file_id}),o={is_attachment:s.is_attachment,root_id:s.root_id,pid:0,space_id:s.space_id,space_name:s.space_name,uid:s.uid,input:s.input,cloud_id:s.cloud_info.id,cloud_name:s.cloud_info.name,type:s.type},r=0==s.root_id;if(!r){const e=await a.findOneBy({root_id:s.root_id,pid:0});if(s.is_attachment){const r=(0,g.getSdk)(t.sdk_domain);if(e)o.pid=e.id;else{const t=await r.getRecordInfo({space_id:s.space_id,record_id:s.task_record_id,refresh:!1}),e=await i.findOneBy({id:s.root_id}),d=await a.save(a.create({is_attachment:!0,root_id:s.root_id,pid:0,space_id:t.space_id,space_name:s.space_name,uid:s.uid,input:e.input,cloud_id:t.id,cloud_name:t.name,type:t.type}));o.pid=d.id}}else if(e)o.pid=e.id;else{const t=await i.findOneBy({id:s.root_id}),e=await a.save(a.create({is_attachment:!1,root_id:s.root_id,pid:0,space_id:t.space_id,space_name:t.space_name,uid:t.uid,input:t.input,cloud_id:t.cloud_info.id,cloud_name:t.cloud_info.name,type:t.type}));o.pid=e.id}await a.update({id:o.pid},{updated_at:new Date})}await a.save(a.create(o)),await i.delete({id:t.file_id}),await e.delete({file_id:t.file_id}),await(0,S.remove)(s.output_root),r||(await i.findOneBy({root_id:s.root_id,type:(0,b.Not)(R.RecordType.FOLDER)})?await i.increment({id:s.root_id},"completed_count",1):(await i.delete({root_id:s.root_id}),await i.delete({id:s.root_id})))}({file_id:i.id,sdk_domain:this.params.sdk_domain,fileRepository:this.fileRepository,partRepository:this.partRepository,successRepository:this.successRepository})}catch(t){await this.fileRepository.update(i.id,{upload_status:p.UploadStatus.UPLOAD_ERROR,error:{type:"upload",message:t.message,details:t.details,cause:t.cause,stack:t.stack}})}finally{J.delete(e)}}async onFileError(t,i){try{this.params.onError&&await this.params.onError(t,i)}catch(t){}}async onFileProgress(t){try{this.params.onProgress&&await this.params.onProgress(t)}catch(t){}}async dequeueOneFile(t){for(;this.isDequeueing;)await new Promise(t=>{setTimeout(t,50)});let i=null;try{this.isDequeueing=!0;const e=`${t}_host_id`;if(i=await this.fileRepository.findOne({where:{uid:this.params.uid,is_paused:!1,upload_status:p.UploadStatus.UPLOAD_READY,[e]:(0,l.Not)((0,l.IsNull)())},order:{id:"ASC"}}),!i)return;if(i.cloud_info?.id||(await U({file_id:i.id,sdk_domain:this.params.sdk_domain,fileRepository:this.fileRepository}),i=await this.fileRepository.findOneBy({id:i.id})),!i.cloud_info?.id)throw new Error("Created cloud folder or file failed");const{encoded:a}=i,s=[],o=[p.HostType.BAIDU,p.HostType.ALI];for await(const t of o){const e=`${t}_host_id`;if(!i[e])continue;const o=i[e],r=a[t].parts||[],{length:d}=r;for(let e=0;e<d;e++){const a=r[e];await this.partRepository.findOneBy({file_id:i.id,part_name:a.file,host_id:o})||s.push({uid:this.params.uid,file_id:i.id,root_id:i.root_id??i.id,part_name:a.file,info:a,output_root:i.output_root,host_id:o,host_type:t})}}if(s.length)await this.partRepository.save(s),await this.fileRepository.update(i.id,{upload_status:p.UploadStatus.UPLOAD_PROCESS});else{await this.fileRepository.update(i.id,{upload_status:p.UploadStatus.UPLOAD_PROCESS});const e=await this.partRepository.findOneBy({file_id:i.id,upload_status:(0,l.Not)(p.NormalStatus.SUCCESS)});await this.partRepository.findOneBy({file_id:i.id,upload_status:p.NormalStatus.SUCCESS});e||await this.onPartSuccess({file_id:i.id});await this.partRepository.findOneBy({file_id:i.id,upload_status:p.NormalStatus.NULL})||(this.isDequeueing=!1,await this.dequeueOneFile(t))}}catch(e){if(!i)throw e;{const a=(0,j.buildDError)(e,{message:"Error occurred during baidu preparing",detail:`Error occurred during baidu preparing: ${i.input.filepath}`});await this.fileRepository.update(i.id,{upload_status:p.UploadStatus.UPLOAD_ERROR,error:{type:"upload",message:a.message,details:a.details,cause:a.cause,stack:a.stack}}),this.isDequeueing=!1,await this.dequeueOneFile(t)}}finally{this.isDequeueing=!1}}getThreadCount(t){return this.threadConfig[t]??2}},V=new Map,G=async t=>{for(;V.get(t.uid)?.initing;)await new Promise(t=>setTimeout(t,100));try{if(V.get(t.uid)?.uploader)return V.get(t.uid).uploader;const i={uploader:new Q(t),initing:!0};return V.set(t.uid,i),await i.uploader.init(),i.uploader}finally{V.get(t.uid)&&(V.get(t.uid).initing=!1)}};

package/dist/main.mjs

@@ -1 +1 @@
-var i=(i=>"undefined"!=typeof require?require:"undefined"!=typeof Proxy?new Proxy(i,{get:(i,t)=>("undefined"!=typeof require?require:i)[t]}):i)(function(i){if("undefined"!=typeof require)return require.apply(this,arguments);throw Error('Dynamic require of "'+i+'" is not supported')});import{NormalStatus as t,HostType as s,UploadStatus as e}from"@soga/types";import{IsNull as a,Not as o}from"typeorm";import{UploadFile as r,UploadPart as d,UploadRunning as n,UploadRunningType as p,UploadSuccess as u}from"@soga/entities";import{getSdk as l}from"@soga/sdk";import{HostType as c,RecordType as f}from"@soga/types";import{v4 as h}from"uuid";var _="1.0.0";import{calculateManifestHostSize as m}from"@soga/utils";var y=new Map;import{getSdk as w}from"@soga/sdk";import{RecordType as g}from"@soga/types";import{remove as R}from"fs-extra";import{Not as b}from"typeorm";import{RecordType as P}from"@soga/types";import{getSdk as S}from"@soga/sdk";async function O({file_id:i,sdk_domain:t,fileRepository:s}){const e=await s.findOneBy({id:i});if(e.is_attachment){const{task_record_id:i}=e,a=S(t),r=await a.getRecordInfo({space_id:e.space_id,record_id:i,refresh:!0}),d=await o(r);await s.update(e.id,{cloud_info:d})}else await async function(){const t=await s.findOneBy({id:i});if(!t)return;const e=await a(t.pid,[]);for(const i of e)await r(i);return e}(),await r(e);async function a(i,t=[]){try{if(0===i)return t;const e=await s.findOneBy({id:i});return e?e.cloud_info?.id?t:(t.unshift(e),0!==e.pid?await a(e.pid,t).catch(i=>{throw i}):t):t}catch(i){throw i}}async function o(i){const{id:t,name:a,cloud_info:o}=i,r={id:t,name:a,hosts:{}};if(o.ali&&(r.hosts.ali={id:o.ali.id,name:o.ali.name,drive_id:o.ali.drive_id,file_id:o.ali.file_id}),o.baidu&&(r.hosts.baidu={id:o.baidu.id,name:o.baidu.name,fs_id:o.baidu.fs_id,path:o.baidu.path},!r.hosts.baidu.path)){const i=await s.findOneBy({id:e.pid}),{path:t}=i.cloud_info.hosts.baidu;t&&(r.hosts.baidu.path=`${t}/${o.baidu.name}`)}return r}async function r(i){const e=await s.findOneBy({id:i.id});if(e.cloud_info?.id)return;const{input:a,space_id:r,task_record_id:d,type:n}=e,p=n===P.FOLDER;let u=d;if(0!==e.pid){u=(await s.findOneBy({id:e.pid})).cloud_info.id}const l=S(t),c={space_id:r,name:a.filename,parent_id:u,type:e.type,ftype:e.ftype},f={};e.baidu_host_id&&(f.baidu=0),e.ali_host_id&&(f.ali=0);const h=await l.createRecord({...c,manifest:JSON.stringify({version:_,complete:!!p,hosts:f,metas:{file:{btime:e.input.local_btime,ctime:e.input.local_ctime,mtime:e.input.local_mtime,file:e.input.filename}}})}),m=await o(h);await s.update({id:e.id},{cloud_info:m})}}import{UploadFile as E,UploadPart as A,UploadRunning as B,UploadRunningType as U}from"@soga/entities";import{NormalStatus as D,UploadStatus as C}from"@soga/types";import{getProgressPercent as T}from"@soga/utils";import{UploadProgress as k}from"@soga/node-types";var v=class{hostType;onProgress=async()=>{};fileRepository;partRepository;runningRepository;isRunning=!1;thread_count=2;maxThreads=8;threads=[];baseParams;getValidThreads(i){return Math.min(Math.max(i??0,0),this.maxThreads)}async onPartProgress(i){const{id:t,data:s,type:e}=i;if("percent"!=e)return;const{part_id:a,percent:o}=s,r=await this.partRepository.findOneBy({id:a});if(!r)return;if(r.upload_percent>=o)return;if(r.upload_status==D.ERROR)return;if(r.upload_status==D.SUCCESS)return;await this.partRepository.update(a,{upload_percent:o});const d=await this.partRepository.findBy({file_id:t,host_type:this.hostType});let n=0,p=0;for(const i of d)n+=i.info.size,p+=i.info.size*i.upload_percent;const u=p/n,l=await this.fileRepository.findOneBy({id:t}),c=k[`upload_${this.hostType}`];l.progress[c].percent=u,l.percent=T(l.progress),await this.fileRepository.update(t,{progress:l.progress,percent:l.percent}),await this.onProgress(l)}async onPartError({error:i,part_id:t,file_id:s}){const e=await this.fileRepository.findOneBy({id:s});e&&(await this.fileRepository.update(s,{upload_status:C.UPLOAD_ERROR,error:{type:"upload",message:i.message,details:i.details,cause:i.cause,stack:i.stack}}),await this.partRepository.update(t,{upload_status:D.ERROR}),await this.fileRepository.update(e.id,{upload_status:C.UPLOAD_ERROR}),await this.baseParams.onError(i,e.id))}constructor(i,t){this.hostType=t,this.baseParams=i;const{dataSource:s}=i;this.fileRepository=s.getRepository(E),this.partRepository=s.getRepository(A),this.runningRepository=s.getRepository(B),i.onProgress&&(this.onProgress=i.onProgress.bind(this))}async setThreads(i){const t=this.getValidThreads(i);this.thread_count=t,await this.start()}async getParts(i){const t={uid:this.baseParams.uid,is_paused:!1,upload_status:D.NULL,host_id:this.baseParams.host_id};for(let s=0;s<i;s++){const s=await this.partRepository.countBy(t);if(s>=i)break;if(s<i){await this.baseParams.onDequeue(this.hostType);if(await this.partRepository.countBy(t)==s)break}}return await this.partRepository.find({where:t,order:{id:"ASC"},take:i})}async start(){for(;this.isRunning;)await new Promise(i=>{setTimeout(i,50)});try{this.isRunning=!0;const i=this.threads.length,t=this.thread_count;if(i<t){const s=t-i,e=await this.getParts(s),a=e.length;for(let i=0;i<a;i++){const t=e[i],s=this.getThread(t);(await this.fileRepository.findOneBy({id:t.file_id})).is_paused||(this.threads.push(s),await this.partRepository.update(t.id,{upload_status:D.PROCESS}),s.start().catch(async i=>{}))}}else if(i>t){const s=i-t,e=this.threads.slice(0,s);for(const i of e)await i.stop()}}finally{this.isRunning=!1}}async stopFiles(i){const t=this.threads.filter(t=>i.includes(t.file_id));await Promise.all(t.map(i=>i.stop()))}async stopAll(){await Promise.all(this.threads.map(i=>i.stop()))}async addRunning({file_id:i,part_id:t,root_id:s}){await this.runningRepository.save(this.runningRepository.create({type:U.UPLOAD,uid:this.baseParams.uid,root_id:s,file_id:i,part_id:t}))}async deleteRunning({file_id:i,part_id:t}){await this.runningRepository.delete({uid:this.baseParams.uid,file_id:i,part_id:t})}};import{HostType as L,NormalStatus as I,UploadStatus as q}from"@soga/types";import M from"piscina";import{MessageChannel as x}from"worker_threads";import{buildDError as z}from"@soga/error";var F=new M({filename:i.resolve("@soga/part-uploader")}),N=class extends v{params;constructor(i){super(i,L.BAIDU),this.params=i}getThread(i){const t=new AbortController,{port1:s,port2:e}=new x;return{file_id:i.file_id,part_id:i.id,uid:i.uid,start:async()=>{let a=null;try{if(a=await this.fileRepository.findOneBy({id:i.file_id}),!a)return;const{file_id:o,host_id:r,id:d,info:n,upload_status:p,root_id:u}=i;if(p==I.SUCCESS)return;await this.addRunning({file_id:o,part_id:d,root_id:u});const{output_root:l}=a;e.on("message",async i=>{await this.onPartProgress(i)});const c={file_id:o,host_id:r,part_id:d,output_root:l,part:n,cloud_folder_path:a.cloud_info.hosts.baidu?.path,sdk_domain:this.params.sdk_domain,port:s,appid:30045106},f=await F.run(c,{name:"uploadBaidu",signal:t.signal,transferList:[s]});if(!f?.fs_id)throw new Error("Upload to Baidu failed, result is null");await this.partRepository.update(i.id,{upload_result:{baidu:f},upload_status:I.SUCCESS}),await this.baseParams.onPartSuccess(o)}catch(s){if(t.abort(),"AbortError"==s.name)await this.partRepository.update(i.id,{upload_status:I.NULL});else{const t=a?a.input.filepath:"unknown filepath",e=z(s,{message:"Error occurred during baidu uploading",detail:`Error occurred during baidu uploading: ${t}`});await this.fileRepository.update(i.file_id,{upload_status:q.UPLOAD_ERROR,error:{type:"upload",message:e.message,details:e.details,cause:e.cause,stack:e.stack}}),await this.onPartError({error:s,part_id:i.id,file_id:i.file_id})}}finally{this.threads=this.threads.filter(t=>t.part_id!=i.id),e.close(),await this.deleteRunning({file_id:i.file_id,part_id:i.id}),await this.start()}},stop:async()=>{t.abort(),this.threads=this.threads.filter(t=>t.part_id!==i.id)}}}};import{HostType as $,NormalStatus as H,UploadStatus as Y}from"@soga/types";import J from"piscina";import{MessageChannel as j}from"worker_threads";import{buildDError as Q}from"@soga/error";var V=new J({filename:i.resolve("@soga/part-uploader")}),G=class extends v{params;constructor(i){super(i,$.ALI),this.params=i}getThread(i){const t=new AbortController,{port1:s,port2:e}=new j;return{file_id:i.file_id,part_id:i.id,uid:i.uid,start:async()=>{const a=await this.fileRepository.findOneBy({id:i.file_id});if(a)try{const{file_id:o,host_id:r,id:d,info:n,upload_status:p,root_id:u}=i;if(p==H.SUCCESS)return;await this.addRunning({file_id:o,part_id:d,root_id:u});const{output_root:l}=a;e.on("message",async i=>{await this.onPartProgress(i)});const c={file_id:o,host_id:r,part_id:d,output_root:l,part:n,cloud_folder_id:a.cloud_info.hosts.ali?.file_id,sdk_domain:this.params.sdk_domain,port:s},f=await V.run(c,{name:"uploadAli",signal:t.signal,transferList:[s]});if(!f?.file_id)throw new Error("Upload to Ali failed, result is null");await this.partRepository.update(i.id,{upload_result:{ali:f},upload_status:H.SUCCESS}),await this.baseParams.onPartSuccess(o)}catch(s){if(t.abort(),"AbortError"==s.name)this.partRepository.update(i.id,{upload_status:H.NULL});else{const t=Q(s,{message:"Error occurred during ali uploading",detail:`Error occurred during ali uploading: ${a.input.filepath}`});await this.fileRepository.update(i.file_id,{upload_status:Y.UPLOAD_ERROR,error:{type:"upload",message:t.message,details:t.details,cause:t.cause,stack:t.stack}}),await this.onPartError({error:s,part_id:i.id,file_id:i.file_id})}}finally{this.threads=this.threads.filter(t=>t.part_id!=i.id),e.close(),await this.deleteRunning({file_id:i.file_id,part_id:i.id}),await this.start()}},stop:async()=>{t.abort(),this.threads=this.threads.filter(t=>t.part_id!==i.id)}}}};import{buildDError as K}from"@soga/error";import{getProgressPercent as W}from"@soga/utils";import{UploadProgress as X}from"@soga/node-types";var Z=new Map,ii=class{params;uid;dataSource;fileRepository;partRepository;runningRepository;successRepository;processorMap=new Map;isDequeueing=!1;threadConfig={baidu:4,ali:2};constructor(i){this.params=i,this.uid=i.uid,this.dataSource=i.dataSource,this.fileRepository=this.dataSource.getRepository(r),this.partRepository=this.dataSource.getRepository(d),this.runningRepository=this.dataSource.getRepository(n),this.successRepository=this.dataSource.getRepository(u)}async init(){await this.runningRepository.delete({uid:this.uid,type:p.UPLOAD}),await this.fileRepository.update({uid:this.uid,upload_status:e.UPLOAD_ERROR},{upload_status:e.UPLOAD_READY}),await this.partRepository.update({uid:this.uid,upload_status:t.PROCESS},{upload_status:t.NULL})}async start(){await this.startCreates(),await Promise.all([this.startBaidu(),this.startAli()]).catch(i=>{console.error("Error starting uploader:",i)})}async stopFiles(i){const t=this.processorMap.values();for(const s of t)await s.stopFiles(i)}async stopAll(){const i=this.processorMap.values();for(const t of i)await t.stopAll()}async setThreadCount(i,t){this.threadConfig[i]=t;const s=this.processorMap.values();for(const e of s)e.hostType==i&&await e.setThreads(t)}isCreating=!1;async startCreates(){for(;this.isCreating;)await new Promise(i=>setTimeout(i,100));try{this.isCreating=!0;const i=await this.fileRepository.findBy({uid:this.uid,upload_status:e.SUBMIT_READY});if(!i.length)return;for(const t of i)await this.onPartSuccess({file_id:t.id})}finally{this.isCreating=!1}}async startHost(i,t){if(this.processorMap.has(i)){const t=this.processorMap.get(i),s=this.getThreadCount(t.hostType);return void await t.setThreads(s)}const e={uid:this.uid,sdk_domain:this.params.sdk_domain,dataSource:this.dataSource,host_id:i,onProgress:async i=>{await this.onFileProgress(i)},onPartSuccess:async i=>{await this.onPartSuccess({file_id:i})},onError:async(i,t)=>{await this.onFileError(i,t)},onDequeue:async i=>{await this.dequeueOneFile(i)}};if(t==s.BAIDU){const t=new N(e);this.processorMap.set(i,t);const a=this.getThreadCount(s.BAIDU);await t.setThreads(a)}else if(t==s.ALI){const t=new G(e);this.processorMap.set(i,t);const a=this.getThreadCount(s.ALI);await t.setThreads(a)}}async startBaidu(){const i=(await this.fileRepository.createQueryBuilder("file").select("DISTINCT file.baidu_host_id").where("file.uid = :uid",{uid:this.uid}).getRawMany()).map(i=>i.baidu_host_id).filter(i=>!!i);if(i.length)for(const t of i)await this.startHost(t,s.BAIDU)}async startAli(){const i=(await this.fileRepository.createQueryBuilder("file").select("DISTINCT file.ali_host_id").where("file.uid = :uid",{uid:this.uid}).getRawMany()).map(i=>i.ali_host_id).filter(i=>!!i);if(i.length)for(const t of i)await this.startHost(t,s.ALI)}async onPartSuccess({file_id:i}){await this.partRepository.findOneBy({file_id:i,upload_status:o(t.SUCCESS)})||await this.onFileSuccess(i)}async onFileSuccess(i){try{const{uid:t}=this;for(;Z.get(t);)await new Promise(i=>setTimeout(i,100));Z.set(t,!0);const a=await this.fileRepository.findOneBy({id:i});if(!a)return;try{const t=await this.partRepository.findBy({file_id:a.id}),o={},r=a.progress;if(a.baidu_host_id){o.baidu_upload_result={};for(const i of t)i.host_type==s.BAIDU&&(o.baidu_upload_result[i.info.file]={...i.upload_result.baidu,md4:i.info.md4});r[X.upload_baidu].percent=1}if(a.ali_host_id){o.ali_upload_result={};for(const i of t)i.host_type==s.ALI&&(o.ali_upload_result[i.info.file]={...i.upload_result.ali,sha1:i.info.sha1});r[X.upload_ali].percent=1}const d=W(r);await this.fileRepository.update(a.id,{...o,progress:r,percent:d,upload_status:e.SUBMIT_READY}),await async function({file_id:i,sdk_domain:t,fileRepository:s}){try{for(;y.get(i);)await new Promise(i=>setTimeout(i,100));y.set(i,!0);const e=await s.findOneBy({id:i});let a=e.task_record_id;const o=l(t),{encoded:r,baidu_host_id:d,ali_host_id:n}=e,{ali:p,baidu:u,meta:w}=r,g=[];if(d&&g.push(c.BAIDU),n&&g.push(c.ALI),!g.length)return;const R=i=>{const t=r[i];if(!t.media?.subtitles?.length)return;const s=[];return t.media.subtitles.forEach(i=>{const{uuid:t,lang:e,name:a,title:o,builtin:r}=i;s.push({uuid:t,lang:e,name:a,title:o,builtin:r})}),s},b=i=>{if(!i)return;if(!i.subtitles)return i;const{subtitles:t,...s}=i,e={};return t.forEach(i=>{const{uuid:t,file:s,start:a,end:o,size:r,source:d}=i;e[t]={file:s,start:a,end:o,size:r,source:d}}),{...s,subtitles:e}};if(e.is_attachment){const i=await o.getRecordInfo({space_id:e.space_id,record_id:e.task_record_id,refresh:!0}),t=Object.assign({version:_,metas:{},hosts:{},attachments:[]},i.manifest);i.type,f.FOLDER;const s={metas:w,uuid:h(),type:e.type,ftype:e.ftype,title:e.input.filename,subtitles:R(g[0])};if(d){const{parts:i,max_size:t,media:a,img:o,txt:r,source:n}=u;s.baidu={host_id:d,media:b(a),max_size:t,img:o,txt:r,source:n,files:{}};const p=e.baidu_upload_result;i.forEach(i=>{const{file:t,md4:e,preview:a,source:o,size:r,md5:d}=i,{fs_id:n}=p[t];s.baidu.files[t]={fs_id:n,md5:d,md4:e,size:r,preview:a,source:o}})}if(n){const{parts:i,max_size:t,media:a,img:o,txt:r,source:d}=p,{drive_id:u}=e.cloud_info.hosts.ali;s.ali={host_id:n,drive_id:u,media:b(a),max_size:t,img:o,txt:r,source:d,files:{}};const l=e.ali_upload_result;i.forEach(i=>{const{file:t,preview:e,source:a,size:o,md5:r}=i,{file_id:d,sha1:n}=l[t];s.ali.files[t]={file_id:d,md5:r,sha1:n,size:o,preview:e,source:a}})}t.attachments=t.attachments||[],t.attachments.push(s),t.hosts||(t.hosts={}),d&&(t.hosts.baidu=m(t,c.BAIDU)),n&&(t.hosts.ali=m(t,c.ALI)),await o.updateRecord({space_id:i.space_id,record_id:i.id,parent_id:i.parent_id,manifest:JSON.stringify(t)})}else{0!==e.pid&&(a=(await s.findOneBy({id:e.pid})).cloud_info.id);const i={version:_,hosts:{},complete:!0,metas:w,subtitles:R(g[0])};if(d){const{parts:t,media:s,...a}=u,o=b(s),r=e.baidu_upload_result;i.baidu={host_id:d,...a,files:{},media:o},t.forEach(t=>{const{file:s,md4:e,preview:a,source:o,size:d,md5:n}=t,{fs_id:p}=r[s];i.baidu.files[s]={fs_id:p,md5:n,md4:e,size:d,preview:a,source:o}})}if(n){const{parts:t,media:s,...a}=p,o=b(s),r=e.ali_upload_result;i.ali={host_id:n,drive_id:"",...a,media:o,files:{}},t.forEach(t=>{const{file:s,preview:e,source:a,size:o,md5:d}=t,{file_id:n,drive_id:p,sha1:u}=r[s];i.ali.drive_id||(i.ali.drive_id=p),i.ali.files[s]={file_id:n,sha1:u,md5:d,size:o,preview:e,source:a}});const d=Object.values(i.ali.files).reduce((i,t)=>i+t.size,0);i.hosts.ali=d}if(d&&(i.hosts.baidu=m(i,c.BAIDU)),n&&(i.hosts.ali=m(i,c.ALI)),e.cloud_info?.id){const i=await o.getRecordInfo({space_id:e.space_id,record_id:e.cloud_info.id,refresh:!0});if(i.manifest?.complete)throw new Error("the file has been modified!")}await o.updateRecord({space_id:e.space_id,record_id:e.cloud_info.id,parent_id:a,manifest:JSON.stringify(i)})}}catch(i){throw i}finally{y.delete(i)}}({file_id:a.id,fileRepository:this.fileRepository,sdk_domain:this.params.sdk_domain}),this.params.onSuccess&&await this.params.onSuccess(i),await async function(i){const{fileRepository:t,partRepository:s,successRepository:e}=i,a=await t.findOneBy({id:i.file_id}),o={is_attachment:a.is_attachment,root_id:a.root_id,pid:0,space_id:a.space_id,space_name:a.space_name,uid:a.uid,input:a.input,cloud_id:a.cloud_info.id,cloud_name:a.cloud_info.name,type:a.type},r=0==a.root_id;if(!r){const s=await e.findOneBy({root_id:a.root_id,pid:0});if(a.is_attachment){const r=w(i.sdk_domain);if(s)o.pid=s.id;else{const i=await r.getRecordInfo({space_id:a.space_id,record_id:a.task_record_id,refresh:!1}),s=await t.findOneBy({id:a.root_id}),d=await e.save(e.create({is_attachment:!0,root_id:a.root_id,pid:0,space_id:i.space_id,space_name:a.space_name,uid:a.uid,input:s.input,cloud_id:i.id,cloud_name:i.name,type:i.type}));o.pid=d.id}}else if(s)o.pid=s.id;else{const i=await t.findOneBy({id:a.root_id}),s=await e.save(e.create({is_attachment:!1,root_id:a.root_id,pid:0,space_id:i.space_id,space_name:i.space_name,uid:i.uid,input:i.input,cloud_id:i.cloud_info.id,cloud_name:i.cloud_info.name,type:i.type}));o.pid=s.id}await e.update({id:o.pid},{updated_at:new Date})}await e.save(e.create(o)),await t.delete({id:i.file_id}),await s.delete({file_id:i.file_id}),await R(a.output_root),r||(await t.findOneBy({root_id:a.root_id,type:b(g.FOLDER)})?await t.increment({id:a.root_id},"completed_count",1):(await t.delete({root_id:a.root_id}),await t.delete({id:a.root_id})))}({file_id:a.id,sdk_domain:this.params.sdk_domain,fileRepository:this.fileRepository,partRepository:this.partRepository,successRepository:this.successRepository})}catch(i){await this.fileRepository.update(a.id,{upload_status:e.UPLOAD_ERROR,error:{type:"upload",message:i.message,details:i.details,cause:i.cause,stack:i.stack}})}finally{Z.delete(t)}}catch(i){}}async onFileError(i,t){try{this.params.onError&&await this.params.onError(i,t)}catch(i){}}async onFileProgress(i){try{this.params.onProgress&&await this.params.onProgress(i)}catch(i){}}async dequeueOneFile(i){for(;this.isDequeueing;)await new Promise(i=>{setTimeout(i,50)});let r=null;try{this.isDequeueing=!0;const d=`${i}_host_id`;if(r=await this.fileRepository.findOne({where:{uid:this.params.uid,is_paused:!1,upload_status:e.UPLOAD_READY,[d]:o(a())},order:{id:"ASC"}}),!r)return;if(r.cloud_info?.id||(await O({file_id:r.id,sdk_domain:this.params.sdk_domain,fileRepository:this.fileRepository}),r=await this.fileRepository.findOneBy({id:r.id})),!r.cloud_info?.id)throw new Error("Created cloud folder or file failed");const{encoded:n}=r,p=[],u=[s.BAIDU,s.ALI];for await(const i of u){const t=`${i}_host_id`;if(!r[t])continue;const s=r[t],e=n[i].parts||[],{length:a}=e;for(let t=0;t<a;t++){const a=e[t];await this.partRepository.findOneBy({file_id:r.id,part_name:a.file,host_id:s})||p.push({uid:this.params.uid,file_id:r.id,root_id:r.root_id??r.id,part_name:a.file,info:a,output_root:r.output_root,host_id:s,host_type:i})}}if(p.length)await this.partRepository.save(p),await this.fileRepository.update(r.id,{upload_status:e.UPLOAD_PROCESS});else{await this.fileRepository.update(r.id,{upload_status:e.UPLOAD_PROCESS});const s=await this.partRepository.findOneBy({file_id:r.id,upload_status:o(t.SUCCESS)});await this.partRepository.findOneBy({file_id:r.id,upload_status:t.SUCCESS});s||await this.onPartSuccess({file_id:r.id});await this.partRepository.findOneBy({file_id:r.id,upload_status:t.NULL})||(this.isDequeueing=!1,await this.dequeueOneFile(i))}}catch(t){if(!r)throw t;{const s=K(t,{message:"Error occurred during baidu preparing",detail:`Error occurred during baidu preparing: ${r.input.filepath}`});await this.fileRepository.update(r.id,{upload_status:e.UPLOAD_ERROR,error:{type:"upload",message:s.message,details:s.details,cause:s.cause,stack:s.stack}}),this.isDequeueing=!1,await this.dequeueOneFile(i)}}finally{this.isDequeueing=!1}}getThreadCount(i){return this.threadConfig[i]??2}},ti=new Map,si=async i=>{for(;ti.get(i.uid)?.initing;)await new Promise(i=>setTimeout(i,100));try{if(ti.get(i.uid)?.uploader)return ti.get(i.uid).uploader;const t={uploader:new ii(i),initing:!0};return ti.set(i.uid,t),await t.uploader.init(),t.uploader}finally{ti.get(i.uid)&&(ti.get(i.uid).initing=!1)}};export{ii as Uploader,si as getUploader};
+var i=(i=>"undefined"!=typeof require?require:"undefined"!=typeof Proxy?new Proxy(i,{get:(i,t)=>("undefined"!=typeof require?require:i)[t]}):i)(function(i){if("undefined"!=typeof require)return require.apply(this,arguments);throw Error('Dynamic require of "'+i+'" is not supported')});import{NormalStatus as t,HostType as s,UploadStatus as e}from"@soga/types";import{IsNull as a,Not as o}from"typeorm";import{UploadFile as r,UploadPart as d,UploadRunning as n,UploadRunningType as p,UploadSuccess as u}from"@soga/entities";import{getSdk as l}from"@soga/sdk";import{HostType as c,RecordType as f}from"@soga/types";import{v4 as _}from"uuid";var h="1.0.0";import{calculateManifestHostSize as m}from"@soga/utils";var y=new Map;import{getSdk as w}from"@soga/sdk";import{RecordType as g}from"@soga/types";import{remove as R}from"fs-extra";import{Not as b}from"typeorm";import{RecordType as S}from"@soga/types";import{getSdk as P}from"@soga/sdk";async function O({file_id:i,sdk_domain:t,fileRepository:s}){const e=await s.findOneBy({id:i});if(e.is_attachment){const{task_record_id:i}=e,a=P(t),r=await a.getRecordInfo({space_id:e.space_id,record_id:i,refresh:!0}),d=await o(r);await s.update(e.id,{cloud_info:d})}else await async function(){const t=await s.findOneBy({id:i});if(!t)return;const e=await a(t.pid,[]);for(const i of e)await r(i);return e}(),await r(e);async function a(i,t=[]){try{if(0===i)return t;const e=await s.findOneBy({id:i});return e?e.cloud_info?.id?t:(t.unshift(e),0!==e.pid?await a(e.pid,t).catch(i=>{throw i}):t):t}catch(i){throw i}}async function o(i){const{id:t,name:a,cloud_info:o}=i,r={id:t,name:a,hosts:{}};if(o.ali&&(r.hosts.ali={id:o.ali.id,name:o.ali.name,drive_id:o.ali.drive_id,file_id:o.ali.file_id}),o.baidu&&(r.hosts.baidu={id:o.baidu.id,name:o.baidu.name,fs_id:o.baidu.fs_id,path:o.baidu.path},!r.hosts.baidu.path)){const i=await s.findOneBy({id:e.pid}),{path:t}=i.cloud_info.hosts.baidu;t&&(r.hosts.baidu.path=`${t}/${o.baidu.name}`)}return r}async function r(i){const e=await s.findOneBy({id:i.id});if(e.cloud_info?.id)return;const{input:a,space_id:r,task_record_id:d,type:n}=e,p=n===S.FOLDER;let u=d;if(0!==e.pid){u=(await s.findOneBy({id:e.pid})).cloud_info.id}const l=P(t),c={space_id:r,name:a.filename,parent_id:u,type:e.type,ftype:e.ftype},f={};e.baidu_host_id&&(f.baidu=0),e.ali_host_id&&(f.ali=0);const _=await l.createRecord({...c,manifest:JSON.stringify({version:h,complete:!!p,hosts:f,metas:{file:{btime:e.input.local_btime,ctime:e.input.local_ctime,mtime:e.input.local_mtime,file:e.input.filename}}})}),m=await o(_);await s.update({id:e.id},{cloud_info:m})}}import{UploadFile as E,UploadPart as A,UploadRunning as B,UploadRunningType as U}from"@soga/entities";import{NormalStatus as D,UploadStatus as T}from"@soga/types";import{getProgressPercent as C}from"@soga/utils";import{UploadProgress as k}from"@soga/node-types";var v=class{hostType;onProgress=async()=>{};fileRepository;partRepository;runningRepository;isRunning=!1;thread_count=2;maxThreads=8;threads=[];baseParams;getValidThreads(i){return Math.min(Math.max(i??0,0),this.maxThreads)}async onPartProgress(i){const{id:t,data:s,type:e}=i;if("percent"!=e)return;const{part_id:a,percent:o}=s,r=await this.partRepository.findOneBy({id:a});if(!r)return;if(r.upload_percent>=o)return;if(r.upload_status==D.ERROR)return;if(r.upload_status==D.SUCCESS)return;await this.partRepository.update(a,{upload_percent:o});const d=await this.partRepository.findBy({file_id:t,host_type:this.hostType});let n=0,p=0;for(const i of d)n+=i.info.size,p+=i.info.size*i.upload_percent;const u=p/n,l=await this.fileRepository.findOneBy({id:t}),c=k[`upload_${this.hostType}`];l.progress[c].percent=u,l.percent=C(l.progress),await this.fileRepository.update(t,{progress:l.progress,percent:l.percent}),await this.onProgress(l)}async onPartError({error:i,part_id:t,file_id:s}){const e=await this.fileRepository.findOneBy({id:s});e&&(await this.fileRepository.update(s,{upload_status:T.UPLOAD_ERROR,error:{type:"upload",message:i.message,details:i.details,cause:i.cause,stack:i.stack}}),await this.partRepository.update(t,{upload_status:D.ERROR}),await this.fileRepository.update(e.id,{upload_status:T.UPLOAD_ERROR}),await this.baseParams.onError(i,e.id))}constructor(i,t){this.hostType=t,this.baseParams=i;const{dataSource:s}=i;this.fileRepository=s.getRepository(E),this.partRepository=s.getRepository(A),this.runningRepository=s.getRepository(B),i.onProgress&&(this.onProgress=i.onProgress.bind(this))}async setThreads(i){const t=this.getValidThreads(i);this.thread_count=t,await this.start()}async getParts(i){const t={uid:this.baseParams.uid,is_paused:!1,upload_status:D.NULL,host_id:this.baseParams.host_id};for(let s=0;s<i;s++){const s=await this.partRepository.countBy(t);if(s>=i)break;if(s<i){await this.baseParams.onDequeue(this.hostType);if(await this.partRepository.countBy(t)==s)break}}return await this.partRepository.find({where:t,order:{id:"ASC"},take:i})}async start(){for(;this.isRunning;)await new Promise(i=>{setTimeout(i,50)});try{this.isRunning=!0;const i=this.threads.length,t=this.thread_count;if(i<t){const s=t-i,e=await this.getParts(s);console.log("parts: ",e);const a=e.length;for(let i=0;i<a;i++){const t=e[i],s=this.getThread(t);(await this.fileRepository.findOneBy({id:t.file_id})).is_paused||(this.threads.push(s),await this.partRepository.update(t.id,{upload_status:D.PROCESS}),s.start().catch(async i=>{}))}}else if(i>t){const s=i-t,e=this.threads.slice(0,s);for(const i of e)await i.stop()}}finally{this.isRunning=!1}}async stopFiles(i){const t=this.threads.filter(t=>i.includes(t.file_id));await Promise.all(t.map(i=>i.stop()))}async stopAll(){await Promise.all(this.threads.map(i=>i.stop()))}async addRunning({file_id:i,part_id:t,root_id:s}){await this.runningRepository.save(this.runningRepository.create({type:U.UPLOAD,uid:this.baseParams.uid,root_id:s,file_id:i,part_id:t}))}async deleteRunning({file_id:i,part_id:t}){await this.runningRepository.delete({uid:this.baseParams.uid,file_id:i,part_id:t})}};import{HostType as L,NormalStatus as I,UploadStatus as M}from"@soga/types";import q from"piscina";import{MessageChannel as x}from"worker_threads";import{buildDError as z}from"@soga/error";var F=new q({filename:i.resolve("@soga/part-uploader")}),N=class extends v{params;constructor(i){super(i,L.BAIDU),this.params=i}getThread(i){const t=new AbortController,{port1:s,port2:e}=new x;return{file_id:i.file_id,part_id:i.id,uid:i.uid,start:async()=>{let a=null;try{if(a=await this.fileRepository.findOneBy({id:i.file_id}),!a)return;const{file_id:o,host_id:r,id:d,info:n,upload_status:p,root_id:u}=i;if(p==I.SUCCESS)return;await this.addRunning({file_id:o,part_id:d,root_id:u});const{output_root:l}=a;e.on("message",async i=>{await this.onPartProgress(i)});const c={file_id:o,host_id:r,part_id:d,output_root:l,part:n,cloud_folder_path:a.cloud_info.hosts.baidu?.path,sdk_domain:this.params.sdk_domain,port:s,appid:30045106},f=await F.run(c,{name:"uploadBaidu",signal:t.signal,transferList:[s]});if(!f?.fs_id)throw new Error("Upload to Baidu failed, result is null");await this.partRepository.update(i.id,{upload_result:{baidu:f},upload_status:I.SUCCESS}),await this.baseParams.onPartSuccess(o)}catch(s){if(t.abort(),"AbortError"==s.name)await this.partRepository.update(i.id,{upload_status:I.NULL});else{const t=a?a.input.filepath:"unknown filepath",e=z(s,{message:"Error occurred during baidu uploading",detail:`Error occurred during baidu uploading: ${t}`});await this.fileRepository.update(i.file_id,{upload_status:M.UPLOAD_ERROR,error:{type:"upload",message:e.message,details:e.details,cause:e.cause,stack:e.stack}}),await this.onPartError({error:s,part_id:i.id,file_id:i.file_id})}}finally{this.threads=this.threads.filter(t=>t.part_id!=i.id),e.close(),await this.deleteRunning({file_id:i.file_id,part_id:i.id}),await this.start()}},stop:async()=>{t.abort(),this.threads=this.threads.filter(t=>t.part_id!==i.id)}}}};import{HostType as $,NormalStatus as H,UploadStatus as Y}from"@soga/types";import J from"piscina";import{MessageChannel as j}from"worker_threads";import{buildDError as Q}from"@soga/error";var V=new J({filename:i.resolve("@soga/part-uploader")}),G=class extends v{params;constructor(i){super(i,$.ALI),this.params=i}getThread(i){const t=new AbortController,{port1:s,port2:e}=new j;return{file_id:i.file_id,part_id:i.id,uid:i.uid,start:async()=>{const a=await this.fileRepository.findOneBy({id:i.file_id});if(a)try{const{file_id:o,host_id:r,id:d,info:n,upload_status:p,root_id:u}=i;if(p==H.SUCCESS)return;await this.addRunning({file_id:o,part_id:d,root_id:u});const{output_root:l}=a;e.on("message",async i=>{await this.onPartProgress(i)});const c={file_id:o,host_id:r,part_id:d,output_root:l,part:n,cloud_folder_id:a.cloud_info.hosts.ali?.file_id,sdk_domain:this.params.sdk_domain,port:s},f=await V.run(c,{name:"uploadAli",signal:t.signal,transferList:[s]});if(!f?.file_id)throw new Error("Upload to Ali failed, result is null");await this.partRepository.update(i.id,{upload_result:{ali:f},upload_status:H.SUCCESS}),await this.baseParams.onPartSuccess(o)}catch(s){if(t.abort(),"AbortError"==s.name)this.partRepository.update(i.id,{upload_status:H.NULL});else{const t=Q(s,{message:"Error occurred during ali uploading",detail:`Error occurred during ali uploading: ${a.input.filepath}`});await this.fileRepository.update(i.file_id,{upload_status:Y.UPLOAD_ERROR,error:{type:"upload",message:t.message,details:t.details,cause:t.cause,stack:t.stack}}),await this.onPartError({error:s,part_id:i.id,file_id:i.file_id})}}finally{this.threads=this.threads.filter(t=>t.part_id!=i.id),e.close(),await this.deleteRunning({file_id:i.file_id,part_id:i.id}),await this.start()}},stop:async()=>{t.abort(),this.threads=this.threads.filter(t=>t.part_id!==i.id)}}}};import{buildDError as K}from"@soga/error";import{getProgressPercent as W}from"@soga/utils";import{UploadProgress as X}from"@soga/node-types";var Z=new Map,ii=class{params;uid;dataSource;fileRepository;partRepository;runningRepository;successRepository;processorMap=new Map;isDequeueing=!1;threadConfig={baidu:4,ali:2};constructor(i){this.params=i,this.uid=i.uid,this.dataSource=i.dataSource,this.fileRepository=this.dataSource.getRepository(r),this.partRepository=this.dataSource.getRepository(d),this.runningRepository=this.dataSource.getRepository(n),this.successRepository=this.dataSource.getRepository(u)}async init(){await this.runningRepository.delete({uid:this.uid,type:p.UPLOAD}),await this.fileRepository.update({uid:this.uid,upload_status:e.UPLOAD_ERROR},{upload_status:e.UPLOAD_READY}),await this.partRepository.update({uid:this.uid,upload_status:t.PROCESS},{upload_status:t.NULL})}async start(){await this.startCreates(),await Promise.all([this.startBaidu(),this.startAli()]).catch(i=>{console.error("Error starting uploader:",i)})}async stopFiles(i){const t=this.processorMap.values();for(const s of t)await s.stopFiles(i)}async stopAll(){const i=this.processorMap.values();for(const t of i)await t.stopAll()}async setThreadCount(i,t){this.threadConfig[i]=t;const s=this.processorMap.values();for(const e of s)e.hostType==i&&await e.setThreads(t)}isCreating=!1;async startCreates(){for(;this.isCreating;)await new Promise(i=>setTimeout(i,100));try{this.isCreating=!0;const i=await this.fileRepository.findBy({uid:this.uid,upload_status:e.SUBMIT_READY});if(!i.length)return;for(const t of i)await this.onPartSuccess({file_id:t.id})}finally{this.isCreating=!1}}async startHost(i,t){if(this.processorMap.has(i)){const t=this.processorMap.get(i),s=this.getThreadCount(t.hostType);return void await t.setThreads(s)}const e={uid:this.uid,sdk_domain:this.params.sdk_domain,dataSource:this.dataSource,host_id:i,onProgress:async i=>{await this.onFileProgress(i)},onPartSuccess:async i=>{await this.onPartSuccess({file_id:i})},onError:async(i,t)=>{await this.onFileError(i,t)},onDequeue:async i=>{await this.dequeueOneFile(i)}};if(t==s.BAIDU){const t=new N(e);this.processorMap.set(i,t);const a=this.getThreadCount(s.BAIDU);await t.setThreads(a)}else if(t==s.ALI){const t=new G(e);this.processorMap.set(i,t);const a=this.getThreadCount(s.ALI);await t.setThreads(a)}}async startBaidu(){const i=(await this.fileRepository.createQueryBuilder("file").select("DISTINCT file.baidu_host_id").where("file.uid = :uid",{uid:this.uid}).getRawMany()).map(i=>i.baidu_host_id).filter(i=>!!i);if(i.length)for(const t of i)await this.startHost(t,s.BAIDU)}async startAli(){const i=(await this.fileRepository.createQueryBuilder("file").select("DISTINCT file.ali_host_id").where("file.uid = :uid",{uid:this.uid}).getRawMany()).map(i=>i.ali_host_id).filter(i=>!!i);if(i.length)for(const t of i)await this.startHost(t,s.ALI)}async onPartSuccess({file_id:i}){await this.partRepository.findOneBy({file_id:i,upload_status:o(t.SUCCESS)})||(await this.fileRepository.update(i,{upload_status:e.SUBMIT_READY}),await this.onFileSuccess(i))}async onFileSuccess(i){const t=await this.fileRepository.findOneBy({id:i});if(t.upload_status!=e.SUBMIT_READY)return;if(!t)return;const{uid:a}=this;for(;Z.get(a);)await new Promise(i=>setTimeout(i,100));try{Z.set(a,!0);const o=await this.partRepository.findBy({file_id:t.id}),r={},d=t.progress;if(t.baidu_host_id){r.baidu_upload_result={};for(const i of o)i.host_type==s.BAIDU&&(r.baidu_upload_result[i.info.file]={...i.upload_result.baidu,md4:i.info.md4});d[X.upload_baidu].percent=1}if(t.ali_host_id){r.ali_upload_result={};for(const i of o)i.host_type==s.ALI&&(r.ali_upload_result[i.info.file]={...i.upload_result.ali,sha1:i.info.sha1});d[X.upload_ali].percent=1}const n=W(d);await this.fileRepository.update(t.id,{...r,progress:d,percent:n,upload_status:e.SUBMIT_READY}),await async function({file_id:i,sdk_domain:t,fileRepository:s}){try{for(;y.get(i);)await new Promise(i=>setTimeout(i,100));y.set(i,!0);const e=await s.findOneBy({id:i});let a=e.task_record_id;const o=l(t),{encoded:r,baidu_host_id:d,ali_host_id:n}=e,{ali:p,baidu:u,meta:w}=r,g=[];if(d&&g.push(c.BAIDU),n&&g.push(c.ALI),!g.length)return;const R=i=>{const t=r[i];if(!t.media?.subtitles?.length)return;const s=[];return t.media.subtitles.forEach(i=>{const{uuid:t,lang:e,name:a,title:o,builtin:r}=i;s.push({uuid:t,lang:e,name:a,title:o,builtin:r})}),s},b=i=>{if(!i)return;if(!i.subtitles)return i;const{subtitles:t,...s}=i,e={};return t.forEach(i=>{const{uuid:t,file:s,start:a,end:o,size:r,source:d}=i;e[t]={file:s,start:a,end:o,size:r,source:d}}),{...s,subtitles:e}};if(e.is_attachment){const i=await o.getRecordInfo({space_id:e.space_id,record_id:e.task_record_id,refresh:!0}),t=Object.assign({version:h,metas:{},hosts:{},attachments:[]},i.manifest);i.type,f.FOLDER;const s={metas:w,uuid:_(),type:e.type,ftype:e.ftype,title:e.input.filename,subtitles:R(g[0])};if(d){const{parts:i,max_size:t,media:a,img:o,txt:r,source:n}=u;s.baidu={host_id:d,media:b(a),max_size:t,img:o,txt:r,source:n,files:{}};const p=e.baidu_upload_result;i.forEach(i=>{const{file:t,md4:e,preview:a,source:o,size:r,md5:d}=i,{fs_id:n}=p[t];s.baidu.files[t]={fs_id:n,md5:d,md4:e,size:r,preview:a,source:o}})}if(n){const{parts:i,max_size:t,media:a,img:o,txt:r,source:d}=p,{drive_id:u}=e.cloud_info.hosts.ali;s.ali={host_id:n,drive_id:u,media:b(a),max_size:t,img:o,txt:r,source:d,files:{}};const l=e.ali_upload_result;i.forEach(i=>{const{file:t,preview:e,source:a,size:o,md5:r}=i,{file_id:d,sha1:n}=l[t];s.ali.files[t]={file_id:d,md5:r,sha1:n,size:o,preview:e,source:a}})}t.attachments=t.attachments||[],t.attachments.push(s),t.hosts||(t.hosts={}),d&&(t.hosts.baidu=m(t,c.BAIDU)),n&&(t.hosts.ali=m(t,c.ALI)),await o.updateRecord({space_id:i.space_id,record_id:i.id,parent_id:i.parent_id,manifest:JSON.stringify(t)})}else{0!==e.pid&&(a=(await s.findOneBy({id:e.pid})).cloud_info.id);const i={version:h,hosts:{},complete:!0,metas:w,subtitles:R(g[0])};if(d){const{parts:t,media:s,...a}=u,o=b(s),r=e.baidu_upload_result;i.baidu={host_id:d,...a,files:{},media:o},t.forEach(t=>{const{file:s,md4:e,preview:a,source:o,size:d,md5:n}=t,{fs_id:p}=r[s];i.baidu.files[s]={fs_id:p,md5:n,md4:e,size:d,preview:a,source:o}})}if(n){const{parts:t,media:s,...a}=p,o=b(s),r=e.ali_upload_result;i.ali={host_id:n,drive_id:"",...a,media:o,files:{}},t.forEach(t=>{const{file:s,preview:e,source:a,size:o,md5:d}=t,{file_id:n,drive_id:p,sha1:u}=r[s];i.ali.drive_id||(i.ali.drive_id=p),i.ali.files[s]={file_id:n,sha1:u,md5:d,size:o,preview:e,source:a}});const d=Object.values(i.ali.files).reduce((i,t)=>i+t.size,0);i.hosts.ali=d}if(d&&(i.hosts.baidu=m(i,c.BAIDU)),n&&(i.hosts.ali=m(i,c.ALI)),e.cloud_info?.id){const i=await o.getRecordInfo({space_id:e.space_id,record_id:e.cloud_info.id,refresh:!0});if(i.manifest?.complete)throw new Error("the file has been modified!")}await o.updateRecord({space_id:e.space_id,record_id:e.cloud_info.id,parent_id:a,manifest:JSON.stringify(i)})}}catch(i){throw i}finally{y.delete(i)}}({file_id:t.id,fileRepository:this.fileRepository,sdk_domain:this.params.sdk_domain}),this.params.onSuccess&&await this.params.onSuccess(i),await async function(i){const{fileRepository:t,partRepository:s,successRepository:e}=i,a=await t.findOneBy({id:i.file_id}),o={is_attachment:a.is_attachment,root_id:a.root_id,pid:0,space_id:a.space_id,space_name:a.space_name,uid:a.uid,input:a.input,cloud_id:a.cloud_info.id,cloud_name:a.cloud_info.name,type:a.type},r=0==a.root_id;if(!r){const s=await e.findOneBy({root_id:a.root_id,pid:0});if(a.is_attachment){const r=w(i.sdk_domain);if(s)o.pid=s.id;else{const i=await r.getRecordInfo({space_id:a.space_id,record_id:a.task_record_id,refresh:!1}),s=await t.findOneBy({id:a.root_id}),d=await e.save(e.create({is_attachment:!0,root_id:a.root_id,pid:0,space_id:i.space_id,space_name:a.space_name,uid:a.uid,input:s.input,cloud_id:i.id,cloud_name:i.name,type:i.type}));o.pid=d.id}}else if(s)o.pid=s.id;else{const i=await t.findOneBy({id:a.root_id}),s=await e.save(e.create({is_attachment:!1,root_id:a.root_id,pid:0,space_id:i.space_id,space_name:i.space_name,uid:i.uid,input:i.input,cloud_id:i.cloud_info.id,cloud_name:i.cloud_info.name,type:i.type}));o.pid=s.id}await e.update({id:o.pid},{updated_at:new Date})}await e.save(e.create(o)),await t.delete({id:i.file_id}),await s.delete({file_id:i.file_id}),await R(a.output_root),r||(await t.findOneBy({root_id:a.root_id,type:b(g.FOLDER)})?await t.increment({id:a.root_id},"completed_count",1):(await t.delete({root_id:a.root_id}),await t.delete({id:a.root_id})))}({file_id:t.id,sdk_domain:this.params.sdk_domain,fileRepository:this.fileRepository,partRepository:this.partRepository,successRepository:this.successRepository})}catch(i){await this.fileRepository.update(t.id,{upload_status:e.UPLOAD_ERROR,error:{type:"upload",message:i.message,details:i.details,cause:i.cause,stack:i.stack}})}finally{Z.delete(a)}}async onFileError(i,t){try{this.params.onError&&await this.params.onError(i,t)}catch(i){}}async onFileProgress(i){try{this.params.onProgress&&await this.params.onProgress(i)}catch(i){}}async dequeueOneFile(i){for(;this.isDequeueing;)await new Promise(i=>{setTimeout(i,50)});let r=null;try{this.isDequeueing=!0;const d=`${i}_host_id`;if(r=await this.fileRepository.findOne({where:{uid:this.params.uid,is_paused:!1,upload_status:e.UPLOAD_READY,[d]:o(a())},order:{id:"ASC"}}),!r)return;if(r.cloud_info?.id||(await O({file_id:r.id,sdk_domain:this.params.sdk_domain,fileRepository:this.fileRepository}),r=await this.fileRepository.findOneBy({id:r.id})),!r.cloud_info?.id)throw new Error("Created cloud folder or file failed");const{encoded:n}=r,p=[],u=[s.BAIDU,s.ALI];for await(const i of u){const t=`${i}_host_id`;if(!r[t])continue;const s=r[t],e=n[i].parts||[],{length:a}=e;for(let t=0;t<a;t++){const a=e[t];await this.partRepository.findOneBy({file_id:r.id,part_name:a.file,host_id:s})||p.push({uid:this.params.uid,file_id:r.id,root_id:r.root_id??r.id,part_name:a.file,info:a,output_root:r.output_root,host_id:s,host_type:i})}}if(p.length)await this.partRepository.save(p),await this.fileRepository.update(r.id,{upload_status:e.UPLOAD_PROCESS});else{await this.fileRepository.update(r.id,{upload_status:e.UPLOAD_PROCESS});const s=await this.partRepository.findOneBy({file_id:r.id,upload_status:o(t.SUCCESS)});await this.partRepository.findOneBy({file_id:r.id,upload_status:t.SUCCESS});s||await this.onPartSuccess({file_id:r.id});await this.partRepository.findOneBy({file_id:r.id,upload_status:t.NULL})||(this.isDequeueing=!1,await this.dequeueOneFile(i))}}catch(t){if(!r)throw t;{const s=K(t,{message:"Error occurred during baidu preparing",detail:`Error occurred during baidu preparing: ${r.input.filepath}`});await this.fileRepository.update(r.id,{upload_status:e.UPLOAD_ERROR,error:{type:"upload",message:s.message,details:s.details,cause:s.cause,stack:s.stack}}),this.isDequeueing=!1,await this.dequeueOneFile(i)}}finally{this.isDequeueing=!1}}getThreadCount(i){return this.threadConfig[i]??2}},ti=new Map,si=async i=>{for(;ti.get(i.uid)?.initing;)await new Promise(i=>setTimeout(i,100));try{if(ti.get(i.uid)?.uploader)return ti.get(i.uid).uploader;const t={uploader:new ii(i),initing:!0};return ti.set(i.uid,t),await t.uploader.init(),t.uploader}finally{ti.get(i.uid)&&(ti.get(i.uid).initing=!1)}};export{ii as Uploader,si as getUploader};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@soga/uploader",
-  "version": "1.3.14",
+  "version": "1.3.15",
   "publishConfig": {
     "access": "public"
   },
@@ -18,13 +18,13 @@
     "typeorm": "^0.3.24",
     "fs-extra": "^11.3.2",
     "uuid": "^11.1.0",
-    "@soga/entities": "1.3.14",
-    "@soga/part-uploader": "1.3.14",
-    "@soga/sdk": "1.3.14",
-    "@soga/utils": "1.3.14",
-    "@soga/node-types": "1.3.14",
-    "@soga/types": "1.3.14",
-    "@soga/error": "1.3.14"
+    "@soga/entities": "1.3.15",
+    "@soga/node-types": "1.3.15",
+    "@soga/part-uploader": "1.3.15",
+    "@soga/sdk": "1.3.15",
+    "@soga/error": "1.3.15",
+    "@soga/utils": "1.3.15",
+    "@soga/types": "1.3.15"
   },
   "devDependencies": {
     "@types/node": "^24.5.2",
@@ -33,7 +33,9 @@
     "tsup": "^8.5.0",
     "typescript": "^5.8.3",
     "@types/fs-extra": "^11.0.4",
-    "@soga/typescript-config": "1.3.14"
+    "tsx": "^4.20.6",
+    "better-sqlite3": "^12.4.6",
+    "@soga/typescript-config": "1.3.15"
   },
   "scripts": {
     "demo": "tsx ./demo/demo.ts",