@soga/uploader 0.3.0 → 1.3.1

package/dist/main.d.mts

@@ -0,0 +1,50 @@
+import { HostType } from '@soga/types';
+import { UploadFile } from '@soga/entities';
+import { DataSource } from 'typeorm';
+
+type OnProgress = (file: UploadFile) => Promise<void>;
+type OnSuccess = (id: number) => Promise<void>;
+type OnError = (error: Error, file_id: number) => Promise<void>;
+interface CommonParams {
+    uid: number;
+    sdk_domain: string;
+    dataSource: DataSource;
+}
+interface UploaderParams extends CommonParams {
+    onProgress?: OnProgress;
+    onSuccess?: OnSuccess;
+    onError?: OnError;
+}
+
+declare class Uploader {
+    private params;
+    private uid;
+    private dataSource;
+    private fileRepository;
+    private partRepository;
+    private runningRepository;
+    private successRepository;
+    private processorMap;
+    private isDequeueing;
+    private threadConfig;
+    constructor(params: UploaderParams);
+    init(): Promise<void>;
+    start(): Promise<void>;
+    stopFiles(ids: number[]): Promise<void>;
+    stopAll(): Promise<void>;
+    setThreadCount(host_type: HostType, count: number): Promise<void>;
+    private isCreating;
+    private startCreates;
+    private startHost;
+    private startBaidu;
+    private startAli;
+    private onPartSuccess;
+    private onFileSuccess;
+    private onFileError;
+    private onFileProgress;
+    private dequeueOneFile;
+    private getThreadCount;
+}
+declare const getUploader: (params: UploaderParams) => Promise<Uploader>;
+
+export { Uploader, type UploaderParams, getUploader };

package/dist/main.d.ts

@@ -1,6 +1,50 @@
-export * from './host-uploader/baidu';
-export * from './host-uploader/ali';
-import { GetUploaderParams } from './types/main';
-import { Uploader } from './uploader';
-export declare const getUploader: (params: GetUploaderParams) => Promise<Uploader>;
-export { UploaderParams } from './types/main';
+import { HostType } from '@soga/types';
+import { UploadFile } from '@soga/entities';
+import { DataSource } from 'typeorm';
+
+type OnProgress = (file: UploadFile) => Promise<void>;
+type OnSuccess = (id: number) => Promise<void>;
+type OnError = (error: Error, file_id: number) => Promise<void>;
+interface CommonParams {
+    uid: number;
+    sdk_domain: string;
+    dataSource: DataSource;
+}
+interface UploaderParams extends CommonParams {
+    onProgress?: OnProgress;
+    onSuccess?: OnSuccess;
+    onError?: OnError;
+}
+
+declare class Uploader {
+    private params;
+    private uid;
+    private dataSource;
+    private fileRepository;
+    private partRepository;
+    private runningRepository;
+    private successRepository;
+    private processorMap;
+    private isDequeueing;
+    private threadConfig;
+    constructor(params: UploaderParams);
+    init(): Promise<void>;
+    start(): Promise<void>;
+    stopFiles(ids: number[]): Promise<void>;
+    stopAll(): Promise<void>;
+    setThreadCount(host_type: HostType, count: number): Promise<void>;
+    private isCreating;
+    private startCreates;
+    private startHost;
+    private startBaidu;
+    private startAli;
+    private onPartSuccess;
+    private onFileSuccess;
+    private onFileError;
+    private onFileProgress;
+    private dequeueOneFile;
+    private getThreadCount;
+}
+declare const getUploader: (params: UploaderParams) => Promise<Uploader>;
+
+export { Uploader, type UploaderParams, getUploader };

package/dist/main.js

@@ -1,45 +1 @@
-"use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __exportStar = (this && this.__exportStar) || function(m, exports) {
-    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.getUploader = void 0;
-__exportStar(require("./host-uploader/baidu"), exports);
-__exportStar(require("./host-uploader/ali"), exports);
-const uploader_1 = require("./uploader");
-const instanceMap = new Map();
-const getUploader = async (params) => {
-    while (instanceMap.get(params.uid)?.initing) {
-        await new Promise((resolve) => setTimeout(resolve, 100));
-    }
-    try {
-        if (instanceMap.get(params.uid)?.uploader) {
-            return instanceMap.get(params.uid).uploader;
-        }
-        const cache = {
-            uploader: new uploader_1.Uploader(params),
-            initing: true,
-        };
-        instanceMap.set(params.uid, cache);
-        await cache.uploader.init();
-        cache.initing = false;
-        return cache.uploader;
-    }
-    finally {
-        if (instanceMap.get(params.uid)) {
-            instanceMap.get(params.uid).initing = false;
-        }
-    }
-};
-exports.getUploader = getUploader;
+var t,i=Object.create,e=Object.defineProperty,a=Object.getOwnPropertyDescriptor,s=Object.getOwnPropertyNames,o=Object.getPrototypeOf,r=Object.prototype.hasOwnProperty,d=(t,i,o,d)=>{if(i&&"object"==typeof i||"function"==typeof i)for(let n of s(i))r.call(t,n)||n===o||e(t,n,{get:()=>i[n],enumerable:!(d=a(i,n))||d.enumerable});return t},n=(t,a,s)=>(s=null!=t?i(o(t)):{},d(!a&&t&&t.__esModule?s:e(s,"default",{value:t,enumerable:!0}),t)),u={};((t,i)=>{for(var a in i)e(t,a,{get:i[a],enumerable:!0})})(u,{Uploader:()=>Q,getUploader:()=>G}),module.exports=(t=u,d(e({},"__esModule",{value:!0}),t));var p=require("@soga/types"),l=require("typeorm"),c=require("@soga/entities"),f=require("@soga/sdk"),_=require("@soga/types"),h=require("uuid"),y="1.0.0",m=require("@soga/utils"),w=new Map;var g=require("@soga/sdk"),R=require("@soga/types"),S=require("fs-extra"),b=require("typeorm");var P=require("@soga/types"),O=require("@soga/sdk");async function U({file_id:t,sdk_domain:i,fileRepository:e}){const a=await e.findOneBy({id:t});if(a.is_attachment){const{task_record_id:t}=a,s=(0,O.getSdk)(i),r=await s.getRecordInfo({space_id:a.space_id,record_id:t,refresh:!0}),d=await o(r);await e.update(a.id,{cloud_info:d})}else await async function(){const i=await e.findOneBy({id:t});if(!i)return;const a=await s(i.pid,[]);for(const t of a)await r(t);return a}(),await r(a);async function s(t,i=[]){try{if(0===t)return i;const a=await e.findOneBy({id:t});return a?a.cloud_info?.id?i:(i.unshift(a),0!==a.pid?await s(a.pid,i).catch(t=>{throw t}):i):i}catch(t){throw t}}async function o(t){const{id:i,name:s,cloud_info:o}=t,r={id:i,name:s,hosts:{}};if(o.ali&&(r.hosts.ali={id:o.ali.id,name:o.ali.name,drive_id:o.ali.drive_id,file_id:o.ali.file_id}),o.baidu&&(r.hosts.baidu={id:o.baidu.id,name:o.baidu.name,fs_id:o.baidu.fs_id,path:o.baidu.path},!r.hosts.baidu.path)){const t=await e.findOneBy({id:a.pid}),{path:i}=t.cloud_info.hosts.baidu;i&&(r.hosts.baidu.path=`${i}/${o.baidu.name}`)}return r}async function r(t){const a=await e.findOneBy({id:t.id});if(a.cloud_info?.id)return;const{input:s,space_id:r,task_record_id:d,type:n}=a,u=n===P.RecordType.FOLDER;let p=d;if(0!==a.pid){p=(await e.findOneBy({id:a.pid})).cloud_info.id}const l=(0,O.getSdk)(i),c={space_id:r,name:s.filename,parent_id:p,type:a.type,ftype:a.ftype},f={};a.baidu_host_id&&(f.baidu=0),a.ali_host_id&&(f.ali=0);const _=await l.createRecord({...c,manifest:JSON.stringify({version:y,complete:!!u,hosts:f,metas:{file:{btime:a.input.local_btime,ctime:a.input.local_ctime,mtime:a.input.local_mtime,file:a.input.filename}}})}),h=await o(_);await e.update({id:a.id},{cloud_info:h})}}var E=require("@soga/entities"),T=require("@soga/types"),A=require("@soga/utils"),B=require("@soga/node-types"),k=class{hostType;onProgress=async()=>{};fileRepository;partRepository;runningRepository;isRunning=!1;thread_count=2;maxThreads=8;threads=[];baseParams;getValidThreads(t){return Math.min(Math.max(t??0,0),this.maxThreads)}async onPartProgress(t){const{id:i,data:e,type:a}=t;if("percent"!=a)return;const{part_id:s,percent:o}=e,r=await this.partRepository.findOneBy({id:s});if(!r)return;if(r.upload_percent>=o)return;if(r.upload_status==T.NormalStatus.ERROR)return;if(r.upload_status==T.NormalStatus.SUCCESS)return;await this.partRepository.update(s,{upload_percent:o});const d=await this.partRepository.findBy({file_id:i,host_type:this.hostType});let n=0,u=0;for(const t of d)n+=t.info.size,u+=t.info.size*t.upload_percent;const p=u/n,l=await this.fileRepository.findOneBy({id:i}),c=B.UploadProgress[`upload_${this.hostType}`];l.progress[c].percent=p,l.percent=(0,A.getProgressPercent)(l.progress),await this.fileRepository.update(i,{progress:l.progress,percent:l.percent}),await this.onProgress(l)}async onPartError({error:t,part_id:i,file_id:e}){const a=await this.fileRepository.findOneBy({id:e});a&&(await this.fileRepository.update(e,{upload_status:T.UploadStatus.UPLOAD_ERROR,error:{type:"upload",message:t.message,details:t.details,cause:t.cause,stack:t.stack}}),await this.partRepository.update(i,{upload_status:T.NormalStatus.ERROR}),await this.fileRepository.update(a.id,{upload_status:T.UploadStatus.UPLOAD_ERROR}),await this.baseParams.onError(t,a.id))}constructor(t,i){this.hostType=i,this.baseParams=t;const{dataSource:e}=t;this.fileRepository=e.getRepository(E.UploadFile),this.partRepository=e.getRepository(E.UploadPart),this.runningRepository=e.getRepository(E.UploadRunning),t.onProgress&&(this.onProgress=t.onProgress.bind(this))}async setThreads(t){const i=this.getValidThreads(t);this.thread_count=i,await this.start()}async getParts(t){const i={uid:this.baseParams.uid,is_paused:!1,upload_status:T.NormalStatus.NULL,host_id:this.baseParams.host_id};for(let e=0;e<t;e++){const e=await this.partRepository.countBy(i);if(e>=t)break;if(e<t){await this.baseParams.onDequeue(this.hostType);if(await this.partRepository.countBy(i)==e)break}}return await this.partRepository.find({where:i,order:{id:"ASC"},take:t})}async start(){for(;this.isRunning;)await new Promise(t=>{setTimeout(t,50)});try{this.isRunning=!0;const t=this.threads.length,i=this.thread_count;if(t<i){const e=i-t,a=await this.getParts(e),s=a.length;for(let t=0;t<s;t++){const i=a[t],e=this.getThread(i);(await this.fileRepository.findOneBy({id:i.file_id})).is_paused||(this.threads.push(e),await this.partRepository.update(i.id,{upload_status:T.NormalStatus.PROCESS}),e.start().catch(async t=>{}))}}else if(t>i){const e=t-i,a=this.threads.slice(0,e);for(const t of a)await t.stop()}}finally{this.isRunning=!1}}async stopFiles(t){const i=this.threads.filter(i=>t.includes(i.file_id));await Promise.all(i.map(t=>t.stop()))}async stopAll(){await Promise.all(this.threads.map(t=>t.stop()))}async addRunning({file_id:t,part_id:i,root_id:e}){await this.runningRepository.save(this.runningRepository.create({type:E.UploadRunningType.UPLOAD,uid:this.baseParams.uid,root_id:e,file_id:t,part_id:i}))}async deleteRunning({file_id:t,part_id:i}){await this.runningRepository.delete({uid:this.baseParams.uid,file_id:t,part_id:i})}},C=require("@soga/types"),D=n(require("piscina")),q=require("worker_threads"),v=require("@soga/error"),L=new D.default({filename:require.resolve("@soga/part-uploader")}),N=class extends k{params;constructor(t){super(t,C.HostType.BAIDU),this.params=t}getThread(t){const i=new AbortController,{port1:e,port2:a}=new q.MessageChannel;return{file_id:t.file_id,part_id:t.id,uid:t.uid,start:async()=>{let s=null;try{if(s=await this.fileRepository.findOneBy({id:t.file_id}),!s)return;const{file_id:o,host_id:r,id:d,info:n,upload_status:u,root_id:p}=t;if(u==C.NormalStatus.SUCCESS)return;await this.addRunning({file_id:o,part_id:d,root_id:p});const{output_root:l}=s;a.on("message",async t=>{await this.onPartProgress(t)});const c={file_id:o,host_id:r,part_id:d,output_root:l,part:n,cloud_folder_path:s.cloud_info.hosts.baidu?.path,sdk_domain:this.params.sdk_domain,port:e,appid:30045106},f=await L.run(c,{name:"uploadBaidu",signal:i.signal,transferList:[e]});if(!f?.fs_id)throw new Error("Upload to Baidu failed, result is null");await this.partRepository.update(t.id,{upload_result:{baidu:f},upload_status:C.NormalStatus.SUCCESS}),await this.baseParams.onPartSuccess(o)}catch(e){if(i.abort(),"AbortError"==e.name)await this.partRepository.update(t.id,{upload_status:C.NormalStatus.NULL});else{const i=s?s.input.filepath:"unknown filepath",a=(0,v.buildDError)(e,{message:"Error occurred during baidu uploading",detail:`Error occurred during baidu uploading: ${i}`});await this.fileRepository.update(t.file_id,{upload_status:C.UploadStatus.UPLOAD_ERROR,error:{type:"upload",message:a.message,details:a.details,cause:a.cause,stack:a.stack}}),await this.onPartError({error:e,part_id:t.id,file_id:t.file_id})}}finally{this.threads=this.threads.filter(i=>i.part_id!=t.id),a.close(),await this.deleteRunning({file_id:t.file_id,part_id:t.id}),await this.start()}},stop:async()=>{i.abort(),this.threads=this.threads.filter(i=>i.part_id!==t.id)}}}},I=require("@soga/types"),H=n(require("piscina")),M=require("worker_threads"),z=require("@soga/error"),F=new H.default({filename:require.resolve("@soga/part-uploader")}),x=class extends k{params;constructor(t){super(t,I.HostType.ALI),this.params=t}getThread(t){const i=new AbortController,{port1:e,port2:a}=new M.MessageChannel;return{file_id:t.file_id,part_id:t.id,uid:t.uid,start:async()=>{const s=await this.fileRepository.findOneBy({id:t.file_id});if(s)try{const{file_id:o,host_id:r,id:d,info:n,upload_status:u,root_id:p}=t;if(u==I.NormalStatus.SUCCESS)return;await this.addRunning({file_id:o,part_id:d,root_id:p});const{output_root:l}=s;a.on("message",async t=>{await this.onPartProgress(t)});const c={file_id:o,host_id:r,part_id:d,output_root:l,part:n,cloud_folder_id:s.cloud_info.hosts.ali?.file_id,sdk_domain:this.params.sdk_domain,port:e},f=await F.run(c,{name:"uploadAli",signal:i.signal,transferList:[e]});if(!f?.file_id)throw new Error("Upload to Ali failed, result is null");await this.partRepository.update(t.id,{upload_result:{ali:f},upload_status:I.NormalStatus.SUCCESS}),await this.baseParams.onPartSuccess(o)}catch(e){if(i.abort(),"AbortError"==e.name)this.partRepository.update(t.id,{upload_status:I.NormalStatus.NULL});else{const i=(0,z.buildDError)(e,{message:"Error occurred during ali uploading",detail:`Error occurred during ali uploading: ${s.input.filepath}`});await this.fileRepository.update(t.file_id,{upload_status:I.UploadStatus.UPLOAD_ERROR,error:{type:"upload",message:i.message,details:i.details,cause:i.cause,stack:i.stack}}),await this.onPartError({error:e,part_id:t.id,file_id:t.file_id})}}finally{this.threads=this.threads.filter(i=>i.part_id!=t.id),a.close(),await this.deleteRunning({file_id:t.file_id,part_id:t.id}),await this.start()}},stop:async()=>{i.abort(),this.threads=this.threads.filter(i=>i.part_id!==t.id)}}}},j=require("@soga/error"),$=require("@soga/utils"),J=require("@soga/node-types"),Y=new Map,Q=class{params;uid;dataSource;fileRepository;partRepository;runningRepository;successRepository;processorMap=new Map;isDequeueing=!1;threadConfig={baidu:4,ali:2};constructor(t){this.params=t,this.uid=t.uid,this.dataSource=t.dataSource,this.fileRepository=this.dataSource.getRepository(c.UploadFile),this.partRepository=this.dataSource.getRepository(c.UploadPart),this.runningRepository=this.dataSource.getRepository(c.UploadRunning),this.successRepository=this.dataSource.getRepository(c.UploadSuccess)}async init(){await this.runningRepository.delete({uid:this.uid,type:c.UploadRunningType.UPLOAD}),await this.fileRepository.update({uid:this.uid,upload_status:p.UploadStatus.UPLOAD_ERROR},{upload_status:p.UploadStatus.UPLOAD_READY}),await this.partRepository.update({uid:this.uid,upload_status:p.NormalStatus.PROCESS},{upload_status:p.NormalStatus.NULL})}async start(){await this.startCreates(),await Promise.all([this.startBaidu(),this.startAli()]).catch(t=>{console.error("Error starting uploader:",t)})}async stopFiles(t){const i=this.processorMap.values();for(const e of i)await e.stopFiles(t)}async stopAll(){const t=this.processorMap.values();for(const i of t)await i.stopAll()}async setThreadCount(t,i){this.threadConfig[t]=i;const e=this.processorMap.values();for(const a of e)a.hostType==t&&await a.setThreads(i)}isCreating=!1;async startCreates(){for(;this.isCreating;)await new Promise(t=>setTimeout(t,100));try{this.isCreating=!0;const t=await this.fileRepository.findBy({uid:this.uid,upload_status:p.UploadStatus.SUBMIT_READY});if(!t.length)return;for(const i of t)await this.onPartSuccess({file_id:i.id})}finally{this.isCreating=!1}}async startHost(t,i){if(this.processorMap.has(t)){const i=this.processorMap.get(t),e=this.getThreadCount(i.hostType);return void await i.setThreads(e)}const e={uid:this.uid,sdk_domain:this.params.sdk_domain,dataSource:this.dataSource,host_id:t,onProgress:async t=>{await this.onFileProgress(t)},onPartSuccess:async t=>{await this.onPartSuccess({file_id:t})},onError:async(t,i)=>{await this.onFileError(t,i)},onDequeue:async t=>{await this.dequeueOneFile(t)}};if(i==p.HostType.BAIDU){const i=new N(e);this.processorMap.set(t,i);const a=this.getThreadCount(p.HostType.BAIDU);await i.setThreads(a)}else if(i==p.HostType.ALI){const i=new x(e);this.processorMap.set(t,i);const a=this.getThreadCount(p.HostType.ALI);await i.setThreads(a)}}async startBaidu(){const t=(await this.fileRepository.createQueryBuilder("file").select("DISTINCT file.baidu_host_id").where("file.uid = :uid",{uid:this.uid}).getRawMany()).map(t=>t.baidu_host_id).filter(t=>!!t);if(t.length)for(const i of t)await this.startHost(i,p.HostType.BAIDU)}async startAli(){const t=(await this.fileRepository.createQueryBuilder("file").select("DISTINCT file.ali_host_id").where("file.uid = :uid",{uid:this.uid}).getRawMany()).map(t=>t.ali_host_id).filter(t=>!!t);if(t.length)for(const i of t)await this.startHost(i,p.HostType.ALI)}async onPartSuccess({file_id:t}){await this.partRepository.findOneBy({file_id:t,upload_status:(0,l.Not)(p.NormalStatus.SUCCESS)})||await this.onFileSuccess(t)}async onFileSuccess(t){try{const{uid:i}=this;for(;Y.get(i);)await new Promise(t=>setTimeout(t,100));Y.set(i,!0);const e=await this.fileRepository.findOneBy({id:t});if(!e)return;try{const i=await this.partRepository.findBy({file_id:e.id}),a={},s=e.progress;if(e.baidu_host_id){a.baidu_upload_result={};for(const t of i)t.host_type==p.HostType.BAIDU&&(a.baidu_upload_result[t.info.file]={...t.upload_result.baidu,md4:t.info.md4});s[J.UploadProgress.upload_baidu].percent=1}if(e.ali_host_id){a.ali_upload_result={};for(const t of i)t.host_type==p.HostType.ALI&&(a.ali_upload_result[t.info.file]={...t.upload_result.ali,sha1:t.info.sha1});s[J.UploadProgress.upload_ali].percent=1}const o=(0,$.getProgressPercent)(s);await this.fileRepository.update(e.id,{...a,progress:s,percent:o}),await async function({file_id:t,sdk_domain:i,fileRepository:e}){try{for(;w.get(t);)await new Promise(t=>setTimeout(t,100));w.set(t,!0);const a=await e.findOneBy({id:t});let s=a.task_record_id;const o=(0,f.getSdk)(i),{encoded:r,baidu_host_id:d,ali_host_id:n}=a,{ali:u,baidu:p,meta:l}=r,c=[];if(d&&c.push(_.HostType.BAIDU),n&&c.push(_.HostType.ALI),!c.length)return;const g=t=>{const i=r[t];if(!i.media?.subtitles?.length)return;const e=[];return i.media.subtitles.forEach(t=>{const{uuid:i,lang:a,name:s,title:o,builtin:r}=t;e.push({uuid:i,lang:a,name:s,title:o,builtin:r})}),e},R=t=>{if(!t)return;if(!t.subtitles)return t;const{subtitles:i,...e}=t,a={};return i.forEach(t=>{const{uuid:i,file:e,start:s,end:o,size:r,source:d}=t;a[i]={file:e,start:s,end:o,size:r,source:d}}),{...e,subtitles:a}};if(a.is_attachment){const t=await o.getRecordInfo({space_id:a.space_id,record_id:a.task_record_id,refresh:!0}),i=Object.assign({version:y,metas:{},hosts:{},attachments:[]},t.manifest);t.type,_.RecordType.FOLDER;const e={metas:l,uuid:(0,h.v4)(),type:a.type,ftype:a.ftype,title:a.input.filename,subtitles:g(c[0])};if(d){const{parts:t,max_size:i,media:s,img:o,txt:r,source:n}=p;e.baidu={host_id:d,media:R(s),max_size:i,img:o,txt:r,source:n,files:{}};const u=a.baidu_upload_result;t.forEach(t=>{const{file:i,md4:a,preview:s,source:o,size:r,md5:d}=t,{fs_id:n}=u[i];e.baidu.files[i]={fs_id:n,md5:d,md4:a,size:r,preview:s,source:o}})}if(n){const{parts:t,max_size:i,media:s,img:o,txt:r,source:d}=u,{drive_id:p}=a.cloud_info.hosts.ali;e.ali={host_id:n,drive_id:p,media:R(s),max_size:i,img:o,txt:r,source:d,files:{}};const l=a.ali_upload_result;t.forEach(t=>{const{file:i,preview:a,source:s,size:o,md5:r}=t,{file_id:d,sha1:n}=l[i];e.ali.files[i]={file_id:d,md5:r,sha1:n,size:o,preview:a,source:s}})}i.attachments=i.attachments||[],i.attachments.push(e),i.hosts||(i.hosts={}),d&&(i.hosts.baidu=(0,m.calculateManifestHostSize)(i,_.HostType.BAIDU)),n&&(i.hosts.ali=(0,m.calculateManifestHostSize)(i,_.HostType.ALI)),await o.updateRecord({space_id:t.space_id,record_id:t.id,parent_id:t.parent_id,manifest:JSON.stringify(i)})}else{0!==a.pid&&(s=(await e.findOneBy({id:a.pid})).cloud_info.id);const t={version:y,hosts:{},complete:!0,metas:l,subtitles:g(c[0])};if(d){const{parts:i,media:e,...s}=p,o=R(e),r=a.baidu_upload_result;t.baidu={host_id:d,...s,files:{},media:o},i.forEach(i=>{const{file:e,md4:a,preview:s,source:o,size:d,md5:n}=i,{fs_id:u}=r[e];t.baidu.files[e]={fs_id:u,md5:n,md4:a,size:d,preview:s,source:o}})}if(n){const{parts:i,media:e,...s}=u,o=R(e),r=a.ali_upload_result;t.ali={host_id:n,drive_id:"",...s,media:o,files:{}},i.forEach(i=>{const{file:e,preview:a,source:s,size:o,md5:d}=i,{file_id:n,drive_id:u,sha1:p}=r[e];t.ali.drive_id||(t.ali.drive_id=u),t.ali.files[e]={file_id:n,sha1:p,md5:d,size:o,preview:a,source:s}});const d=Object.values(t.ali.files).reduce((t,i)=>t+i.size,0);t.hosts.ali=d}if(d&&(t.hosts.baidu=(0,m.calculateManifestHostSize)(t,_.HostType.BAIDU)),n&&(t.hosts.ali=(0,m.calculateManifestHostSize)(t,_.HostType.ALI)),a.cloud_info?.id){const t=await o.getRecordInfo({space_id:a.space_id,record_id:a.cloud_info.id,refresh:!0});if(t.manifest?.complete)throw new Error("the file has been modified!")}await o.updateRecord({space_id:a.space_id,record_id:a.cloud_info.id,parent_id:s,manifest:JSON.stringify(t)})}}catch(t){throw t}finally{w.delete(t)}}({file_id:e.id,fileRepository:this.fileRepository,sdk_domain:this.params.sdk_domain}),this.params.onSuccess&&await this.params.onSuccess(t),await async function(t){const{fileRepository:i,partRepository:e,successRepository:a}=t,s=await i.findOneBy({id:t.file_id}),o={is_attachment:s.is_attachment,root_id:s.root_id,pid:0,space_id:s.space_id,space_name:s.space_name,uid:s.uid,input:s.input,cloud_id:s.cloud_info.id,cloud_name:s.cloud_info.name,type:s.type},r=0==s.root_id;if(!r){const e=await a.findOneBy({root_id:s.root_id,pid:0});if(s.is_attachment){const r=(0,g.getSdk)(t.sdk_domain);if(e)o.pid=e.id;else{const t=await r.getRecordInfo({space_id:s.space_id,record_id:s.task_record_id,refresh:!1}),e=await i.findOneBy({id:s.root_id}),d=await a.save(a.create({is_attachment:!0,root_id:s.root_id,pid:0,space_id:t.space_id,space_name:s.space_name,uid:s.uid,input:e.input,cloud_id:t.id,cloud_name:t.name,type:t.type}));o.pid=d.id}}else if(e)o.pid=e.id;else{const t=await i.findOneBy({id:s.root_id}),e=await a.save(a.create({is_attachment:!1,root_id:s.root_id,pid:0,space_id:t.space_id,space_name:t.space_name,uid:t.uid,input:t.input,cloud_id:t.cloud_info.id,cloud_name:t.cloud_info.name,type:t.type}));o.pid=e.id}await a.update({id:o.pid},{updated_at:new Date})}await a.save(a.create(o)),await i.delete({id:t.file_id}),await e.delete({file_id:t.file_id}),await(0,S.remove)(s.output_root),r||(await i.findOneBy({root_id:s.root_id,type:(0,b.Not)(R.RecordType.FOLDER)})?await i.increment({id:s.root_id},"completed_count",1):(await i.delete({root_id:s.root_id}),await i.delete({id:s.root_id})))}({file_id:e.id,sdk_domain:this.params.sdk_domain,fileRepository:this.fileRepository,partRepository:this.partRepository,successRepository:this.successRepository})}catch(t){await this.fileRepository.update(e.id,{upload_status:p.UploadStatus.UPLOAD_ERROR,error:{type:"upload",message:t.message,details:t.details,cause:t.cause,stack:t.stack}})}finally{Y.delete(i)}}catch(t){}}async onFileError(t,i){try{this.params.onError&&await this.params.onError(t,i)}catch(t){}}async onFileProgress(t){try{this.params.onProgress&&await this.params.onProgress(t)}catch(t){}}async dequeueOneFile(t){for(;this.isDequeueing;)await new Promise(t=>{setTimeout(t,50)});let i=null;try{this.isDequeueing=!0;const e=`${t}_host_id`;if(i=await this.fileRepository.findOne({where:{uid:this.params.uid,is_paused:!1,upload_status:p.UploadStatus.UPLOAD_READY,[e]:(0,l.Not)((0,l.IsNull)())},order:{id:"ASC"}}),!i)return;if(i.cloud_info?.id||(await U({file_id:i.id,sdk_domain:this.params.sdk_domain,fileRepository:this.fileRepository}),i=await this.fileRepository.findOneBy({id:i.id})),!i.cloud_info?.id)throw new Error("Created cloud folder or file failed");const{encoded:a}=i,s=[],o=[p.HostType.BAIDU,p.HostType.ALI];for await(const t of o){const e=`${t}_host_id`;if(!i[e])continue;const o=i[e],r=a[t].parts||[],{length:d}=r;for(let e=0;e<d;e++){const a=r[e];await this.partRepository.findOneBy({file_id:i.id,part_name:a.file,host_id:o})||s.push({uid:this.params.uid,file_id:i.id,root_id:i.root_id??i.id,part_name:a.file,info:a,output_root:i.output_root,host_id:o,host_type:t})}}if(s.length)await this.partRepository.save(s),await this.fileRepository.update(i.id,{upload_status:p.UploadStatus.UPLOAD_PROCESS});else{await this.fileRepository.update(i.id,{upload_status:p.UploadStatus.UPLOAD_PROCESS});const e=await this.partRepository.findOneBy({file_id:i.id,upload_status:(0,l.Not)(p.NormalStatus.SUCCESS)});await this.partRepository.findOneBy({file_id:i.id,upload_status:p.NormalStatus.SUCCESS});e||await this.onPartSuccess({file_id:i.id});await this.partRepository.findOneBy({file_id:i.id,upload_status:p.NormalStatus.NULL})||(this.isDequeueing=!1,await this.dequeueOneFile(t))}}catch(e){if(!i)throw e;{const a=(0,j.buildDError)(e,{message:"Error occurred during baidu preparing",detail:`Error occurred during baidu preparing: ${i.input.filepath}`});await this.fileRepository.update(i.id,{upload_status:p.UploadStatus.UPLOAD_ERROR,error:{type:"upload",message:a.message,details:a.details,cause:a.cause,stack:a.stack}}),this.isDequeueing=!1,await this.dequeueOneFile(t)}}finally{this.isDequeueing=!1}}getThreadCount(t){return this.threadConfig[t]??2}},V=new Map,G=async t=>{for(;V.get(t.uid)?.initing;)await new Promise(t=>setTimeout(t,100));try{if(V.get(t.uid)?.uploader)return V.get(t.uid).uploader;const i={uploader:new Q(t),initing:!0};return V.set(t.uid,i),await i.uploader.init(),i.uploader}finally{V.get(t.uid)&&(V.get(t.uid).initing=!1)}};

package/dist/main.mjs

@@ -0,0 +1 @@
+var i=(i=>"undefined"!=typeof require?require:"undefined"!=typeof Proxy?new Proxy(i,{get:(i,t)=>("undefined"!=typeof require?require:i)[t]}):i)(function(i){if("undefined"!=typeof require)return require.apply(this,arguments);throw Error('Dynamic require of "'+i+'" is not supported')});import{NormalStatus as t,HostType as e,UploadStatus as s}from"@soga/types";import{IsNull as a,Not as o}from"typeorm";import{UploadFile as r,UploadPart as d,UploadRunning as n,UploadRunningType as p,UploadSuccess as u}from"@soga/entities";import{getSdk as l}from"@soga/sdk";import{HostType as c,RecordType as f}from"@soga/types";import{v4 as h}from"uuid";var _="1.0.0";import{calculateManifestHostSize as m}from"@soga/utils";var y=new Map;import{getSdk as w}from"@soga/sdk";import{RecordType as g}from"@soga/types";import{remove as R}from"fs-extra";import{Not as b}from"typeorm";import{RecordType as P}from"@soga/types";import{getSdk as S}from"@soga/sdk";async function O({file_id:i,sdk_domain:t,fileRepository:e}){const s=await e.findOneBy({id:i});if(s.is_attachment){const{task_record_id:i}=s,a=S(t),r=await a.getRecordInfo({space_id:s.space_id,record_id:i,refresh:!0}),d=await o(r);await e.update(s.id,{cloud_info:d})}else await async function(){const t=await e.findOneBy({id:i});if(!t)return;const s=await a(t.pid,[]);for(const i of s)await r(i);return s}(),await r(s);async function a(i,t=[]){try{if(0===i)return t;const s=await e.findOneBy({id:i});return s?s.cloud_info?.id?t:(t.unshift(s),0!==s.pid?await a(s.pid,t).catch(i=>{throw i}):t):t}catch(i){throw i}}async function o(i){const{id:t,name:a,cloud_info:o}=i,r={id:t,name:a,hosts:{}};if(o.ali&&(r.hosts.ali={id:o.ali.id,name:o.ali.name,drive_id:o.ali.drive_id,file_id:o.ali.file_id}),o.baidu&&(r.hosts.baidu={id:o.baidu.id,name:o.baidu.name,fs_id:o.baidu.fs_id,path:o.baidu.path},!r.hosts.baidu.path)){const i=await e.findOneBy({id:s.pid}),{path:t}=i.cloud_info.hosts.baidu;t&&(r.hosts.baidu.path=`${t}/${o.baidu.name}`)}return r}async function r(i){const s=await e.findOneBy({id:i.id});if(s.cloud_info?.id)return;const{input:a,space_id:r,task_record_id:d,type:n}=s,p=n===P.FOLDER;let u=d;if(0!==s.pid){u=(await e.findOneBy({id:s.pid})).cloud_info.id}const l=S(t),c={space_id:r,name:a.filename,parent_id:u,type:s.type,ftype:s.ftype},f={};s.baidu_host_id&&(f.baidu=0),s.ali_host_id&&(f.ali=0);const h=await l.createRecord({...c,manifest:JSON.stringify({version:_,complete:!!p,hosts:f,metas:{file:{btime:s.input.local_btime,ctime:s.input.local_ctime,mtime:s.input.local_mtime,file:s.input.filename}}})}),m=await o(h);await e.update({id:s.id},{cloud_info:m})}}import{UploadFile as E,UploadPart as A,UploadRunning as B,UploadRunningType as U}from"@soga/entities";import{NormalStatus as C,UploadStatus as D}from"@soga/types";import{getProgressPercent as k}from"@soga/utils";import{UploadProgress as T}from"@soga/node-types";var v=class{hostType;onProgress=async()=>{};fileRepository;partRepository;runningRepository;isRunning=!1;thread_count=2;maxThreads=8;threads=[];baseParams;getValidThreads(i){return Math.min(Math.max(i??0,0),this.maxThreads)}async onPartProgress(i){const{id:t,data:e,type:s}=i;if("percent"!=s)return;const{part_id:a,percent:o}=e,r=await this.partRepository.findOneBy({id:a});if(!r)return;if(r.upload_percent>=o)return;if(r.upload_status==C.ERROR)return;if(r.upload_status==C.SUCCESS)return;await this.partRepository.update(a,{upload_percent:o});const d=await this.partRepository.findBy({file_id:t,host_type:this.hostType});let n=0,p=0;for(const i of d)n+=i.info.size,p+=i.info.size*i.upload_percent;const u=p/n,l=await this.fileRepository.findOneBy({id:t}),c=T[`upload_${this.hostType}`];l.progress[c].percent=u,l.percent=k(l.progress),await this.fileRepository.update(t,{progress:l.progress,percent:l.percent}),await this.onProgress(l)}async onPartError({error:i,part_id:t,file_id:e}){const s=await this.fileRepository.findOneBy({id:e});s&&(await this.fileRepository.update(e,{upload_status:D.UPLOAD_ERROR,error:{type:"upload",message:i.message,details:i.details,cause:i.cause,stack:i.stack}}),await this.partRepository.update(t,{upload_status:C.ERROR}),await this.fileRepository.update(s.id,{upload_status:D.UPLOAD_ERROR}),await this.baseParams.onError(i,s.id))}constructor(i,t){this.hostType=t,this.baseParams=i;const{dataSource:e}=i;this.fileRepository=e.getRepository(E),this.partRepository=e.getRepository(A),this.runningRepository=e.getRepository(B),i.onProgress&&(this.onProgress=i.onProgress.bind(this))}async setThreads(i){const t=this.getValidThreads(i);this.thread_count=t,await this.start()}async getParts(i){const t={uid:this.baseParams.uid,is_paused:!1,upload_status:C.NULL,host_id:this.baseParams.host_id};for(let e=0;e<i;e++){const e=await this.partRepository.countBy(t);if(e>=i)break;if(e<i){await this.baseParams.onDequeue(this.hostType);if(await this.partRepository.countBy(t)==e)break}}return await this.partRepository.find({where:t,order:{id:"ASC"},take:i})}async start(){for(;this.isRunning;)await new Promise(i=>{setTimeout(i,50)});try{this.isRunning=!0;const i=this.threads.length,t=this.thread_count;if(i<t){const e=t-i,s=await this.getParts(e),a=s.length;for(let i=0;i<a;i++){const t=s[i],e=this.getThread(t);(await this.fileRepository.findOneBy({id:t.file_id})).is_paused||(this.threads.push(e),await this.partRepository.update(t.id,{upload_status:C.PROCESS}),e.start().catch(async i=>{}))}}else if(i>t){const e=i-t,s=this.threads.slice(0,e);for(const i of s)await i.stop()}}finally{this.isRunning=!1}}async stopFiles(i){const t=this.threads.filter(t=>i.includes(t.file_id));await Promise.all(t.map(i=>i.stop()))}async stopAll(){await Promise.all(this.threads.map(i=>i.stop()))}async addRunning({file_id:i,part_id:t,root_id:e}){await this.runningRepository.save(this.runningRepository.create({type:U.UPLOAD,uid:this.baseParams.uid,root_id:e,file_id:i,part_id:t}))}async deleteRunning({file_id:i,part_id:t}){await this.runningRepository.delete({uid:this.baseParams.uid,file_id:i,part_id:t})}};import{HostType as L,NormalStatus as I,UploadStatus as q}from"@soga/types";import M from"piscina";import{MessageChannel as x}from"worker_threads";import{buildDError as z}from"@soga/error";var F=new M({filename:i.resolve("@soga/part-uploader")}),N=class extends v{params;constructor(i){super(i,L.BAIDU),this.params=i}getThread(i){const t=new AbortController,{port1:e,port2:s}=new x;return{file_id:i.file_id,part_id:i.id,uid:i.uid,start:async()=>{let a=null;try{if(a=await this.fileRepository.findOneBy({id:i.file_id}),!a)return;const{file_id:o,host_id:r,id:d,info:n,upload_status:p,root_id:u}=i;if(p==I.SUCCESS)return;await this.addRunning({file_id:o,part_id:d,root_id:u});const{output_root:l}=a;s.on("message",async i=>{await this.onPartProgress(i)});const c={file_id:o,host_id:r,part_id:d,output_root:l,part:n,cloud_folder_path:a.cloud_info.hosts.baidu?.path,sdk_domain:this.params.sdk_domain,port:e,appid:30045106},f=await F.run(c,{name:"uploadBaidu",signal:t.signal,transferList:[e]});if(!f?.fs_id)throw new Error("Upload to Baidu failed, result is null");await this.partRepository.update(i.id,{upload_result:{baidu:f},upload_status:I.SUCCESS}),await this.baseParams.onPartSuccess(o)}catch(e){if(t.abort(),"AbortError"==e.name)await this.partRepository.update(i.id,{upload_status:I.NULL});else{const t=a?a.input.filepath:"unknown filepath",s=z(e,{message:"Error occurred during baidu uploading",detail:`Error occurred during baidu uploading: ${t}`});await this.fileRepository.update(i.file_id,{upload_status:q.UPLOAD_ERROR,error:{type:"upload",message:s.message,details:s.details,cause:s.cause,stack:s.stack}}),await this.onPartError({error:e,part_id:i.id,file_id:i.file_id})}}finally{this.threads=this.threads.filter(t=>t.part_id!=i.id),s.close(),await this.deleteRunning({file_id:i.file_id,part_id:i.id}),await this.start()}},stop:async()=>{t.abort(),this.threads=this.threads.filter(t=>t.part_id!==i.id)}}}};import{HostType as $,NormalStatus as H,UploadStatus as J}from"@soga/types";import Y from"piscina";import{MessageChannel as j}from"worker_threads";import{buildDError as Q}from"@soga/error";var V=new Y({filename:i.resolve("@soga/part-uploader")}),G=class extends v{params;constructor(i){super(i,$.ALI),this.params=i}getThread(i){const t=new AbortController,{port1:e,port2:s}=new j;return{file_id:i.file_id,part_id:i.id,uid:i.uid,start:async()=>{const a=await this.fileRepository.findOneBy({id:i.file_id});if(a)try{const{file_id:o,host_id:r,id:d,info:n,upload_status:p,root_id:u}=i;if(p==H.SUCCESS)return;await this.addRunning({file_id:o,part_id:d,root_id:u});const{output_root:l}=a;s.on("message",async i=>{await this.onPartProgress(i)});const c={file_id:o,host_id:r,part_id:d,output_root:l,part:n,cloud_folder_id:a.cloud_info.hosts.ali?.file_id,sdk_domain:this.params.sdk_domain,port:e},f=await V.run(c,{name:"uploadAli",signal:t.signal,transferList:[e]});if(!f?.file_id)throw new Error("Upload to Ali failed, result is null");await this.partRepository.update(i.id,{upload_result:{ali:f},upload_status:H.SUCCESS}),await this.baseParams.onPartSuccess(o)}catch(e){if(t.abort(),"AbortError"==e.name)this.partRepository.update(i.id,{upload_status:H.NULL});else{const t=Q(e,{message:"Error occurred during ali uploading",detail:`Error occurred during ali uploading: ${a.input.filepath}`});await this.fileRepository.update(i.file_id,{upload_status:J.UPLOAD_ERROR,error:{type:"upload",message:t.message,details:t.details,cause:t.cause,stack:t.stack}}),await this.onPartError({error:e,part_id:i.id,file_id:i.file_id})}}finally{this.threads=this.threads.filter(t=>t.part_id!=i.id),s.close(),await this.deleteRunning({file_id:i.file_id,part_id:i.id}),await this.start()}},stop:async()=>{t.abort(),this.threads=this.threads.filter(t=>t.part_id!==i.id)}}}};import{buildDError as K}from"@soga/error";import{getProgressPercent as W}from"@soga/utils";import{UploadProgress as X}from"@soga/node-types";var Z=new Map,ii=class{params;uid;dataSource;fileRepository;partRepository;runningRepository;successRepository;processorMap=new Map;isDequeueing=!1;threadConfig={baidu:4,ali:2};constructor(i){this.params=i,this.uid=i.uid,this.dataSource=i.dataSource,this.fileRepository=this.dataSource.getRepository(r),this.partRepository=this.dataSource.getRepository(d),this.runningRepository=this.dataSource.getRepository(n),this.successRepository=this.dataSource.getRepository(u)}async init(){await this.runningRepository.delete({uid:this.uid,type:p.UPLOAD}),await this.fileRepository.update({uid:this.uid,upload_status:s.UPLOAD_ERROR},{upload_status:s.UPLOAD_READY}),await this.partRepository.update({uid:this.uid,upload_status:t.PROCESS},{upload_status:t.NULL})}async start(){await this.startCreates(),await Promise.all([this.startBaidu(),this.startAli()]).catch(i=>{console.error("Error starting uploader:",i)})}async stopFiles(i){const t=this.processorMap.values();for(const e of t)await e.stopFiles(i)}async stopAll(){const i=this.processorMap.values();for(const t of i)await t.stopAll()}async setThreadCount(i,t){this.threadConfig[i]=t;const e=this.processorMap.values();for(const s of e)s.hostType==i&&await s.setThreads(t)}isCreating=!1;async startCreates(){for(;this.isCreating;)await new Promise(i=>setTimeout(i,100));try{this.isCreating=!0;const i=await this.fileRepository.findBy({uid:this.uid,upload_status:s.SUBMIT_READY});if(!i.length)return;for(const t of i)await this.onPartSuccess({file_id:t.id})}finally{this.isCreating=!1}}async startHost(i,t){if(this.processorMap.has(i)){const t=this.processorMap.get(i),e=this.getThreadCount(t.hostType);return void await t.setThreads(e)}const s={uid:this.uid,sdk_domain:this.params.sdk_domain,dataSource:this.dataSource,host_id:i,onProgress:async i=>{await this.onFileProgress(i)},onPartSuccess:async i=>{await this.onPartSuccess({file_id:i})},onError:async(i,t)=>{await this.onFileError(i,t)},onDequeue:async i=>{await this.dequeueOneFile(i)}};if(t==e.BAIDU){const t=new N(s);this.processorMap.set(i,t);const a=this.getThreadCount(e.BAIDU);await t.setThreads(a)}else if(t==e.ALI){const t=new G(s);this.processorMap.set(i,t);const a=this.getThreadCount(e.ALI);await t.setThreads(a)}}async startBaidu(){const i=(await this.fileRepository.createQueryBuilder("file").select("DISTINCT file.baidu_host_id").where("file.uid = :uid",{uid:this.uid}).getRawMany()).map(i=>i.baidu_host_id).filter(i=>!!i);if(i.length)for(const t of i)await this.startHost(t,e.BAIDU)}async startAli(){const i=(await this.fileRepository.createQueryBuilder("file").select("DISTINCT file.ali_host_id").where("file.uid = :uid",{uid:this.uid}).getRawMany()).map(i=>i.ali_host_id).filter(i=>!!i);if(i.length)for(const t of i)await this.startHost(t,e.ALI)}async onPartSuccess({file_id:i}){await this.partRepository.findOneBy({file_id:i,upload_status:o(t.SUCCESS)})||await this.onFileSuccess(i)}async onFileSuccess(i){try{const{uid:t}=this;for(;Z.get(t);)await new Promise(i=>setTimeout(i,100));Z.set(t,!0);const a=await this.fileRepository.findOneBy({id:i});if(!a)return;try{const t=await this.partRepository.findBy({file_id:a.id}),s={},o=a.progress;if(a.baidu_host_id){s.baidu_upload_result={};for(const i of t)i.host_type==e.BAIDU&&(s.baidu_upload_result[i.info.file]={...i.upload_result.baidu,md4:i.info.md4});o[X.upload_baidu].percent=1}if(a.ali_host_id){s.ali_upload_result={};for(const i of t)i.host_type==e.ALI&&(s.ali_upload_result[i.info.file]={...i.upload_result.ali,sha1:i.info.sha1});o[X.upload_ali].percent=1}const r=W(o);await this.fileRepository.update(a.id,{...s,progress:o,percent:r}),await async function({file_id:i,sdk_domain:t,fileRepository:e}){try{for(;y.get(i);)await new Promise(i=>setTimeout(i,100));y.set(i,!0);const s=await e.findOneBy({id:i});let a=s.task_record_id;const o=l(t),{encoded:r,baidu_host_id:d,ali_host_id:n}=s,{ali:p,baidu:u,meta:w}=r,g=[];if(d&&g.push(c.BAIDU),n&&g.push(c.ALI),!g.length)return;const R=i=>{const t=r[i];if(!t.media?.subtitles?.length)return;const e=[];return t.media.subtitles.forEach(i=>{const{uuid:t,lang:s,name:a,title:o,builtin:r}=i;e.push({uuid:t,lang:s,name:a,title:o,builtin:r})}),e},b=i=>{if(!i)return;if(!i.subtitles)return i;const{subtitles:t,...e}=i,s={};return t.forEach(i=>{const{uuid:t,file:e,start:a,end:o,size:r,source:d}=i;s[t]={file:e,start:a,end:o,size:r,source:d}}),{...e,subtitles:s}};if(s.is_attachment){const i=await o.getRecordInfo({space_id:s.space_id,record_id:s.task_record_id,refresh:!0}),t=Object.assign({version:_,metas:{},hosts:{},attachments:[]},i.manifest);i.type,f.FOLDER;const e={metas:w,uuid:h(),type:s.type,ftype:s.ftype,title:s.input.filename,subtitles:R(g[0])};if(d){const{parts:i,max_size:t,media:a,img:o,txt:r,source:n}=u;e.baidu={host_id:d,media:b(a),max_size:t,img:o,txt:r,source:n,files:{}};const p=s.baidu_upload_result;i.forEach(i=>{const{file:t,md4:s,preview:a,source:o,size:r,md5:d}=i,{fs_id:n}=p[t];e.baidu.files[t]={fs_id:n,md5:d,md4:s,size:r,preview:a,source:o}})}if(n){const{parts:i,max_size:t,media:a,img:o,txt:r,source:d}=p,{drive_id:u}=s.cloud_info.hosts.ali;e.ali={host_id:n,drive_id:u,media:b(a),max_size:t,img:o,txt:r,source:d,files:{}};const l=s.ali_upload_result;i.forEach(i=>{const{file:t,preview:s,source:a,size:o,md5:r}=i,{file_id:d,sha1:n}=l[t];e.ali.files[t]={file_id:d,md5:r,sha1:n,size:o,preview:s,source:a}})}t.attachments=t.attachments||[],t.attachments.push(e),t.hosts||(t.hosts={}),d&&(t.hosts.baidu=m(t,c.BAIDU)),n&&(t.hosts.ali=m(t,c.ALI)),await o.updateRecord({space_id:i.space_id,record_id:i.id,parent_id:i.parent_id,manifest:JSON.stringify(t)})}else{0!==s.pid&&(a=(await e.findOneBy({id:s.pid})).cloud_info.id);const i={version:_,hosts:{},complete:!0,metas:w,subtitles:R(g[0])};if(d){const{parts:t,media:e,...a}=u,o=b(e),r=s.baidu_upload_result;i.baidu={host_id:d,...a,files:{},media:o},t.forEach(t=>{const{file:e,md4:s,preview:a,source:o,size:d,md5:n}=t,{fs_id:p}=r[e];i.baidu.files[e]={fs_id:p,md5:n,md4:s,size:d,preview:a,source:o}})}if(n){const{parts:t,media:e,...a}=p,o=b(e),r=s.ali_upload_result;i.ali={host_id:n,drive_id:"",...a,media:o,files:{}},t.forEach(t=>{const{file:e,preview:s,source:a,size:o,md5:d}=t,{file_id:n,drive_id:p,sha1:u}=r[e];i.ali.drive_id||(i.ali.drive_id=p),i.ali.files[e]={file_id:n,sha1:u,md5:d,size:o,preview:s,source:a}});const d=Object.values(i.ali.files).reduce((i,t)=>i+t.size,0);i.hosts.ali=d}if(d&&(i.hosts.baidu=m(i,c.BAIDU)),n&&(i.hosts.ali=m(i,c.ALI)),s.cloud_info?.id){const i=await o.getRecordInfo({space_id:s.space_id,record_id:s.cloud_info.id,refresh:!0});if(i.manifest?.complete)throw new Error("the file has been modified!")}await o.updateRecord({space_id:s.space_id,record_id:s.cloud_info.id,parent_id:a,manifest:JSON.stringify(i)})}}catch(i){throw i}finally{y.delete(i)}}({file_id:a.id,fileRepository:this.fileRepository,sdk_domain:this.params.sdk_domain}),this.params.onSuccess&&await this.params.onSuccess(i),await async function(i){const{fileRepository:t,partRepository:e,successRepository:s}=i,a=await t.findOneBy({id:i.file_id}),o={is_attachment:a.is_attachment,root_id:a.root_id,pid:0,space_id:a.space_id,space_name:a.space_name,uid:a.uid,input:a.input,cloud_id:a.cloud_info.id,cloud_name:a.cloud_info.name,type:a.type},r=0==a.root_id;if(!r){const e=await s.findOneBy({root_id:a.root_id,pid:0});if(a.is_attachment){const r=w(i.sdk_domain);if(e)o.pid=e.id;else{const i=await r.getRecordInfo({space_id:a.space_id,record_id:a.task_record_id,refresh:!1}),e=await t.findOneBy({id:a.root_id}),d=await s.save(s.create({is_attachment:!0,root_id:a.root_id,pid:0,space_id:i.space_id,space_name:a.space_name,uid:a.uid,input:e.input,cloud_id:i.id,cloud_name:i.name,type:i.type}));o.pid=d.id}}else if(e)o.pid=e.id;else{const i=await t.findOneBy({id:a.root_id}),e=await s.save(s.create({is_attachment:!1,root_id:a.root_id,pid:0,space_id:i.space_id,space_name:i.space_name,uid:i.uid,input:i.input,cloud_id:i.cloud_info.id,cloud_name:i.cloud_info.name,type:i.type}));o.pid=e.id}await s.update({id:o.pid},{updated_at:new Date})}await s.save(s.create(o)),await t.delete({id:i.file_id}),await e.delete({file_id:i.file_id}),await R(a.output_root),r||(await t.findOneBy({root_id:a.root_id,type:b(g.FOLDER)})?await t.increment({id:a.root_id},"completed_count",1):(await t.delete({root_id:a.root_id}),await t.delete({id:a.root_id})))}({file_id:a.id,sdk_domain:this.params.sdk_domain,fileRepository:this.fileRepository,partRepository:this.partRepository,successRepository:this.successRepository})}catch(i){await this.fileRepository.update(a.id,{upload_status:s.UPLOAD_ERROR,error:{type:"upload",message:i.message,details:i.details,cause:i.cause,stack:i.stack}})}finally{Z.delete(t)}}catch(i){}}async onFileError(i,t){try{this.params.onError&&await this.params.onError(i,t)}catch(i){}}async onFileProgress(i){try{this.params.onProgress&&await this.params.onProgress(i)}catch(i){}}async dequeueOneFile(i){for(;this.isDequeueing;)await new Promise(i=>{setTimeout(i,50)});let r=null;try{this.isDequeueing=!0;const d=`${i}_host_id`;if(r=await this.fileRepository.findOne({where:{uid:this.params.uid,is_paused:!1,upload_status:s.UPLOAD_READY,[d]:o(a())},order:{id:"ASC"}}),!r)return;if(r.cloud_info?.id||(await O({file_id:r.id,sdk_domain:this.params.sdk_domain,fileRepository:this.fileRepository}),r=await this.fileRepository.findOneBy({id:r.id})),!r.cloud_info?.id)throw new Error("Created cloud folder or file failed");const{encoded:n}=r,p=[],u=[e.BAIDU,e.ALI];for await(const i of u){const t=`${i}_host_id`;if(!r[t])continue;const e=r[t],s=n[i].parts||[],{length:a}=s;for(let t=0;t<a;t++){const a=s[t];await this.partRepository.findOneBy({file_id:r.id,part_name:a.file,host_id:e})||p.push({uid:this.params.uid,file_id:r.id,root_id:r.root_id??r.id,part_name:a.file,info:a,output_root:r.output_root,host_id:e,host_type:i})}}if(p.length)await this.partRepository.save(p),await this.fileRepository.update(r.id,{upload_status:s.UPLOAD_PROCESS});else{await this.fileRepository.update(r.id,{upload_status:s.UPLOAD_PROCESS});const e=await this.partRepository.findOneBy({file_id:r.id,upload_status:o(t.SUCCESS)});await this.partRepository.findOneBy({file_id:r.id,upload_status:t.SUCCESS});e||await this.onPartSuccess({file_id:r.id});await this.partRepository.findOneBy({file_id:r.id,upload_status:t.NULL})||(this.isDequeueing=!1,await this.dequeueOneFile(i))}}catch(t){if(!r)throw t;{const e=K(t,{message:"Error occurred during baidu preparing",detail:`Error occurred during baidu preparing: ${r.input.filepath}`});await this.fileRepository.update(r.id,{upload_status:s.UPLOAD_ERROR,error:{type:"upload",message:e.message,details:e.details,cause:e.cause,stack:e.stack}}),this.isDequeueing=!1,await this.dequeueOneFile(i)}}finally{this.isDequeueing=!1}}getThreadCount(i){return this.threadConfig[i]??2}},ti=new Map,ei=async i=>{for(;ti.get(i.uid)?.initing;)await new Promise(i=>setTimeout(i,100));try{if(ti.get(i.uid)?.uploader)return ti.get(i.uid).uploader;const t={uploader:new ii(i),initing:!0};return ti.set(i.uid,t),await t.uploader.init(),t.uploader}finally{ti.get(i.uid)&&(ti.get(i.uid).initing=!1)}};export{ii as Uploader,ei as getUploader};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@soga/uploader",
-  "version": "0.3.0",
+  "version": "1.3.1",
   "publishConfig": {
     "access": "public"
   },
@@ -10,28 +10,32 @@
   "files": [
     "dist"
   ],
-  "devDependencies": {
-    "@soga/encoder": "^0.3.0"
-  },
   "keywords": [],
   "author": "",
   "license": "ISC",
   "dependencies": {
-    "@soga/entities": "^0.3.0",
-    "@soga/part-uploader": "^0.3.0",
-    "@soga/sdk": "^0.3.0",
-    "@soga/types": "^0.3.0",
-    "@soga/utils": "^0.3.0",
     "piscina": "^5.0.0",
-    "typeorm": "^0.3.24"
+    "typeorm": "^0.3.24",
+    "fs-extra": "^11.3.2",
+    "uuid": "^11.1.0",
+    "@soga/entities": "^1.0.0",
+    "@soga/part-uploader": "^1.0.0",
+    "@soga/utils": "^1.0.0",
+    "@soga/types": "^1.0.1",
+    "@soga/sdk": "^1.0.0",
+    "@soga/node-types": "^1.0.0",
+    "@soga/error": "^1.0.8"
+  },
+  "devDependencies": {
+    "@types/node": "^24.5.2",
+    "rimraf": "^6.0.1",
+    "terser": "^5.43.1",
+    "tsup": "^8.5.0",
+    "typescript": "^5.8.3",
+    "@types/fs-extra": "^11.0.4",
+    "@soga/typescript-config": "^1.0.0"
   },
   "scripts": {
-    "build": "rimraf dist && tsc && ts-node ./scripts/minify",
-    "minify": "ts-node ./scripts/minify",
-    "demo_backup": "ts-node ./demo/demo.ts",
-    "demo": "ts-node ./demo/demo.ts",
-    "test": "jest",
-    "dev": "ts-node ./src/main.ts",
-    "lint": "eslint . --ext .ts"
+    "build": "rimraf dist && tsup src/main.ts --format cjs,esm --dts --minify terser"
   }
 }

package/dist/hooks/complete.d.ts

@@ -1,7 +0,0 @@
-import { UploadFile } from '@soga/entities';
-import { Repository } from 'typeorm';
-export declare function complete({ file_id, sdk_domain, fileRepository, }: {
-    file_id: number;
-    sdk_domain: string;
-    fileRepository: Repository<UploadFile>;
-}): Promise<void>;

package/dist/hooks/complete.js

@@ -1,210 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.complete = complete;
-const sdk_1 = require("@soga/sdk");
-const types_1 = require("@soga/types");
-async function complete({ file_id, sdk_domain, fileRepository, }) {
-    const file = await fileRepository.findOneBy({ id: file_id });
-    let pid = file.task_record_id;
-    const sdk = (0, sdk_1.getSdk)(sdk_domain);
-    const partMap = {};
-    let manifest;
-    if (file.type == types_1.RecordType.AFFIX) {
-        const record = await sdk.getRecordInfo({
-            space_id: file.space_id,
-            record_id: file.task_record_id,
-            refresh: true,
-        });
-        const { parent_id } = record;
-        pid = parent_id;
-        await processAffixManifest(file, sdk);
-    }
-    else {
-        if (file.pid !== 0) {
-            const parent = await fileRepository.findOneBy({ id: file.pid });
-            pid = parent.cloud_info.id;
-        }
-        await processNormalManifest(file);
-    }
-    await sdk.updateRecord({
-        space_id: file.space_id,
-        record_id: file.cloud_info.id,
-        parent_id: pid,
-        manifest: JSON.stringify(manifest),
-    });
-    async function processAffixManifest(file, sdk) {
-        file.encoded.affix?.forEach((item) => {
-            item.parts.forEach((part) => {
-                partMap[part.file] = {
-                    md5: part.md5,
-                    size: part.size,
-                    source: true,
-                    preview: true,
-                };
-            });
-        });
-        const record = await sdk.getRecordInfo({
-            space_id: file.space_id,
-            record_id: file.task_record_id,
-            refresh: true,
-        });
-        if (!record.manifest) {
-            record.manifest = {};
-        }
-        manifest = record.manifest;
-        if (!manifest.meta) {
-            manifest.meta = {
-                host_size: 0,
-            };
-        }
-        if (!manifest.parts) {
-            manifest.parts = {};
-        }
-        if (!manifest.affix) {
-            manifest.affix = [];
-        }
-        file.encoded.affix?.forEach((item) => {
-            const { parts, ...fields } = item;
-            manifest.affix.push({ ...fields, parts: parts.map((item) => item.file) });
-        });
-        Object.assign(manifest.parts, partMap);
-        setManifestHosts();
-        setHostSize();
-    }
-    async function processNormalManifest(file) {
-        const { type, inputs, encoded } = file;
-        if (encoded.source?.parts) {
-            let preview_need = true;
-            if (type == types_1.RecordType.VIDEO || type == types_1.RecordType.AUDIO) {
-                preview_need = false;
-            }
-            if (type == types_1.RecordType.TXT) {
-                preview_need = false;
-            }
-            encoded.source.parts.forEach((part) => {
-                partMap[part.file] = {
-                    md5: part.md5,
-                    size: part.size,
-                    source: true,
-                    preview: preview_need,
-                };
-            });
-        }
-        if (encoded.media?.parts) {
-            encoded.media.parts.forEach((part) => {
-                partMap[part.file] = {
-                    md5: part.md5,
-                    size: part.size,
-                    source: false,
-                    preview: true,
-                };
-            });
-        }
-        if (encoded.img?.parts) {
-            encoded.img.parts.forEach((part) => {
-                partMap[part.file] = {
-                    md5: part.md5,
-                    size: part.size,
-                    source: false,
-                    preview: true,
-                };
-            });
-        }
-        if (encoded.txt?.parts) {
-            encoded.txt.parts.forEach((part) => {
-                partMap[part.file] = {
-                    md5: part.md5,
-                    size: part.size,
-                    source: false,
-                    preview: true,
-                };
-            });
-        }
-        const input = inputs[0];
-        const meta_field = {
-            host_size: 0,
-            filesize: input.filesize,
-            filename: input.filename,
-            btime: input.local_btime,
-            ctime: input.local_ctime,
-            mtime: input.local_mtime,
-        };
-        manifest = { meta: meta_field, parts: partMap };
-        if (encoded.source?.parts) {
-            manifest.source = {
-                head: encoded.source.head,
-                parts: encoded.source.parts.map((part) => part.file),
-            };
-        }
-        if (encoded.media?.parts) {
-            const { parts, ...fields } = encoded.media;
-            manifest.media = { ...fields, parts: parts.map((item) => item.file) };
-        }
-        if (encoded.img?.parts) {
-            const { meta, parts } = encoded.img;
-            manifest.img = {
-                meta,
-                preview: parts.map((item) => ({
-                    file: item.file,
-                    start: item.start,
-                    end: item.end,
-                })),
-                parts: parts.map((item) => item.file),
-            };
-        }
-        if (encoded.txt?.parts) {
-            const { map, pad, pages, parts } = encoded.txt;
-            manifest.txt = {
-                entrance: map,
-                pad,
-                pages,
-                parts: parts.map((item) => item.file),
-            };
-        }
-        setHostSize();
-        setManifestHosts();
-        return manifest;
-    }
-    function setHostSize() {
-        const parts = Object.values(partMap);
-        const sizes = { host: 0, preview: 0, source: 0 };
-        parts.forEach((part) => {
-            sizes.host += part.size;
-        });
-        manifest.meta.host_size = sizes.host;
-    }
-    function setManifestHosts() {
-        if (file.ali_host_id && !manifest.ali) {
-            manifest.ali = {
-                drive_id: '',
-                host_id: file.ali_host_id,
-                info: {},
-            };
-            const data = file.ali_upload_result;
-            Object.keys(partMap).forEach((filename) => {
-                const result = data[filename];
-                manifest.ali.info[filename] = {
-                    file_id: result.file_id,
-                    sha1: result.sha1,
-                };
-                if (!manifest.ali.drive_id) {
-                    manifest.ali.drive_id = result.drive_id;
-                }
-            });
-        }
-        if (file.baidu_host_id) {
-            manifest.baidu = {
-                host_id: file.baidu_host_id,
-                info: {},
-            };
-            const data = file.baidu_upload_result;
-            Object.keys(partMap).forEach((file) => {
-                const result = data[file];
-                manifest.baidu.info[file] = {
-                    fs_id: result.fs_id,
-                    md4: result.md4,
-                };
-            });
-        }
-    }
-}

package/dist/hooks/prepare.d.ts

@@ -1,7 +0,0 @@
-import { UploadFile } from '@soga/entities';
-import { Repository } from 'typeorm';
-export declare function prepare({ file_id, sdk_domain, fileRepository, }: {
-    file_id: number;
-    sdk_domain: string;
-    fileRepository: Repository<UploadFile>;
-}): Promise<void>;

package/dist/hooks/prepare.js

@@ -1,104 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.prepare = prepare;
-const types_1 = require("@soga/types");
-const sdk_1 = require("@soga/sdk");
-async function prepare({ file_id, sdk_domain, fileRepository, }) {
-    const file = await fileRepository.findOneBy({ id: file_id });
-    if (file.type != types_1.RecordType.AFFIX) {
-        await ensureParents();
-        await createCloudFolderOrFile(file);
-    }
-    else {
-        const { task_record_id } = file;
-        const sdk = (0, sdk_1.getSdk)(sdk_domain);
-        const record = await sdk.getRecordInfo({
-            space_id: file.space_id,
-            record_id: task_record_id,
-            refresh: true,
-        });
-        const cloud_info = await parseCloudInfo(record);
-        await fileRepository.update(file.id, {
-            cloud_info,
-        });
-    }
-    async function getLocalFolderList(id, list = []) {
-        if (id === 0) {
-            return list;
-        }
-        const file = await fileRepository.findOneBy({ id });
-        if (!file) {
-            return list;
-        }
-        if (!file.cloud_info?.id) {
-            list.unshift(file);
-            if (file.pid !== 0) {
-                return await getLocalFolderList(file.pid, list);
-            }
-            else {
-                return list;
-            }
-        }
-        else {
-            return list;
-        }
-    }
-    async function parseCloudInfo(record) {
-        const { id, name, cloud_info } = record;
-        const info = { id, name, hosts: {} };
-        if (cloud_info.ali) {
-            info.hosts.ali = {
-                id: cloud_info.ali.id,
-                name: cloud_info.ali.name,
-                drive_id: cloud_info.ali.drive_id,
-                file_id: cloud_info.ali.file_id,
-            };
-        }
-        if (cloud_info.baidu) {
-            info.hosts.baidu = {
-                id: cloud_info.baidu.id,
-                name: cloud_info.baidu.name,
-                fs_id: cloud_info.baidu.fs_id,
-                path: cloud_info.baidu.path,
-            };
-            if (!info.hosts.baidu.path) {
-                const parent = await fileRepository.findOneBy({ id: file.pid });
-                const { path: parentPath } = parent.cloud_info.hosts.baidu;
-                if (parentPath) {
-                    info.hosts.baidu.path = `${parentPath}/${cloud_info.baidu.name}`;
-                }
-            }
-        }
-        return info;
-    }
-    async function createCloudFolderOrFile(file) {
-        if (file.cloud_info?.id)
-            return;
-        const { inputs, space_id, task_record_id } = file;
-        let pid = task_record_id;
-        if (file.pid !== 0) {
-            const parent = await fileRepository.findOneBy({ id: file.pid });
-            pid = parent.cloud_info.id;
-        }
-        const sdk = (0, sdk_1.getSdk)(sdk_domain);
-        const recordInfo = await sdk.createRecord({
-            space_id,
-            name: inputs[0].filename,
-            parent_id: pid,
-            type: file.type,
-            ftype: types_1.RecordFtype.NONE,
-        });
-        const info = await parseCloudInfo(recordInfo);
-        await fileRepository.update({ id: file.id }, { cloud_info: info });
-    }
-    async function ensureParents() {
-        const file = await fileRepository.findOneBy({ id: file_id });
-        if (!file)
-            return;
-        const unCloudParentList = await getLocalFolderList(file.pid, []);
-        for (const item of unCloudParentList) {
-            await createCloudFolderOrFile(item);
-        }
-        return unCloudParentList;
-    }
-}

package/dist/hooks/trasform.d.ts

@@ -1,8 +0,0 @@
-import { UploadFile, UploadPart, UploadSuccess } from '@soga/entities';
-import { Repository } from 'typeorm';
-export declare function transform(params: {
-    file_id: number;
-    fileRepository: Repository<UploadFile>;
-    partRepository: Repository<UploadPart>;
-    successRepository: Repository<UploadSuccess>;
-}): Promise<void>;