@soga/uploader 0.2.45 → 0.2.47

package/dist/hooks/complete.js

@@ -1 +1,211 @@
-"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.complete=complete;const sdk_1=require("@soga/sdk"),types_1=require("@soga/types");async function complete({file_id:e,sdk_domain:i,fileRepository:s}){const t=await s.findOneBy({id:e});let a=t.task_record_id;const r=(0,sdk_1.getSdk)(i),d={};let o;if(t.type==types_1.RecordType.AFFIX){const e=await r.getRecordInfo({space_id:t.space_id,record_id:t.task_record_id,refresh:!0}),{parent_id:i}=e;a=i,await async function(e,i){e.encoded.affix?.forEach((e=>{e.parts.forEach((e=>{d[e.file]={md5:e.md5,size:e.size,source:!0,preview:!0}}))}));const s=await i.getRecordInfo({space_id:e.space_id,record_id:e.task_record_id,refresh:!0});s.manifest||(s.manifest={});o=s.manifest,o.meta||(o.meta={host_size:0});o.parts||(o.parts={});o.affix||(o.affix=[]);e.encoded.affix?.forEach((e=>{const{parts:i,...s}=e;o.affix.push({...s,parts:i.map((e=>e.file))})})),Object.assign(o.parts,d),p(),c()}(t,r)}else{if(0!==t.pid){a=(await s.findOneBy({id:t.pid})).cloud_info.id}await async function(e){const{type:i,inputs:s,encoded:t}=e;if(t.source?.parts){let e=!0;i!=types_1.RecordType.VIDEO&&i!=types_1.RecordType.AUDIO||(e=!1),i==types_1.RecordType.TXT&&(e=!1),t.source.parts.forEach((i=>{d[i.file]={md5:i.md5,size:i.size,source:!0,preview:e}}))}t.media?.parts&&t.media.parts.forEach((e=>{d[e.file]={md5:e.md5,size:e.size,source:!1,preview:!0}}));t.img?.parts&&t.img.parts.forEach((e=>{d[e.file]={md5:e.md5,size:e.size,source:!1,preview:!0}}));t.txt?.parts&&t.txt.parts.forEach((e=>{d[e.file]={md5:e.md5,size:e.size,source:!1,preview:!0}}));const a=s[0],r={host_size:0,filesize:a.filesize,filename:a.filename,btime:a.local_btime,ctime:a.local_ctime,mtime:a.local_mtime};o={meta:r,parts:d},t.source?.parts&&(o.source={head:t.source.head,parts:t.source.parts.map((e=>e.file))});if(t.media?.parts){const{parts:e,...i}=t.media;o.media={...i,parts:e.map((e=>e.file))}}if(t.img?.parts){const{meta:e,parts:i}=t.img;o.img={meta:e,preview:i.map((e=>({file:e.file,start:e.start,end:e.end}))),parts:i.map((e=>e.file))}}if(t.txt?.parts){const{map:e,pad:i,pages:s,parts:a}=t.txt;o.txt={entrance:e,pad:i,pages:s,parts:a.map((e=>e.file))}}return c(),p(),o}(t)}function c(){const e=Object.values(d),i={host:0,preview:0,source:0};e.forEach((e=>{i.host+=e.size})),o.meta.host_size=i.host}function p(){if(t.ali_host_id&&!o.ali){o.ali={drive_id:"",host_id:t.ali_host_id,info:{}};const e=t.ali_upload_result;console.log("part_map: ",Object.keys(d)),Object.keys(d).forEach((i=>{const s=e[i];o.ali.info[i]={file_id:s.file_id,sha1:s.sha1},o.ali.drive_id||(o.ali.drive_id=s.drive_id)}))}if(t.baidu_host_id){o.baidu={host_id:t.baidu_host_id,info:{}};const e=t.baidu_upload_result;Object.keys(d).forEach((i=>{const s=e[i];o.baidu.info[i]={fs_id:s.fs_id,md4:s.md4}}))}}await r.updateRecord({space_id:t.space_id,record_id:t.cloud_info.id,parent_id:a,manifest:JSON.stringify(o)})}
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.complete = complete;
+const sdk_1 = require("@soga/sdk");
+const types_1 = require("@soga/types");
+async function complete({ file_id, sdk_domain, fileRepository, }) {
+    const file = await fileRepository.findOneBy({ id: file_id });
+    let pid = file.task_record_id;
+    const sdk = (0, sdk_1.getSdk)(sdk_domain);
+    const partMap = {};
+    let manifest;
+    if (file.type == types_1.RecordType.AFFIX) {
+        const record = await sdk.getRecordInfo({
+            space_id: file.space_id,
+            record_id: file.task_record_id,
+            refresh: true,
+        });
+        const { parent_id } = record;
+        pid = parent_id;
+        await processAffixManifest(file, sdk);
+    }
+    else {
+        if (file.pid !== 0) {
+            const parent = await fileRepository.findOneBy({ id: file.pid });
+            pid = parent.cloud_info.id;
+        }
+        await processNormalManifest(file);
+    }
+    await sdk.updateRecord({
+        space_id: file.space_id,
+        record_id: file.cloud_info.id,
+        parent_id: pid,
+        manifest: JSON.stringify(manifest),
+    });
+    async function processAffixManifest(file, sdk) {
+        file.encoded.affix?.forEach((item) => {
+            item.parts.forEach((part) => {
+                partMap[part.file] = {
+                    md5: part.md5,
+                    size: part.size,
+                    source: true,
+                    preview: true,
+                };
+            });
+        });
+        const record = await sdk.getRecordInfo({
+            space_id: file.space_id,
+            record_id: file.task_record_id,
+            refresh: true,
+        });
+        if (!record.manifest) {
+            record.manifest = {};
+        }
+        manifest = record.manifest;
+        if (!manifest.meta) {
+            manifest.meta = {
+                host_size: 0,
+            };
+        }
+        if (!manifest.parts) {
+            manifest.parts = {};
+        }
+        if (!manifest.affix) {
+            manifest.affix = [];
+        }
+        file.encoded.affix?.forEach((item) => {
+            const { parts, ...fields } = item;
+            manifest.affix.push({ ...fields, parts: parts.map((item) => item.file) });
+        });
+        Object.assign(manifest.parts, partMap);
+        setManifestHosts();
+        setHostSize();
+    }
+    async function processNormalManifest(file) {
+        const { type, inputs, encoded } = file;
+        if (encoded.source?.parts) {
+            let preview_need = true;
+            if (type == types_1.RecordType.VIDEO || type == types_1.RecordType.AUDIO) {
+                preview_need = false;
+            }
+            if (type == types_1.RecordType.TXT) {
+                preview_need = false;
+            }
+            encoded.source.parts.forEach((part) => {
+                partMap[part.file] = {
+                    md5: part.md5,
+                    size: part.size,
+                    source: true,
+                    preview: preview_need,
+                };
+            });
+        }
+        if (encoded.media?.parts) {
+            encoded.media.parts.forEach((part) => {
+                partMap[part.file] = {
+                    md5: part.md5,
+                    size: part.size,
+                    source: false,
+                    preview: true,
+                };
+            });
+        }
+        if (encoded.img?.parts) {
+            encoded.img.parts.forEach((part) => {
+                partMap[part.file] = {
+                    md5: part.md5,
+                    size: part.size,
+                    source: false,
+                    preview: true,
+                };
+            });
+        }
+        if (encoded.txt?.parts) {
+            encoded.txt.parts.forEach((part) => {
+                partMap[part.file] = {
+                    md5: part.md5,
+                    size: part.size,
+                    source: false,
+                    preview: true,
+                };
+            });
+        }
+        const input = inputs[0];
+        const meta_field = {
+            host_size: 0,
+            filesize: input.filesize,
+            filename: input.filename,
+            btime: input.local_btime,
+            ctime: input.local_ctime,
+            mtime: input.local_mtime,
+        };
+        manifest = { meta: meta_field, parts: partMap };
+        if (encoded.source?.parts) {
+            manifest.source = {
+                head: encoded.source.head,
+                parts: encoded.source.parts.map((part) => part.file),
+            };
+        }
+        if (encoded.media?.parts) {
+            const { parts, ...fields } = encoded.media;
+            manifest.media = { ...fields, parts: parts.map((item) => item.file) };
+        }
+        if (encoded.img?.parts) {
+            const { meta, parts } = encoded.img;
+            manifest.img = {
+                meta,
+                preview: parts.map((item) => ({
+                    file: item.file,
+                    start: item.start,
+                    end: item.end,
+                })),
+                parts: parts.map((item) => item.file),
+            };
+        }
+        if (encoded.txt?.parts) {
+            const { map, pad, pages, parts } = encoded.txt;
+            manifest.txt = {
+                entrance: map,
+                pad,
+                pages,
+                parts: parts.map((item) => item.file),
+            };
+        }
+        setHostSize();
+        setManifestHosts();
+        return manifest;
+    }
+    function setHostSize() {
+        const parts = Object.values(partMap);
+        const sizes = { host: 0, preview: 0, source: 0 };
+        parts.forEach((part) => {
+            sizes.host += part.size;
+        });
+        manifest.meta.host_size = sizes.host;
+    }
+    function setManifestHosts() {
+        if (file.ali_host_id && !manifest.ali) {
+            manifest.ali = {
+                drive_id: '',
+                host_id: file.ali_host_id,
+                info: {},
+            };
+            const data = file.ali_upload_result;
+            console.log('part_map: ', Object.keys(partMap));
+            Object.keys(partMap).forEach((filename) => {
+                const result = data[filename];
+                manifest.ali.info[filename] = {
+                    file_id: result.file_id,
+                    sha1: result.sha1,
+                };
+                if (!manifest.ali.drive_id) {
+                    manifest.ali.drive_id = result.drive_id;
+                }
+            });
+        }
+        if (file.baidu_host_id) {
+            manifest.baidu = {
+                host_id: file.baidu_host_id,
+                info: {},
+            };
+            const data = file.baidu_upload_result;
+            Object.keys(partMap).forEach((file) => {
+                const result = data[file];
+                manifest.baidu.info[file] = {
+                    fs_id: result.fs_id,
+                    md4: result.md4,
+                };
+            });
+        }
+    }
+}

package/dist/hooks/prepare.js

@@ -1 +1,104 @@
-"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.prepare=prepare;const types_1=require("@soga/types"),sdk_1=require("@soga/sdk");async function prepare({file_id:i,sdk_domain:a,fileRepository:e}){const d=await e.findOneBy({id:i});if(d.type!=types_1.RecordType.AFFIX)await async function(){const a=await e.findOneBy({id:i});if(!a)return;const d=await t(a.pid,[]);for(const i of d)await s(i);return d}(),await s(d);else{const{task_record_id:i}=d,t=(0,sdk_1.getSdk)(a),s=await t.getRecordInfo({space_id:d.space_id,record_id:i,refresh:!0}),o=await n(s);await e.update(d.id,{cloud_info:o})}async function t(i,a=[]){if(0===i)return a;const d=await e.findOneBy({id:i});return d?d.cloud_info?.id?a:(a.unshift(d),0!==d.pid?await t(d.pid,a):a):a}async function n(i){const{id:a,name:t,cloud_info:n}=i,s={id:a,name:t,hosts:{}};if(n.ali&&(s.hosts.ali={id:n.ali.id,name:n.ali.name,drive_id:n.ali.drive_id,file_id:n.ali.file_id}),n.baidu&&(s.hosts.baidu={id:n.baidu.id,name:n.baidu.name,fs_id:n.baidu.fs_id,path:n.baidu.path},!s.hosts.baidu.path)){const i=await e.findOneBy({id:d.pid}),{path:a}=i.cloud_info.hosts.baidu;a&&(s.hosts.baidu.path=`${a}/${n.baidu.name}`)}return s}async function s(i){if(i.cloud_info?.id)return;const{inputs:d,space_id:t,task_record_id:s}=i;let o=s;if(0!==i.pid){o=(await e.findOneBy({id:i.pid})).cloud_info.id}const r=(0,sdk_1.getSdk)(a),c=await r.createRecord({space_id:t,name:d[0].filename,parent_id:o,type:i.type,ftype:types_1.RecordFtype.NONE}),p=await n(c);await e.update({id:i.id},{cloud_info:p})}}
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.prepare = prepare;
+const types_1 = require("@soga/types");
+const sdk_1 = require("@soga/sdk");
+async function prepare({ file_id, sdk_domain, fileRepository, }) {
+    const file = await fileRepository.findOneBy({ id: file_id });
+    if (file.type != types_1.RecordType.AFFIX) {
+        await ensureParents();
+        await createCloudFolderOrFile(file);
+    }
+    else {
+        const { task_record_id } = file;
+        const sdk = (0, sdk_1.getSdk)(sdk_domain);
+        const record = await sdk.getRecordInfo({
+            space_id: file.space_id,
+            record_id: task_record_id,
+            refresh: true,
+        });
+        const cloud_info = await parseCloudInfo(record);
+        await fileRepository.update(file.id, {
+            cloud_info,
+        });
+    }
+    async function getLocalFolderList(id, list = []) {
+        if (id === 0) {
+            return list;
+        }
+        const file = await fileRepository.findOneBy({ id });
+        if (!file) {
+            return list;
+        }
+        if (!file.cloud_info?.id) {
+            list.unshift(file);
+            if (file.pid !== 0) {
+                return await getLocalFolderList(file.pid, list);
+            }
+            else {
+                return list;
+            }
+        }
+        else {
+            return list;
+        }
+    }
+    async function parseCloudInfo(record) {
+        const { id, name, cloud_info } = record;
+        const info = { id, name, hosts: {} };
+        if (cloud_info.ali) {
+            info.hosts.ali = {
+                id: cloud_info.ali.id,
+                name: cloud_info.ali.name,
+                drive_id: cloud_info.ali.drive_id,
+                file_id: cloud_info.ali.file_id,
+            };
+        }
+        if (cloud_info.baidu) {
+            info.hosts.baidu = {
+                id: cloud_info.baidu.id,
+                name: cloud_info.baidu.name,
+                fs_id: cloud_info.baidu.fs_id,
+                path: cloud_info.baidu.path,
+            };
+            if (!info.hosts.baidu.path) {
+                const parent = await fileRepository.findOneBy({ id: file.pid });
+                const { path: parentPath } = parent.cloud_info.hosts.baidu;
+                if (parentPath) {
+                    info.hosts.baidu.path = `${parentPath}/${cloud_info.baidu.name}`;
+                }
+            }
+        }
+        return info;
+    }
+    async function createCloudFolderOrFile(file) {
+        if (file.cloud_info?.id)
+            return;
+        const { inputs, space_id, task_record_id } = file;
+        let pid = task_record_id;
+        if (file.pid !== 0) {
+            const parent = await fileRepository.findOneBy({ id: file.pid });
+            pid = parent.cloud_info.id;
+        }
+        const sdk = (0, sdk_1.getSdk)(sdk_domain);
+        const recordInfo = await sdk.createRecord({
+            space_id,
+            name: inputs[0].filename,
+            parent_id: pid,
+            type: file.type,
+            ftype: types_1.RecordFtype.NONE,
+        });
+        const info = await parseCloudInfo(recordInfo);
+        await fileRepository.update({ id: file.id }, { cloud_info: info });
+    }
+    async function ensureParents() {
+        const file = await fileRepository.findOneBy({ id: file_id });
+        if (!file)
+            return;
+        const unCloudParentList = await getLocalFolderList(file.pid, []);
+        for (const item of unCloudParentList) {
+            await createCloudFolderOrFile(item);
+        }
+        return unCloudParentList;
+    }
+}

package/dist/hooks/trasform.js

@@ -1 +1,86 @@
-"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.transform=transform;const types_1=require("@soga/types"),fs_extra_1=require("fs-extra"),typeorm_1=require("typeorm");async function transform(e){const{fileRepository:i,partRepository:t,successRepository:a}=e,o=await i.findOneBy({id:e.file_id}),d={root_id:o.root_id,pid:0,space_id:o.space_id,space_name:o.space_name,uid:o.uid,inputs:o.inputs.map((e=>({filename:e.filename,filepath:e.filepath,filesize:e.filesize}))),cloud_id:o.cloud_info.id,cloud_name:o.cloud_info.name,type:o.type},p=0==o.root_id;if(!p){const e=await a.findOneBy({root_id:o.root_id,pid:0});if(e)d.pid=e.id;else{const e=await i.findOneBy({id:o.root_id}),t=await a.save(a.create({root_id:o.root_id,pid:0,space_id:e.space_id,space_name:e.space_name,uid:e.uid,inputs:e.inputs?.map((e=>({filename:e.filename,filepath:e.filepath,filesize:e.filesize}))),cloud_id:e.cloud_info.id,cloud_name:e.cloud_info.name,type:e.type}));d.pid=t.id}await a.update({id:d.pid},{updated_at:new Date})}if(await a.save(a.create(d)),await i.delete({id:e.file_id}),await t.delete({file_id:e.file_id}),await(0,fs_extra_1.remove)(o.output_root),!p){await i.findOneBy({root_id:o.root_id,type:(0,typeorm_1.Not)(types_1.RecordType.FOLDER)})?await i.increment({id:o.root_id},"completed_count",1):(await i.delete({root_id:o.root_id}),await i.delete({id:o.root_id}))}}
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.transform = transform;
+const types_1 = require("@soga/types");
+const fs_extra_1 = require("fs-extra");
+const typeorm_1 = require("typeorm");
+async function transform(params) {
+    const { fileRepository, partRepository, successRepository } = params;
+    const file = await fileRepository.findOneBy({ id: params.file_id });
+    const success_file_fields = {
+        root_id: file.root_id,
+        pid: 0,
+        space_id: file.space_id,
+        space_name: file.space_name,
+        uid: file.uid,
+        inputs: file.inputs.map((item) => ({
+            filename: item.filename,
+            filepath: item.filepath,
+            filesize: item.filesize,
+        })),
+        cloud_id: file.cloud_info.id,
+        cloud_name: file.cloud_info.name,
+        type: file.type,
+    };
+    const isRoot = file.root_id == 0;
+    if (!isRoot) {
+        const success_root = await successRepository.findOneBy({
+            root_id: file.root_id,
+            pid: 0,
+        });
+        if (!success_root) {
+            const file_root = await fileRepository.findOneBy({
+                id: file.root_id,
+            });
+            const success_root_result = await successRepository.save(successRepository.create({
+                root_id: file.root_id,
+                pid: 0,
+                space_id: file_root.space_id,
+                space_name: file_root.space_name,
+                uid: file_root.uid,
+                inputs: file_root.inputs?.map((item) => ({
+                    filename: item.filename,
+                    filepath: item.filepath,
+                    filesize: item.filesize,
+                })),
+                cloud_id: file_root.cloud_info.id,
+                cloud_name: file_root.cloud_info.name,
+                type: file_root.type,
+            }));
+            success_file_fields.pid = success_root_result.id;
+        }
+        else {
+            success_file_fields.pid = success_root.id;
+        }
+        await successRepository.update({
+            id: success_file_fields.pid,
+        }, {
+            updated_at: new Date(),
+        });
+    }
+    await successRepository.save(successRepository.create(success_file_fields));
+    await fileRepository.delete({
+        id: params.file_id,
+    });
+    await partRepository.delete({
+        file_id: params.file_id,
+    });
+    await (0, fs_extra_1.remove)(file.output_root);
+    if (!isRoot) {
+        const exist = await fileRepository.findOneBy({
+            root_id: file.root_id,
+            type: (0, typeorm_1.Not)(types_1.RecordType.FOLDER),
+        });
+        if (!exist) {
+            await fileRepository.delete({
+                root_id: file.root_id,
+            });
+            await fileRepository.delete({
+                id: file.root_id,
+            });
+        }
+        else {
+            await fileRepository.increment({ id: file.root_id }, 'completed_count', 1);
+        }
+    }
+}

package/dist/host-uploader/ali.js

@@ -1 +1,107 @@
-"use strict";var __importDefault=this&&this.__importDefault||function(t){return t&&t.__esModule?t:{default:t}};Object.defineProperty(exports,"__esModule",{value:!0}),exports.getAliUploader=exports.AliUploader=void 0;const base_1=require("./base"),types_1=require("@soga/types"),piscina_1=__importDefault(require("piscina")),node_worker_threads_1=require("node:worker_threads"),piscina=new piscina_1.default({filename:require.resolve("@soga/part-uploader")}),uploaders=new Map;class AliUploader extends base_1.BaseUploader{params;constructor(t){super(t,types_1.HostType.ALI),this.params=t}getThread(t){const e=new AbortController,{port1:r,port2:a}=new node_worker_threads_1.MessageChannel;return{file_id:t.file_id,part_id:t.id,uid:t.uid,start:async()=>{try{const{file_id:s,host_id:i,id:o,info:d}=t,l=await this.fileRepository.findOneBy({id:s}),{output_root:p}=l;a.on("message",(async t=>{await this.onPartProgress(t)})),a.on("error",(t=>{console.log("ali upload part error evt: "),console.log(t)}));const n={file_id:s,host_id:i,part_id:o,output_root:p,part:d,cloud_folder_id:l.cloud_info.hosts.ali?.file_id,sdk_domain:this.params.sdk_domain,port:r},_=await piscina.run(n,{name:"uploadAli",signal:e.signal,transferList:[r]});_&&(await this.partRepository.update(t.id,{upload_result:{ali:_},upload_status:types_1.UploadStatus.SUCCESS}),await this.onPartSuccess({file_id:s,host_type:types_1.HostType.ALI}))}catch(e){"AbortError"==e.name?this.partRepository.update(t.id,{upload_status:types_1.UploadStatus.NULL}):await this.onPartError({error:e,part_id:t.id,file_id:t.file_id})}finally{this.threads=this.threads.filter((e=>e.part_id!=t.id)),a.close(),await this.start()}},stop:async()=>{e.abort(),this.threads=this.threads.filter((e=>e.part_id!==t.id))}}}}exports.AliUploader=AliUploader;const getAliUploader=async t=>{const{uid:e,host_id:r}=t,a=`${e}_${r}`;let s=uploaders.get(a);return s||(s=new AliUploader(t),s.setThreads(3),uploaders.set(a,s)),s};exports.getAliUploader=getAliUploader;
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getAliUploader = exports.AliUploader = void 0;
+const base_1 = require("./base");
+const types_1 = require("@soga/types");
+const piscina_1 = __importDefault(require("piscina"));
+const node_worker_threads_1 = require("node:worker_threads");
+const piscina = new piscina_1.default({
+    filename: require.resolve('@soga/part-uploader'),
+});
+const uploaders = new Map();
+class AliUploader extends base_1.BaseUploader {
+    params;
+    constructor(params) {
+        super(params, types_1.HostType.ALI);
+        this.params = params;
+    }
+    getThread(part) {
+        const abortController = new AbortController();
+        const { port1, port2 } = new node_worker_threads_1.MessageChannel();
+        return {
+            file_id: part.file_id,
+            part_id: part.id,
+            uid: part.uid,
+            start: async () => {
+                try {
+                    const { file_id, host_id, id: part_id, info } = part;
+                    const file = await this.fileRepository.findOneBy({ id: file_id });
+                    const { output_root } = file;
+                    port2.on('message', async (event) => {
+                        await this.onPartProgress(event);
+                    });
+                    port2.on('error', (evt) => {
+                        console.log('ali upload part error evt: ');
+                        console.log(evt);
+                    });
+                    const params = {
+                        file_id,
+                        host_id,
+                        part_id,
+                        output_root,
+                        part: info,
+                        cloud_folder_id: file.cloud_info.hosts.ali?.file_id,
+                        sdk_domain: this.params.sdk_domain,
+                        port: port1,
+                    };
+                    const result = await piscina.run(params, {
+                        name: 'uploadAli',
+                        signal: abortController.signal,
+                        transferList: [port1],
+                    });
+                    if (result) {
+                        await this.partRepository.update(part.id, {
+                            upload_result: {
+                                ali: result,
+                            },
+                            upload_status: types_1.UploadStatus.SUCCESS,
+                        });
+                        await this.onPartSuccess({
+                            file_id,
+                            host_type: types_1.HostType.ALI,
+                        });
+                    }
+                }
+                catch (error) {
+                    if (error.name == 'AbortError') {
+                        this.partRepository.update(part.id, {
+                            upload_status: types_1.UploadStatus.NULL,
+                        });
+                    }
+                    else {
+                        await this.onPartError({
+                            error,
+                            part_id: part.id,
+                            file_id: part.file_id,
+                        });
+                    }
+                }
+                finally {
+                    this.threads = this.threads.filter((thread) => thread.part_id != part.id);
+                    port2.close();
+                    await this.start();
+                }
+            },
+            stop: async () => {
+                abortController.abort();
+                this.threads = this.threads.filter((thread) => thread.part_id !== part.id);
+            },
+        };
+    }
+}
+exports.AliUploader = AliUploader;
+const getAliUploader = async (params) => {
+    const { uid, host_id } = params;
+    const key = `${uid}_${host_id}`;
+    let uploader = uploaders.get(key);
+    if (!uploader) {
+        uploader = new AliUploader(params);
+        uploader.setThreads(3);
+        uploaders.set(key, uploader);
+    }
+    return uploader;
+};
+exports.getAliUploader = getAliUploader;

package/dist/host-uploader/baidu.js

@@ -1 +1,110 @@
-"use strict";var __importDefault=this&&this.__importDefault||function(e){return e&&e.__esModule?e:{default:e}};Object.defineProperty(exports,"__esModule",{value:!0}),exports.getBaiduUploader=exports.BaiduUploader=void 0;const base_1=require("./base"),types_1=require("@soga/types"),piscina_1=__importDefault(require("piscina")),node_worker_threads_1=require("node:worker_threads"),piscina=new piscina_1.default({filename:require.resolve("@soga/part-uploader")}),uploaders=new Map;class BaiduUploader extends base_1.BaseUploader{params;constructor(e){super(e,types_1.HostType.BAIDU),this.params=e}getThread(e){const t=new AbortController,{port1:a,port2:r}=new node_worker_threads_1.MessageChannel;return{file_id:e.file_id,part_id:e.id,uid:e.uid,start:async()=>{try{const{file_id:o,host_id:s,id:i,info:d}=e,p=await this.fileRepository.findOneBy({id:o}),{output_root:l}=p;r.on("message",(async e=>{await this.onPartProgress(e)})),r.on("error",(async e=>{console.log("baidu upload part error evt: "),console.log(e)}));const u={file_id:o,host_id:s,part_id:i,output_root:l,part:d,cloud_folder_path:p.cloud_info.hosts.baidu?.path,sdk_domain:this.params.sdk_domain,port:a},n=await piscina.run(u,{name:"uploadBaidu",signal:t.signal,transferList:[a]});n&&(await this.partRepository.update(e.id,{upload_result:{baidu:n},upload_status:types_1.UploadStatus.SUCCESS}),await this.onPartSuccess({file_id:o,host_type:types_1.HostType.BAIDU}))}catch(t){console.log("error_name: ",t.name),"AbortError"==t.name?this.partRepository.update(e.id,{upload_status:types_1.UploadStatus.NULL}):(console.log("baidu upload part error: "),console.log(t),await this.onPartError({error:t,part_id:e.id,file_id:e.file_id}))}finally{this.threads=this.threads.filter((t=>t.part_id!=e.id)),r.close(),await this.start()}},stop:async()=>{t.abort(),this.threads=this.threads.filter((t=>t.part_id!==e.id))}}}}exports.BaiduUploader=BaiduUploader;const getBaiduUploader=async e=>{const{uid:t,host_id:a}=e,r=`${t}_${a}`;let o=uploaders.get(r);return o||(o=new BaiduUploader(e),o.setThreads(6),uploaders.set(r,o)),o};exports.getBaiduUploader=getBaiduUploader;
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getBaiduUploader = exports.BaiduUploader = void 0;
+const base_1 = require("./base");
+const types_1 = require("@soga/types");
+const piscina_1 = __importDefault(require("piscina"));
+const node_worker_threads_1 = require("node:worker_threads");
+const piscina = new piscina_1.default({
+    filename: require.resolve('@soga/part-uploader'),
+});
+const uploaders = new Map();
+class BaiduUploader extends base_1.BaseUploader {
+    params;
+    constructor(params) {
+        super(params, types_1.HostType.BAIDU);
+        this.params = params;
+    }
+    getThread(part) {
+        const abortController = new AbortController();
+        const { port1, port2 } = new node_worker_threads_1.MessageChannel();
+        return {
+            file_id: part.file_id,
+            part_id: part.id,
+            uid: part.uid,
+            start: async () => {
+                try {
+                    const { file_id, host_id, id: part_id, info } = part;
+                    const file = await this.fileRepository.findOneBy({ id: file_id });
+                    const { output_root } = file;
+                    port2.on('message', async (event) => {
+                        await this.onPartProgress(event);
+                    });
+                    port2.on('error', async (evt) => {
+                        console.log('baidu upload part error evt: ');
+                        console.log(evt);
+                    });
+                    const params = {
+                        file_id,
+                        host_id,
+                        part_id,
+                        output_root,
+                        part: info,
+                        cloud_folder_path: file.cloud_info.hosts.baidu?.path,
+                        sdk_domain: this.params.sdk_domain,
+                        port: port1,
+                    };
+                    const result = await piscina.run(params, {
+                        name: 'uploadBaidu',
+                        signal: abortController.signal,
+                        transferList: [port1],
+                    });
+                    if (result) {
+                        await this.partRepository.update(part.id, {
+                            upload_result: {
+                                baidu: result,
+                            },
+                            upload_status: types_1.UploadStatus.SUCCESS,
+                        });
+                        await this.onPartSuccess({
+                            file_id,
+                            host_type: types_1.HostType.BAIDU,
+                        });
+                    }
+                }
+                catch (error) {
+                    console.log('error_name: ', error.name);
+                    if (error.name == 'AbortError') {
+                        await this.partRepository.update(part.id, {
+                            upload_status: types_1.UploadStatus.NULL,
+                        });
+                    }
+                    else {
+                        console.log('baidu upload part error: ');
+                        console.log(error);
+                        await this.onPartError({
+                            error,
+                            part_id: part.id,
+                            file_id: part.file_id,
+                        });
+                    }
+                }
+                finally {
+                    this.threads = this.threads.filter((thread) => thread.part_id != part.id);
+                    port2.close();
+                    await this.start();
+                }
+            },
+            stop: async () => {
+                abortController.abort();
+                this.threads = this.threads.filter((thread) => thread.part_id !== part.id);
+            },
+        };
+    }
+}
+exports.BaiduUploader = BaiduUploader;
+const getBaiduUploader = async (params) => {
+    const { uid, host_id } = params;
+    const key = `${uid}_${host_id}`;
+    let uploader = uploaders.get(key);
+    if (!uploader) {
+        uploader = new BaiduUploader(params);
+        uploader.setThreads(6);
+        uploaders.set(key, uploader);
+    }
+    return uploader;
+};
+exports.getBaiduUploader = getBaiduUploader;

package/dist/host-uploader/base.js

@@ -1 +1,426 @@
-"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.BaseUploader=void 0;const typeorm_1=require("typeorm"),entities_1=require("@soga/entities"),types_1=require("@soga/types"),prepare_1=require("../hooks/prepare"),utils_1=require("@soga/utils"),complete_1=require("../hooks/complete"),trasform_1=require("../hooks/trasform");let isDequeueing=!1;const runningMap=new Map,successMap=new Map;class BaseUploader{hostType;onProgress=async()=>{};onSuccess=async()=>{};onError=async()=>{};successRepository;fileRepository;partRepository;thread_count=1;maxThreads=10;threads=[];baseParams;getValidThreads(t){return Math.min(Math.max(t??0,0),this.maxThreads)}async onPartProgress(t){const{id:s,data:e,type:i}=t;if("percent"!=i)return;const{part_id:a,percent:o}=e,r=await this.partRepository.findOneBy({id:a});if(!r)return;if(r.upload_percent>=o)return;if(r.upload_status==types_1.UploadStatus.ERROR)return;if(r.upload_status==types_1.UploadStatus.SUCCESS)return;await this.partRepository.update(a,{upload_percent:o});const p=await this.partRepository.findBy({file_id:s,host_type:this.hostType});let d=0,n=0;for(const t of p)d+=t.info.size,n+=t.info.size*t.upload_percent;const u=n/d,l=await this.fileRepository.findOneBy({id:s}),h=types_1.UploadProcessStep[`upload_${this.hostType}`];l.progress[h].percent=u,l.percent=(0,utils_1.getProgressPercent)(l.progress),await this.fileRepository.update(s,{progress:l.progress,percent:l.percent}),await this.onProgress(l)}async onPartSuccess({file_id:t,host_type:s}){for(;successMap.get(t);)await new Promise((t=>{setTimeout(t,100)}));try{successMap.set(t,!0);const e=await this.fileRepository.findOneBy({id:t}),i=`${s}_upload_status`;if(e[i]==types_1.UploadStatus.ERROR)return;if(e.upload_status==types_1.UploadStatus.ERROR)return;if(await this.partRepository.findOneBy({file_id:e.id,host_type:s,upload_status:(0,typeorm_1.Not)(types_1.UploadStatus.SUCCESS)}))return;const a=await this.partRepository.findBy({file_id:e.id,host_type:s}),o={};if(s==types_1.HostType.BAIDU)for(const t of a)o[t.info.file]={...t.upload_result.baidu,md4:t.info.md4};else if(s==types_1.HostType.ALI)for(const t of a)o[t.info.file]={...t.upload_result.ali,sha1:t.info.sha1};const r=`${s}_upload_result`,p=await this.fileRepository.findOneBy({id:e.id}),d=types_1.UploadProcessStep[`upload_${this.hostType}`];p.progress[d].percent=1,p.percent=(0,utils_1.getProgressPercent)(p.progress),await this.fileRepository.update(e.id,{[i]:types_1.UploadStatus.SUCCESS,[r]:o,progress:p.progress,percent:p.percent});const n=await this.fileRepository.findOneBy({id:e.id});if(!await this.checkAllHostsUploaded(n))return;await this.onFileSuccess(n)}finally{successMap.delete(t)}}async onFileSuccess(t){await(0,complete_1.complete)({file_id:t.id,fileRepository:this.fileRepository,sdk_domain:this.baseParams.sdk_domain});const s=await this.fileRepository.findOneBy({id:t.id});await this.onSuccess(s),await(0,trasform_1.transform)({file_id:t.id,fileRepository:this.fileRepository,partRepository:this.partRepository,successRepository:this.successRepository})}async onPartError({error:t,part_id:s,file_id:e}){await this.partRepository.update(s,{upload_status:types_1.UploadStatus.ERROR});const i=`${this.hostType}_upload_status`,a=await this.fileRepository.findOneBy({id:e});a[i]!=types_1.UploadStatus.ERROR&&(await this.fileRepository.update(a.id,{[i]:types_1.UploadStatus.ERROR,upload_status:types_1.UploadStatus.ERROR}),await this.onError(t,a))}constructor(t,s){this.hostType=s,this.baseParams=t;const{dataSource:e}=t;this.fileRepository=e.getRepository(entities_1.UploadFile),this.successRepository=e.getRepository(entities_1.UploadSuccess),this.partRepository=e.getRepository(entities_1.UploadPart),this.thread_count=this.getValidThreads(t.threads??this.thread_count),t.onProgress&&(this.onProgress=t.onProgress.bind(this)),t.onSuccess&&(this.onSuccess=t.onSuccess.bind(this)),t.onError&&(this.onError=t.onError.bind(this))}async setThreads(t){const s=this.getValidThreads(t);this.thread_count=s,await this.run()}async start(){await this.run()}async stopFiles(t){const s=this.threads.filter((s=>t.includes(s.file_id)));await Promise.all(s.map((t=>t.stop()))),await this.start()}async deleteFiles(t){const s=this.threads.filter((s=>t.includes(s.file_id)));await Promise.all(s.map((t=>t.stop())))}async stopAll(){await Promise.all(this.threads.map((t=>t.stop())))}async dequeueOneFile(){for(;isDequeueing;)await new Promise((t=>{setTimeout(t,50)}));try{isDequeueing=!0;const t=`${this.hostType}_upload_status`,s=`${this.hostType}_host_id`,e=await this.fileRepository.findOne({where:{uid:this.baseParams.uid,is_paused:!1,encode_status:types_1.EncodeStatus.SUCCESS,[t]:(0,typeorm_1.IsNull)(),upload_status:(0,typeorm_1.In)([types_1.UploadStatus.NULL,types_1.UploadStatus.PROCESS])},order:{created_at:"ASC"}});if(!e)return;await this.fileRepository.update(e.id,{[t]:types_1.UploadStatus.PROCESS,upload_status:types_1.UploadStatus.PROCESS}),await(0,prepare_1.prepare)({file_id:e.id,sdk_domain:this.baseParams.sdk_domain,fileRepository:this.fileRepository});const{encoded:i}=e,{affix:a,source:o,txt:r,img:p,media:d}=i,n=[...o?.parts??[],...r?.parts??[],...p?.parts??[],...d?.parts??[]];a?.forEach((t=>{t.parts?.forEach((t=>{n.push(t)}))}));const u=e[s],l=[],{length:h}=n;for(let t=0;t<h;t++){const s=n[t];await this.partRepository.findOneBy({file_id:e.id,part_name:s.file,host_id:u})||l.push({uid:this.baseParams.uid,file_id:e.id,part_name:s.file,info:s,output_root:e.output_root,host_id:u,host_type:this.hostType})}return void await this.partRepository.save(l)}finally{isDequeueing=!1}}async checkProcessingFile(t){if(t.baidu_host_id&&t.baidu_upload_status==types_1.UploadStatus.PROCESS){await this.partRepository.findOneBy({file_id:t.id,host_type:types_1.HostType.BAIDU,upload_status:(0,typeorm_1.Not)(types_1.UploadStatus.SUCCESS)})||await this.onPartSuccess({file_id:t.id,host_type:types_1.HostType.BAIDU})}if(t.ali_host_id&&t.ali_upload_status==types_1.UploadStatus.PROCESS){await this.partRepository.findOneBy({file_id:t.id,host_type:types_1.HostType.ALI,upload_status:(0,typeorm_1.Not)(types_1.UploadStatus.SUCCESS)})||await this.onPartSuccess({file_id:t.id,host_type:types_1.HostType.ALI})}}async run(){const t=this.baseParams.uid;for(;runningMap.get(t);)await new Promise((t=>{setTimeout(t,100)}));runningMap.set(t,!0);const s=Date.now(),e=await this.fileRepository.findBy({uid:this.baseParams.uid,is_paused:!1,upload_status:types_1.UploadStatus.PROCESS});if(e.length){const t=this.threads.map((t=>t.file_id)),s=e.filter((s=>!t.includes(s.id)));for(const t of s)await this.checkProcessingFile(t)}console.log("exception time1: ",Date.now()-s);const i=Date.now(),a=await this.partRepository.findBy({uid:this.baseParams.uid,is_paused:!1,upload_status:types_1.UploadStatus.PROCESS,host_id:this.baseParams.host_id});if(a.length){const t=this.threads.map((t=>t.part_id)),s=a.filter((s=>!t.includes(s.id)));if(s.length){const t=s.map((t=>t.id));console.log("exception ids: ",t),await this.partRepository.update({id:(0,typeorm_1.In)(t),upload_status:types_1.UploadStatus.PROCESS},{upload_status:types_1.UploadStatus.NULL})}}console.log("exception time2: ",Date.now()-i);const o=this.threads.length,r=this.thread_count;if(o<r){const s=r-o,e=async()=>await this.partRepository.findOne({where:{uid:this.baseParams.uid,is_paused:!1,upload_status:types_1.UploadStatus.NULL,host_id:this.baseParams.host_id},order:{created_at:"ASC"}});for(let t=0;t<s;t++){const t=await e();if(!t)break;const s=this.getThread(t);this.threads.push(s),await this.partRepository.update(t.id,{upload_status:types_1.UploadStatus.PROCESS}),s.start()}if(this.threads.length<r){await this.dequeueOneFile();await e()?(runningMap.delete(t),await this.run()):runningMap.delete(t)}else runningMap.delete(t)}else{const s=o-r,e=this.threads.slice(0,s);for(const t of e)await t.stop();runningMap.delete(t)}}async checkAllHostsUploaded(t){return(!t.baidu_host_id||t.baidu_upload_status==types_1.UploadStatus.SUCCESS)&&(!t.ali_host_id||t.ali_upload_status==types_1.UploadStatus.SUCCESS)}}exports.BaseUploader=BaseUploader;
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BaseUploader = void 0;
+const typeorm_1 = require("typeorm");
+const entities_1 = require("@soga/entities");
+const types_1 = require("@soga/types");
+const prepare_1 = require("../hooks/prepare");
+const utils_1 = require("@soga/utils");
+const complete_1 = require("../hooks/complete");
+const trasform_1 = require("../hooks/trasform");
+let isDequeueing = false;
+const runningMap = new Map();
+const successMap = new Map();
+class BaseUploader {
+    hostType;
+    onProgress = async () => { };
+    onSuccess = async () => { };
+    onError = async () => { };
+    successRepository;
+    fileRepository;
+    partRepository;
+    thread_count = 1;
+    maxThreads = 10;
+    threads = [];
+    baseParams;
+    getValidThreads(threads) {
+        return Math.min(Math.max(threads ?? 0, 0), this.maxThreads);
+    }
+    async onPartProgress(params) {
+        const { id: file_id, data, type } = params;
+        if (type != 'percent')
+            return;
+        const { part_id, percent } = data;
+        const part = await this.partRepository.findOneBy({
+            id: part_id,
+        });
+        if (!part)
+            return;
+        if (part.upload_percent >= percent)
+            return;
+        if (part.upload_status == types_1.UploadStatus.ERROR)
+            return;
+        if (part.upload_status == types_1.UploadStatus.SUCCESS)
+            return;
+        await this.partRepository.update(part_id, {
+            upload_percent: percent,
+        });
+        const parts = await this.partRepository.findBy({
+            file_id,
+            host_type: this.hostType,
+        });
+        let total = 0;
+        let uploaded = 0;
+        for (const part of parts) {
+            total += part.info.size;
+            uploaded += part.info.size * part.upload_percent;
+        }
+        const global_percent = uploaded / total;
+        const file = await this.fileRepository.findOneBy({ id: file_id });
+        const step = types_1.UploadProcessStep[`upload_${this.hostType}`];
+        file.progress[step].percent = global_percent;
+        file.percent = (0, utils_1.getProgressPercent)(file.progress);
+        await this.fileRepository.update(file_id, {
+            progress: file.progress,
+            percent: file.percent,
+        });
+        await this.onProgress(file);
+    }
+    async onPartSuccess({ file_id, host_type, }) {
+        while (successMap.get(file_id)) {
+            await new Promise((resolve) => {
+                setTimeout(resolve, 100);
+            });
+        }
+        try {
+            successMap.set(file_id, true);
+            const file = await this.fileRepository.findOneBy({
+                id: file_id,
+            });
+            if (!file)
+                return;
+            const status = `${host_type}_upload_status`;
+            if (file[status] == types_1.UploadStatus.ERROR) {
+                return;
+            }
+            if (file.upload_status == types_1.UploadStatus.ERROR)
+                return;
+            const exist = await this.partRepository.findOneBy({
+                file_id: file.id,
+                host_type,
+                upload_status: (0, typeorm_1.Not)(types_1.UploadStatus.SUCCESS),
+            });
+            if (exist)
+                return;
+            const parts = await this.partRepository.findBy({
+                file_id: file.id,
+                host_type,
+            });
+            const result = {};
+            if (host_type == types_1.HostType.BAIDU) {
+                for (const part of parts) {
+                    result[part.info.file] = {
+                        ...part.upload_result.baidu,
+                        md4: part.info.md4,
+                    };
+                }
+            }
+            else if (host_type == types_1.HostType.ALI) {
+                for (const part of parts) {
+                    result[part.info.file] = {
+                        ...part.upload_result.ali,
+                        sha1: part.info.sha1,
+                    };
+                }
+            }
+            const result_field = `${host_type}_upload_result`;
+            const current_file = await this.fileRepository.findOneBy({
+                id: file.id,
+            });
+            const step = types_1.UploadProcessStep[`upload_${this.hostType}`];
+            current_file.progress[step].percent = 1;
+            current_file.percent = (0, utils_1.getProgressPercent)(current_file.progress);
+            await this.fileRepository.update(file.id, {
+                [status]: types_1.UploadStatus.SUCCESS,
+                [result_field]: result,
+                progress: current_file.progress,
+                percent: current_file.percent,
+            });
+            const latest_file = await this.fileRepository.findOneBy({ id: file.id });
+            const allHostUploaded = await this.checkAllHostsUploaded(latest_file);
+            if (!allHostUploaded)
+                return;
+            await this.onFileSuccess(latest_file);
+        }
+        finally {
+            successMap.delete(file_id);
+        }
+    }
+    async onFileSuccess(file) {
+        await (0, complete_1.complete)({
+            file_id: file.id,
+            fileRepository: this.fileRepository,
+            sdk_domain: this.baseParams.sdk_domain,
+        });
+        const latest_file = await this.fileRepository.findOneBy({
+            id: file.id,
+        });
+        await this.onSuccess(latest_file);
+        await (0, trasform_1.transform)({
+            file_id: file.id,
+            fileRepository: this.fileRepository,
+            partRepository: this.partRepository,
+            successRepository: this.successRepository,
+        });
+    }
+    async onPartError({ error, part_id, file_id, }) {
+        await this.partRepository.update(part_id, {
+            upload_status: types_1.UploadStatus.ERROR,
+        });
+        const status = `${this.hostType}_upload_status`;
+        const file = await this.fileRepository.findOneBy({ id: file_id });
+        if (file[status] == types_1.UploadStatus.ERROR)
+            return;
+        await this.fileRepository.update(file.id, {
+            [status]: types_1.UploadStatus.ERROR,
+            upload_status: types_1.UploadStatus.ERROR,
+        });
+        await this.onError(error, file);
+    }
+    constructor(baseParams, host_type) {
+        this.hostType = host_type;
+        this.baseParams = baseParams;
+        const { dataSource } = baseParams;
+        this.fileRepository = dataSource.getRepository(entities_1.UploadFile);
+        this.successRepository = dataSource.getRepository(entities_1.UploadSuccess);
+        this.partRepository = dataSource.getRepository(entities_1.UploadPart);
+        this.thread_count = this.getValidThreads(baseParams.threads ?? this.thread_count);
+        if (baseParams.onProgress) {
+            this.onProgress = baseParams.onProgress.bind(this);
+        }
+        if (baseParams.onSuccess) {
+            this.onSuccess = baseParams.onSuccess.bind(this);
+        }
+        if (baseParams.onError) {
+            this.onError = baseParams.onError.bind(this);
+        }
+    }
+    async setThreads(threads) {
+        const validThreads = this.getValidThreads(threads);
+        this.thread_count = validThreads;
+        await this.run();
+    }
+    async start() {
+        await this.run();
+    }
+    async stopFiles(ids) {
+        const threads = this.threads.filter((thread) => ids.includes(thread.file_id));
+        await Promise.all(threads.map((thread) => thread.stop()));
+        await this.start();
+    }
+    async deleteFiles(ids) {
+        const threads = this.threads.filter((thread) => ids.includes(thread.file_id));
+        await Promise.all(threads.map((thread) => thread.stop()));
+    }
+    async stopAll() {
+        await Promise.all(this.threads.map((thread) => thread.stop()));
+    }
+    async dequeueOneFile() {
+        while (isDequeueing) {
+            await new Promise((resolve) => {
+                setTimeout(resolve, 50);
+            });
+        }
+        try {
+            isDequeueing = true;
+            const status_field = `${this.hostType}_upload_status`;
+            const host_id_field = `${this.hostType}_host_id`;
+            const file = await this.fileRepository.findOne({
+                where: {
+                    uid: this.baseParams.uid,
+                    is_paused: false,
+                    encode_status: types_1.EncodeStatus.SUCCESS,
+                    [status_field]: (0, typeorm_1.IsNull)(),
+                    upload_status: (0, typeorm_1.In)([types_1.UploadStatus.NULL, types_1.UploadStatus.PROCESS]),
+                },
+                order: {
+                    created_at: 'ASC',
+                },
+            });
+            if (!file)
+                return;
+            await this.fileRepository.update(file.id, {
+                [status_field]: types_1.UploadStatus.PROCESS,
+                upload_status: types_1.UploadStatus.PROCESS,
+            });
+            await (0, prepare_1.prepare)({
+                file_id: file.id,
+                sdk_domain: this.baseParams.sdk_domain,
+                fileRepository: this.fileRepository,
+            });
+            const { encoded } = file;
+            const { affix: affix_data, source: source_data, txt: txt_data, img: img_data, media: media_data, } = encoded;
+            const parts = [
+                ...(source_data?.parts ?? []),
+                ...(txt_data?.parts ?? []),
+                ...(img_data?.parts ?? []),
+                ...(media_data?.parts ?? []),
+            ];
+            affix_data?.forEach((item) => {
+                item.parts?.forEach((part) => {
+                    parts.push(part);
+                });
+            });
+            const host_id = file[host_id_field];
+            const lists = [];
+            const { length } = parts;
+            for (let i = 0; i < length; i++) {
+                const item = parts[i];
+                const exist = await this.partRepository.findOneBy({
+                    file_id: file.id,
+                    part_name: item.file,
+                    host_id,
+                });
+                if (!exist) {
+                    lists.push({
+                        uid: this.baseParams.uid,
+                        file_id: file.id,
+                        part_name: item.file,
+                        info: item,
+                        output_root: file.output_root,
+                        host_id,
+                        host_type: this.hostType,
+                    });
+                }
+            }
+            await this.partRepository.save(lists);
+            return;
+        }
+        finally {
+            isDequeueing = false;
+        }
+    }
+    async checkProcessingFile(file) {
+        if (!file)
+            return;
+        if (file.baidu_host_id &&
+            file.baidu_upload_status == types_1.UploadStatus.PROCESS) {
+            const exist = await this.partRepository.findOneBy({
+                file_id: file.id,
+                host_type: types_1.HostType.BAIDU,
+                upload_status: (0, typeorm_1.Not)(types_1.UploadStatus.SUCCESS),
+            });
+            if (!exist) {
+                await this.onPartSuccess({
+                    file_id: file.id,
+                    host_type: types_1.HostType.BAIDU,
+                });
+            }
+        }
+        if (file.ali_host_id && file.ali_upload_status == types_1.UploadStatus.PROCESS) {
+            const exist = await this.partRepository.findOneBy({
+                file_id: file.id,
+                host_type: types_1.HostType.ALI,
+                upload_status: (0, typeorm_1.Not)(types_1.UploadStatus.SUCCESS),
+            });
+            if (!exist) {
+                await this.onPartSuccess({
+                    file_id: file.id,
+                    host_type: types_1.HostType.ALI,
+                });
+            }
+        }
+    }
+    async run() {
+        const uid = this.baseParams.uid;
+        while (runningMap.get(uid)) {
+            await new Promise((resolve) => {
+                setTimeout(resolve, 100);
+            });
+        }
+        runningMap.set(uid, true);
+        const time1 = Date.now();
+        const files = await this.fileRepository.findBy({
+            uid: this.baseParams.uid,
+            is_paused: false,
+            upload_status: types_1.UploadStatus.PROCESS,
+        });
+        if (files.length) {
+            const thread_file_ids = this.threads.map((thread) => thread.file_id);
+            const exception_files = files.filter((file) => {
+                return !thread_file_ids.includes(file.id);
+            });
+            for (const file of exception_files) {
+                await this.checkProcessingFile(file);
+            }
+        }
+        console.log('exception time1: ', Date.now() - time1);
+        const time2 = Date.now();
+        const parts = await this.partRepository.findBy({
+            uid: this.baseParams.uid,
+            is_paused: false,
+            upload_status: types_1.UploadStatus.PROCESS,
+            host_id: this.baseParams.host_id,
+        });
+        if (parts.length) {
+            const thread_part_ids = this.threads.map((thread) => thread.part_id);
+            const exception_parts = parts.filter((part) => {
+                return !thread_part_ids.includes(part.id);
+            });
+            if (exception_parts.length) {
+                const ids = exception_parts.map((part) => part.id);
+                console.log('exception ids: ', ids);
+                await this.partRepository.update({
+                    id: (0, typeorm_1.In)(ids),
+                    upload_status: types_1.UploadStatus.PROCESS,
+                }, {
+                    upload_status: types_1.UploadStatus.NULL,
+                });
+            }
+        }
+        console.log('exception time2: ', Date.now() - time2);
+        const real_count = this.threads.length;
+        const def_count = this.thread_count;
+        if (real_count < def_count) {
+            const less = def_count - real_count;
+            const getOnePart = async () => {
+                const part = await this.partRepository.findOne({
+                    where: {
+                        uid: this.baseParams.uid,
+                        is_paused: false,
+                        upload_status: types_1.UploadStatus.NULL,
+                        host_id: this.baseParams.host_id,
+                    },
+                    order: {
+                        created_at: 'ASC',
+                    },
+                });
+                return part;
+            };
+            for (let i = 0; i < less; i++) {
+                const part = await getOnePart();
+                if (!part)
+                    break;
+                const thread = this.getThread(part);
+                this.threads.push(thread);
+                await this.partRepository.update(part.id, {
+                    upload_status: types_1.UploadStatus.PROCESS,
+                });
+                thread.start();
+            }
+            if (this.threads.length < def_count) {
+                await this.dequeueOneFile();
+                const exist_part = await getOnePart();
+                if (exist_part) {
+                    runningMap.delete(uid);
+                    await this.run();
+                }
+                else {
+                    runningMap.delete(uid);
+                }
+            }
+            else {
+                runningMap.delete(uid);
+            }
+        }
+        else {
+            const more = real_count - def_count;
+            const threads = this.threads.slice(0, more);
+            for (const thread of threads) {
+                await thread.stop();
+            }
+            runningMap.delete(uid);
+        }
+    }
+    async checkAllHostsUploaded(file) {
+        if (file.baidu_host_id &&
+            file.baidu_upload_status != types_1.UploadStatus.SUCCESS) {
+            return false;
+        }
+        if (file.ali_host_id && file.ali_upload_status != types_1.UploadStatus.SUCCESS) {
+            return false;
+        }
+        return true;
+    }
+}
+exports.BaseUploader = BaseUploader;

package/dist/main.js

@@ -1 +1,45 @@
-"use strict";var __createBinding=this&&this.__createBinding||(Object.create?function(e,t,r,i){void 0===i&&(i=r);var a=Object.getOwnPropertyDescriptor(t,r);a&&!("get"in a?!t.__esModule:a.writable||a.configurable)||(a={enumerable:!0,get:function(){return t[r]}}),Object.defineProperty(e,i,a)}:function(e,t,r,i){void 0===i&&(i=r),e[i]=t[r]}),__exportStar=this&&this.__exportStar||function(e,t){for(var r in e)"default"===r||Object.prototype.hasOwnProperty.call(t,r)||__createBinding(t,e,r)};Object.defineProperty(exports,"__esModule",{value:!0}),exports.getUploader=void 0,__exportStar(require("./host-uploader/baidu"),exports),__exportStar(require("./host-uploader/ali"),exports);const uploader_1=require("./uploader"),instanceMap=new Map,getUploader=async e=>{for(;instanceMap.get(e.uid)?.initing;)await new Promise((e=>setTimeout(e,100)));try{if(instanceMap.get(e.uid)?.uploader)return instanceMap.get(e.uid).uploader;const t={uploader:new uploader_1.Uploader(e),initing:!0};return instanceMap.set(e.uid,t),await t.uploader.init(),t.initing=!1,t.uploader}finally{instanceMap.get(e.uid)&&(instanceMap.get(e.uid).initing=!1)}};exports.getUploader=getUploader;
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getUploader = void 0;
+__exportStar(require("./host-uploader/baidu"), exports);
+__exportStar(require("./host-uploader/ali"), exports);
+const uploader_1 = require("./uploader");
+const instanceMap = new Map();
+const getUploader = async (params) => {
+    while (instanceMap.get(params.uid)?.initing) {
+        await new Promise((resolve) => setTimeout(resolve, 100));
+    }
+    try {
+        if (instanceMap.get(params.uid)?.uploader) {
+            return instanceMap.get(params.uid).uploader;
+        }
+        const cache = {
+            uploader: new uploader_1.Uploader(params),
+            initing: true,
+        };
+        instanceMap.set(params.uid, cache);
+        await cache.uploader.init();
+        cache.initing = false;
+        return cache.uploader;
+    }
+    finally {
+        if (instanceMap.get(params.uid)) {
+            instanceMap.get(params.uid).initing = false;
+        }
+    }
+};
+exports.getUploader = getUploader;

package/dist/types/main.js

@@ -1 +1,2 @@
-"use strict";Object.defineProperty(exports,"__esModule",{value:!0});
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/uploader.js

@@ -1 +1,102 @@
-"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.Uploader=void 0;const types_1=require("@soga/types"),entities_1=require("@soga/entities"),baidu_1=require("./host-uploader/baidu"),ali_1=require("./host-uploader/ali");class Uploader{params;uid;dataSource;fileRepository;partRepository;constructor(t){this.params=t,this.uid=t.uid,this.dataSource=t.dataSource,this.fileRepository=this.dataSource.getRepository(entities_1.UploadFile),this.partRepository=this.dataSource.getRepository(entities_1.UploadPart)}processorMap=new Map;async init(){}async start(){await Promise.all([this.startAli(),this.startBaidu()])}async stopAll(){const t=this.processorMap.values();for(const s of t)await s.stopAll()}async stopFiles(t){const s=this.processorMap.values();for(const e of s)await e.stopFiles(t)}async deleteFiles(t){const s=this.processorMap.values();for(const e of s)await e.deleteFiles(t)}async startHost(t,s){if(this.processorMap.has(t)){const s=this.processorMap.get(t);return void await s.start()}const e={uid:this.uid,sdk_domain:this.params.sdk_domain,dataSource:this.dataSource,host_id:t,onProgress:this.params.onProgress,onSuccess:this.params.onSuccess,onError:this.params.onError};if(s==types_1.HostType.BAIDU){const s=await(0,baidu_1.getBaiduUploader)(e);this.processorMap.set(t,s),await s.start()}else if(s==types_1.HostType.ALI){const s=await(0,ali_1.getAliUploader)(e);this.processorMap.set(t,s),await s.start()}}async startAli(){const t=(await this.fileRepository.createQueryBuilder("file").select("DISTINCT file.ali_host_id").where("file.uid = :uid",{uid:this.uid}).getRawMany()).map((t=>t.ali_host_id)).filter((t=>!!t));if(t.length)for(const s of t)await this.startHost(s,types_1.HostType.ALI)}async startBaidu(){const t=(await this.fileRepository.createQueryBuilder("file").select("DISTINCT file.baidu_host_id").where("file.uid = :uid",{uid:this.uid}).getRawMany()).map((t=>t.baidu_host_id)).filter((t=>!!t));if(t.length)for(const s of t)await this.startHost(s,types_1.HostType.BAIDU)}}exports.Uploader=Uploader;
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Uploader = void 0;
+const types_1 = require("@soga/types");
+const entities_1 = require("@soga/entities");
+const baidu_1 = require("./host-uploader/baidu");
+const ali_1 = require("./host-uploader/ali");
+class Uploader {
+    params;
+    uid;
+    dataSource;
+    fileRepository;
+    partRepository;
+    constructor(params) {
+        this.params = params;
+        this.uid = params.uid;
+        this.dataSource = params.dataSource;
+        this.fileRepository = this.dataSource.getRepository(entities_1.UploadFile);
+        this.partRepository = this.dataSource.getRepository(entities_1.UploadPart);
+    }
+    processorMap = new Map();
+    async init() {
+    }
+    async start() {
+        await Promise.all([this.startAli(), this.startBaidu()]);
+    }
+    async stopAll() {
+        const processor_iterator = this.processorMap.values();
+        for (const processor of processor_iterator) {
+            await processor.stopAll();
+        }
+    }
+    async stopFiles(ids) {
+        const processor_iterator = this.processorMap.values();
+        for (const processor of processor_iterator) {
+            await processor.stopFiles(ids);
+        }
+    }
+    async deleteFiles(ids) {
+        const processor_iterator = this.processorMap.values();
+        for (const processor of processor_iterator) {
+            await processor.deleteFiles(ids);
+        }
+    }
+    async startHost(host_id, host_type) {
+        if (this.processorMap.has(host_id)) {
+            const processor = this.processorMap.get(host_id);
+            await processor.start();
+            return;
+        }
+        const params = {
+            uid: this.uid,
+            sdk_domain: this.params.sdk_domain,
+            dataSource: this.dataSource,
+            host_id,
+            onProgress: this.params.onProgress,
+            onSuccess: this.params.onSuccess,
+            onError: this.params.onError,
+        };
+        if (host_type == types_1.HostType.BAIDU) {
+            const baiduUploader = await (0, baidu_1.getBaiduUploader)(params);
+            this.processorMap.set(host_id, baiduUploader);
+            await baiduUploader.start();
+        }
+        else if (host_type == types_1.HostType.ALI) {
+            const aliUploader = await (0, ali_1.getAliUploader)(params);
+            this.processorMap.set(host_id, aliUploader);
+            await aliUploader.start();
+        }
+    }
+    async startAli() {
+        const ali_hosts = await this.fileRepository
+            .createQueryBuilder('file')
+            .select('DISTINCT file.ali_host_id')
+            .where('file.uid = :uid', { uid: this.uid })
+            .getRawMany();
+        const ali_host_ids = ali_hosts
+            .map((file) => file.ali_host_id)
+            .filter((id) => !!id);
+        if (ali_host_ids.length) {
+            for (const ali_host_id of ali_host_ids) {
+                await this.startHost(ali_host_id, types_1.HostType.ALI);
+            }
+        }
+    }
+    async startBaidu() {
+        const baidu_hosts = await this.fileRepository
+            .createQueryBuilder('file')
+            .select('DISTINCT file.baidu_host_id')
+            .where('file.uid = :uid', { uid: this.uid })
+            .getRawMany();
+        const baidu_host_ids = baidu_hosts
+            .map((file) => file.baidu_host_id)
+            .filter((id) => !!id);
+        if (baidu_host_ids.length) {
+            for (const baidu_host_id of baidu_host_ids) {
+                await this.startHost(baidu_host_id, types_1.HostType.BAIDU);
+            }
+        }
+    }
+}
+exports.Uploader = Uploader;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@soga/uploader",
-  "version": "0.2.45",
+  "version": "0.2.47",
   "publishConfig": {
     "access": "public"
   },
@@ -55,5 +55,5 @@
     "@soga/utils": "^0.2.33",
     "piscina": "^4.9.2"
   },
-  "gitHead": "1c1e8cb300236c3b0fbceac6e24989bac9b7209f"
+  "gitHead": "4e92f73b09132af1a2c2a3760bc5998ba2538ce9"
 }