@soga/uploader 0.3.0 → 1.0.0

package/dist/host-uploader/base.js

@@ -1,423 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.BaseUploader = void 0;
-const typeorm_1 = require("typeorm");
-const entities_1 = require("@soga/entities");
-const types_1 = require("@soga/types");
-const prepare_1 = require("../hooks/prepare");
-const utils_1 = require("@soga/utils");
-const complete_1 = require("../hooks/complete");
-const trasform_1 = require("../hooks/trasform");
-let isDequeueing = false;
-const runningMap = new Map();
-const successMap = new Map();
-class BaseUploader {
-    hostType;
-    onProgress = async () => { };
-    onSuccess = async () => { };
-    onError = async () => { };
-    successRepository;
-    fileRepository;
-    partRepository;
-    thread_count = 1;
-    maxThreads = 10;
-    threads = [];
-    baseParams;
-    getValidThreads(threads) {
-        return Math.min(Math.max(threads ?? 0, 0), this.maxThreads);
-    }
-    async onPartProgress(params) {
-        const { id: file_id, data, type } = params;
-        if (type != 'percent')
-            return;
-        const { part_id, percent } = data;
-        const part = await this.partRepository.findOneBy({
-            id: part_id,
-        });
-        if (!part)
-            return;
-        if (part.upload_percent >= percent)
-            return;
-        if (part.upload_status == types_1.UploadStatus.ERROR)
-            return;
-        if (part.upload_status == types_1.UploadStatus.SUCCESS)
-            return;
-        await this.partRepository.update(part_id, {
-            upload_percent: percent,
-        });
-        const parts = await this.partRepository.findBy({
-            file_id,
-            host_type: this.hostType,
-        });
-        let total = 0;
-        let uploaded = 0;
-        for (const part of parts) {
-            total += part.info.size;
-            uploaded += part.info.size * part.upload_percent;
-        }
-        const global_percent = uploaded / total;
-        const file = await this.fileRepository.findOneBy({ id: file_id });
-        const step = types_1.UploadProcessStep[`upload_${this.hostType}`];
-        file.progress[step].percent = global_percent;
-        file.percent = (0, utils_1.getProgressPercent)(file.progress);
-        await this.fileRepository.update(file_id, {
-            progress: file.progress,
-            percent: file.percent,
-        });
-        await this.onProgress(file);
-    }
-    async onPartSuccess({ file_id, host_type, }) {
-        while (successMap.get(file_id)) {
-            await new Promise((resolve) => {
-                setTimeout(resolve, 100);
-            });
-        }
-        try {
-            successMap.set(file_id, true);
-            const file = await this.fileRepository.findOneBy({
-                id: file_id,
-            });
-            if (!file)
-                return;
-            const status = `${host_type}_upload_status`;
-            if (file[status] == types_1.UploadStatus.ERROR) {
-                return;
-            }
-            if (file.upload_status == types_1.UploadStatus.ERROR)
-                return;
-            const exist = await this.partRepository.findOneBy({
-                file_id: file.id,
-                host_type,
-                upload_status: (0, typeorm_1.Not)(types_1.UploadStatus.SUCCESS),
-            });
-            if (exist)
-                return;
-            const parts = await this.partRepository.findBy({
-                file_id: file.id,
-                host_type,
-            });
-            const result = {};
-            if (host_type == types_1.HostType.BAIDU) {
-                for (const part of parts) {
-                    result[part.info.file] = {
-                        ...part.upload_result.baidu,
-                        md4: part.info.md4,
-                    };
-                }
-            }
-            else if (host_type == types_1.HostType.ALI) {
-                for (const part of parts) {
-                    result[part.info.file] = {
-                        ...part.upload_result.ali,
-                        sha1: part.info.sha1,
-                    };
-                }
-            }
-            const result_field = `${host_type}_upload_result`;
-            const current_file = await this.fileRepository.findOneBy({
-                id: file.id,
-            });
-            const step = types_1.UploadProcessStep[`upload_${this.hostType}`];
-            current_file.progress[step].percent = 1;
-            current_file.percent = (0, utils_1.getProgressPercent)(current_file.progress);
-            await this.fileRepository.update(file.id, {
-                [status]: types_1.UploadStatus.SUCCESS,
-                [result_field]: result,
-                progress: current_file.progress,
-                percent: current_file.percent,
-            });
-            const latest_file = await this.fileRepository.findOneBy({ id: file.id });
-            const allHostUploaded = await this.checkAllHostsUploaded(latest_file);
-            if (!allHostUploaded)
-                return;
-            await this.onFileSuccess(latest_file);
-        }
-        finally {
-            successMap.delete(file_id);
-        }
-    }
-    async onFileSuccess(file) {
-        await (0, complete_1.complete)({
-            file_id: file.id,
-            fileRepository: this.fileRepository,
-            sdk_domain: this.baseParams.sdk_domain,
-        });
-        const latest_file = await this.fileRepository.findOneBy({
-            id: file.id,
-        });
-        await this.onSuccess(latest_file);
-        await (0, trasform_1.transform)({
-            file_id: file.id,
-            fileRepository: this.fileRepository,
-            partRepository: this.partRepository,
-            successRepository: this.successRepository,
-        });
-    }
-    async onPartError({ error, part_id, file_id, }) {
-        await this.partRepository.update(part_id, {
-            upload_status: types_1.UploadStatus.ERROR,
-        });
-        const status = `${this.hostType}_upload_status`;
-        const file = await this.fileRepository.findOneBy({ id: file_id });
-        if (file[status] == types_1.UploadStatus.ERROR)
-            return;
-        await this.fileRepository.update(file.id, {
-            [status]: types_1.UploadStatus.ERROR,
-            upload_status: types_1.UploadStatus.ERROR,
-        });
-        await this.onError(error, file);
-    }
-    constructor(baseParams, host_type) {
-        this.hostType = host_type;
-        this.baseParams = baseParams;
-        const { dataSource } = baseParams;
-        this.fileRepository = dataSource.getRepository(entities_1.UploadFile);
-        this.successRepository = dataSource.getRepository(entities_1.UploadSuccess);
-        this.partRepository = dataSource.getRepository(entities_1.UploadPart);
-        this.thread_count = this.getValidThreads(baseParams.threads ?? this.thread_count);
-        if (baseParams.onProgress) {
-            this.onProgress = baseParams.onProgress.bind(this);
-        }
-        if (baseParams.onSuccess) {
-            this.onSuccess = baseParams.onSuccess.bind(this);
-        }
-        if (baseParams.onError) {
-            this.onError = baseParams.onError.bind(this);
-        }
-    }
-    async setThreads(threads) {
-        const validThreads = this.getValidThreads(threads);
-        this.thread_count = validThreads;
-        await this.run();
-    }
-    async start() {
-        await this.run();
-    }
-    async stopFiles(ids) {
-        const threads = this.threads.filter((thread) => ids.includes(thread.file_id));
-        await Promise.all(threads.map((thread) => thread.stop()));
-        await this.start();
-    }
-    async deleteFiles(ids) {
-        const threads = this.threads.filter((thread) => ids.includes(thread.file_id));
-        await Promise.all(threads.map((thread) => thread.stop()));
-    }
-    async stopAll() {
-        await Promise.all(this.threads.map((thread) => thread.stop()));
-    }
-    async dequeueOneFile() {
-        while (isDequeueing) {
-            await new Promise((resolve) => {
-                setTimeout(resolve, 50);
-            });
-        }
-        try {
-            isDequeueing = true;
-            const status_field = `${this.hostType}_upload_status`;
-            const host_id_field = `${this.hostType}_host_id`;
-            const file = await this.fileRepository.findOne({
-                where: {
-                    uid: this.baseParams.uid,
-                    is_paused: false,
-                    encode_status: types_1.EncodeStatus.SUCCESS,
-                    [status_field]: (0, typeorm_1.IsNull)(),
-                    upload_status: (0, typeorm_1.In)([types_1.UploadStatus.NULL, types_1.UploadStatus.PROCESS]),
-                },
-                order: {
-                    created_at: 'ASC',
-                },
-            });
-            if (!file)
-                return;
-            await this.fileRepository.update(file.id, {
-                [status_field]: types_1.UploadStatus.PROCESS,
-                upload_status: types_1.UploadStatus.PROCESS,
-            });
-            await (0, prepare_1.prepare)({
-                file_id: file.id,
-                sdk_domain: this.baseParams.sdk_domain,
-                fileRepository: this.fileRepository,
-            });
-            const { encoded } = file;
-            const { affix: affix_data, source: source_data, txt: txt_data, img: img_data, media: media_data, } = encoded;
-            const parts = [
-                ...(source_data?.parts ?? []),
-                ...(txt_data?.parts ?? []),
-                ...(img_data?.parts ?? []),
-                ...(media_data?.parts ?? []),
-            ];
-            affix_data?.forEach((item) => {
-                item.parts?.forEach((part) => {
-                    parts.push(part);
-                });
-            });
-            const host_id = file[host_id_field];
-            const lists = [];
-            const { length } = parts;
-            for (let i = 0; i < length; i++) {
-                const item = parts[i];
-                const exist = await this.partRepository.findOneBy({
-                    file_id: file.id,
-                    part_name: item.file,
-                    host_id,
-                });
-                if (!exist) {
-                    lists.push({
-                        uid: this.baseParams.uid,
-                        file_id: file.id,
-                        part_name: item.file,
-                        info: item,
-                        output_root: file.output_root,
-                        host_id,
-                        host_type: this.hostType,
-                    });
-                }
-            }
-            await this.partRepository.save(lists);
-            return;
-        }
-        finally {
-            isDequeueing = false;
-        }
-    }
-    async checkProcessingFile(file) {
-        if (!file)
-            return;
-        if (file.baidu_host_id &&
-            file.baidu_upload_status == types_1.UploadStatus.PROCESS) {
-            const exist = await this.partRepository.findOneBy({
-                file_id: file.id,
-                host_type: types_1.HostType.BAIDU,
-                upload_status: (0, typeorm_1.Not)(types_1.UploadStatus.SUCCESS),
-            });
-            if (!exist) {
-                await this.onPartSuccess({
-                    file_id: file.id,
-                    host_type: types_1.HostType.BAIDU,
-                });
-            }
-        }
-        if (file.ali_host_id && file.ali_upload_status == types_1.UploadStatus.PROCESS) {
-            const exist = await this.partRepository.findOneBy({
-                file_id: file.id,
-                host_type: types_1.HostType.ALI,
-                upload_status: (0, typeorm_1.Not)(types_1.UploadStatus.SUCCESS),
-            });
-            if (!exist) {
-                await this.onPartSuccess({
-                    file_id: file.id,
-                    host_type: types_1.HostType.ALI,
-                });
-            }
-        }
-    }
-    async run() {
-        const uid = this.baseParams.uid;
-        while (runningMap.get(uid)) {
-            await new Promise((resolve) => {
-                setTimeout(resolve, 100);
-            });
-        }
-        runningMap.set(uid, true);
-        const time1 = Date.now();
-        const files = await this.fileRepository.findBy({
-            uid: this.baseParams.uid,
-            is_paused: false,
-            upload_status: types_1.UploadStatus.PROCESS,
-        });
-        if (files.length) {
-            const thread_file_ids = this.threads.map((thread) => thread.file_id);
-            const exception_files = files.filter((file) => {
-                return !thread_file_ids.includes(file.id);
-            });
-            for (const file of exception_files) {
-                await this.checkProcessingFile(file);
-            }
-        }
-        const time2 = Date.now();
-        const parts = await this.partRepository.findBy({
-            uid: this.baseParams.uid,
-            is_paused: false,
-            upload_status: types_1.UploadStatus.PROCESS,
-            host_id: this.baseParams.host_id,
-        });
-        if (parts.length) {
-            const thread_part_ids = this.threads.map((thread) => thread.part_id);
-            const exception_parts = parts.filter((part) => {
-                return !thread_part_ids.includes(part.id);
-            });
-            if (exception_parts.length) {
-                const ids = exception_parts.map((part) => part.id);
-                await this.partRepository.update({
-                    id: (0, typeorm_1.In)(ids),
-                    upload_status: types_1.UploadStatus.PROCESS,
-                }, {
-                    upload_status: types_1.UploadStatus.NULL,
-                });
-            }
-        }
-        const real_count = this.threads.length;
-        const def_count = this.thread_count;
-        if (real_count < def_count) {
-            const less = def_count - real_count;
-            const getOnePart = async () => {
-                const part = await this.partRepository.findOne({
-                    where: {
-                        uid: this.baseParams.uid,
-                        is_paused: false,
-                        upload_status: types_1.UploadStatus.NULL,
-                        host_id: this.baseParams.host_id,
-                    },
-                    order: {
-                        created_at: 'ASC',
-                    },
-                });
-                return part;
-            };
-            for (let i = 0; i < less; i++) {
-                const part = await getOnePart();
-                if (!part)
-                    break;
-                const thread = this.getThread(part);
-                this.threads.push(thread);
-                await this.partRepository.update(part.id, {
-                    upload_status: types_1.UploadStatus.PROCESS,
-                });
-                thread.start();
-            }
-            if (this.threads.length < def_count) {
-                await this.dequeueOneFile();
-                const exist_part = await getOnePart();
-                if (exist_part) {
-                    runningMap.delete(uid);
-                    await this.run();
-                }
-                else {
-                    runningMap.delete(uid);
-                }
-            }
-            else {
-                runningMap.delete(uid);
-            }
-        }
-        else {
-            const more = real_count - def_count;
-            const threads = this.threads.slice(0, more);
-            for (const thread of threads) {
-                await thread.stop();
-            }
-            runningMap.delete(uid);
-        }
-    }
-    async checkAllHostsUploaded(file) {
-        if (file.baidu_host_id &&
-            file.baidu_upload_status != types_1.UploadStatus.SUCCESS) {
-            return false;
-        }
-        if (file.ali_host_id && file.ali_upload_status != types_1.UploadStatus.SUCCESS) {
-            return false;
-        }
-        return true;
-    }
-}
-exports.BaseUploader = BaseUploader;

package/dist/types/main.d.ts

@@ -1,25 +0,0 @@
-import { UploadFile } from '@soga/entities';
-import { DataSource } from 'typeorm';
-interface CommonParams {
-    uid: number;
-    sdk_domain: string;
-    dataSource: DataSource;
-    onProgress?: (file: UploadFile) => Promise<void>;
-    onSuccess?: (file: UploadFile) => Promise<void>;
-    onError?: (error: Error, file: UploadFile) => Promise<void>;
-}
-export interface UploaderParams extends CommonParams {
-    host_id: number;
-    threads?: number;
-    debug?: boolean;
-}
-export interface GetUploaderParams extends CommonParams {
-}
-export type ThreadType = {
-    file_id: number;
-    part_id: number;
-    uid: number;
-    start: () => Promise<void>;
-    stop: () => Promise<void>;
-};
-export {};

package/dist/types/main.js

@@ -1,2 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });

package/dist/uploader.d.ts

@@ -1,21 +0,0 @@
-import { DataSource, Repository } from 'typeorm';
-import { UploadFile, UploadPart } from '@soga/entities';
-import { GetUploaderParams } from './types/main';
-import { BaseUploader } from './host-uploader/base';
-export declare class Uploader {
-    protected params: GetUploaderParams;
-    protected uid: number;
-    protected dataSource: DataSource;
-    protected fileRepository: Repository<UploadFile>;
-    protected partRepository: Repository<UploadPart>;
-    constructor(params: GetUploaderParams);
-    protected processorMap: Map<number, BaseUploader>;
-    init(): Promise<void>;
-    start(): Promise<void>;
-    stopAll(): Promise<void>;
-    stopFiles(ids: number[]): Promise<void>;
-    deleteFiles(ids: number[]): Promise<void>;
-    private startHost;
-    private startAli;
-    private startBaidu;
-}

package/dist/uploader.js

@@ -1,102 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.Uploader = void 0;
-const types_1 = require("@soga/types");
-const entities_1 = require("@soga/entities");
-const baidu_1 = require("./host-uploader/baidu");
-const ali_1 = require("./host-uploader/ali");
-class Uploader {
-    params;
-    uid;
-    dataSource;
-    fileRepository;
-    partRepository;
-    constructor(params) {
-        this.params = params;
-        this.uid = params.uid;
-        this.dataSource = params.dataSource;
-        this.fileRepository = this.dataSource.getRepository(entities_1.UploadFile);
-        this.partRepository = this.dataSource.getRepository(entities_1.UploadPart);
-    }
-    processorMap = new Map();
-    async init() {
-    }
-    async start() {
-        await Promise.all([this.startAli(), this.startBaidu()]);
-    }
-    async stopAll() {
-        const processor_iterator = this.processorMap.values();
-        for (const processor of processor_iterator) {
-            await processor.stopAll();
-        }
-    }
-    async stopFiles(ids) {
-        const processor_iterator = this.processorMap.values();
-        for (const processor of processor_iterator) {
-            await processor.stopFiles(ids);
-        }
-    }
-    async deleteFiles(ids) {
-        const processor_iterator = this.processorMap.values();
-        for (const processor of processor_iterator) {
-            await processor.deleteFiles(ids);
-        }
-    }
-    async startHost(host_id, host_type) {
-        if (this.processorMap.has(host_id)) {
-            const processor = this.processorMap.get(host_id);
-            await processor.start();
-            return;
-        }
-        const params = {
-            uid: this.uid,
-            sdk_domain: this.params.sdk_domain,
-            dataSource: this.dataSource,
-            host_id,
-            onProgress: this.params.onProgress,
-            onSuccess: this.params.onSuccess,
-            onError: this.params.onError,
-        };
-        if (host_type == types_1.HostType.BAIDU) {
-            const baiduUploader = await (0, baidu_1.getBaiduUploader)(params);
-            this.processorMap.set(host_id, baiduUploader);
-            await baiduUploader.start();
-        }
-        else if (host_type == types_1.HostType.ALI) {
-            const aliUploader = await (0, ali_1.getAliUploader)(params);
-            this.processorMap.set(host_id, aliUploader);
-            await aliUploader.start();
-        }
-    }
-    async startAli() {
-        const ali_hosts = await this.fileRepository
-            .createQueryBuilder('file')
-            .select('DISTINCT file.ali_host_id')
-            .where('file.uid = :uid', { uid: this.uid })
-            .getRawMany();
-        const ali_host_ids = ali_hosts
-            .map((file) => file.ali_host_id)
-            .filter((id) => !!id);
-        if (ali_host_ids.length) {
-            for (const ali_host_id of ali_host_ids) {
-                await this.startHost(ali_host_id, types_1.HostType.ALI);
-            }
-        }
-    }
-    async startBaidu() {
-        const baidu_hosts = await this.fileRepository
-            .createQueryBuilder('file')
-            .select('DISTINCT file.baidu_host_id')
-            .where('file.uid = :uid', { uid: this.uid })
-            .getRawMany();
-        const baidu_host_ids = baidu_hosts
-            .map((file) => file.baidu_host_id)
-            .filter((id) => !!id);
-        if (baidu_host_ids.length) {
-            for (const baidu_host_id of baidu_host_ids) {
-                await this.startHost(baidu_host_id, types_1.HostType.BAIDU);
-            }
-        }
-    }
-}
-exports.Uploader = Uploader;