@soga/uploader 0.3.0 → 1.3.1

package/dist/hooks/trasform.js

@@ -1,86 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.transform = transform;
-const types_1 = require("@soga/types");
-const fs_extra_1 = require("fs-extra");
-const typeorm_1 = require("typeorm");
-async function transform(params) {
-    const { fileRepository, partRepository, successRepository } = params;
-    const file = await fileRepository.findOneBy({ id: params.file_id });
-    const success_file_fields = {
-        root_id: file.root_id,
-        pid: 0,
-        space_id: file.space_id,
-        space_name: file.space_name,
-        uid: file.uid,
-        inputs: file.inputs.map((item) => ({
-            filename: item.filename,
-            filepath: item.filepath,
-            filesize: item.filesize,
-        })),
-        cloud_id: file.cloud_info.id,
-        cloud_name: file.cloud_info.name,
-        type: file.type,
-    };
-    const isRoot = file.root_id == 0;
-    if (!isRoot) {
-        const success_root = await successRepository.findOneBy({
-            root_id: file.root_id,
-            pid: 0,
-        });
-        if (!success_root) {
-            const file_root = await fileRepository.findOneBy({
-                id: file.root_id,
-            });
-            const success_root_result = await successRepository.save(successRepository.create({
-                root_id: file.root_id,
-                pid: 0,
-                space_id: file_root.space_id,
-                space_name: file_root.space_name,
-                uid: file_root.uid,
-                inputs: file_root.inputs?.map((item) => ({
-                    filename: item.filename,
-                    filepath: item.filepath,
-                    filesize: item.filesize,
-                })),
-                cloud_id: file_root.cloud_info.id,
-                cloud_name: file_root.cloud_info.name,
-                type: file_root.type,
-            }));
-            success_file_fields.pid = success_root_result.id;
-        }
-        else {
-            success_file_fields.pid = success_root.id;
-        }
-        await successRepository.update({
-            id: success_file_fields.pid,
-        }, {
-            updated_at: new Date(),
-        });
-    }
-    await successRepository.save(successRepository.create(success_file_fields));
-    await fileRepository.delete({
-        id: params.file_id,
-    });
-    await partRepository.delete({
-        file_id: params.file_id,
-    });
-    await (0, fs_extra_1.remove)(file.output_root);
-    if (!isRoot) {
-        const exist = await fileRepository.findOneBy({
-            root_id: file.root_id,
-            type: (0, typeorm_1.Not)(types_1.RecordType.FOLDER),
-        });
-        if (!exist) {
-            await fileRepository.delete({
-                root_id: file.root_id,
-            });
-            await fileRepository.delete({
-                id: file.root_id,
-            });
-        }
-        else {
-            await fileRepository.increment({ id: file.root_id }, 'completed_count', 1);
-        }
-    }
-}

package/dist/host-uploader/ali.d.ts

@@ -1,9 +0,0 @@
-import { ThreadType, UploaderParams } from '../types/main';
-import { BaseUploader } from './base';
-import { UploadPart } from '@soga/entities';
-export declare class AliUploader extends BaseUploader {
-    private params;
-    constructor(params: UploaderParams);
-    getThread(part: UploadPart): ThreadType;
-}
-export declare const getAliUploader: (params: UploaderParams) => Promise<AliUploader>;

package/dist/host-uploader/ali.js

@@ -1,106 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.getAliUploader = exports.AliUploader = void 0;
-const base_1 = require("./base");
-const types_1 = require("@soga/types");
-const piscina_1 = __importDefault(require("piscina"));
-const node_worker_threads_1 = require("node:worker_threads");
-const piscina = new piscina_1.default({
-    filename: require.resolve('@soga/part-uploader'),
-});
-const uploaders = new Map();
-class AliUploader extends base_1.BaseUploader {
-    params;
-    constructor(params) {
-        super(params, types_1.HostType.ALI);
-        this.params = params;
-    }
-    getThread(part) {
-        const abortController = new AbortController();
-        const { port1, port2 } = new node_worker_threads_1.MessageChannel();
-        return {
-            file_id: part.file_id,
-            part_id: part.id,
-            uid: part.uid,
-            start: async () => {
-                try {
-                    const { file_id, host_id, id: part_id, info } = part;
-                    const file = await this.fileRepository.findOneBy({ id: file_id });
-                    const { output_root } = file;
-                    port2.on('message', async (event) => {
-                        await this.onPartProgress(event);
-                    });
-                    port2.on('error', (evt) => {
-                        throw new Error('Ali upload part error');
-                    });
-                    const params = {
-                        file_id,
-                        host_id,
-                        part_id,
-                        output_root,
-                        part: info,
-                        cloud_folder_id: file.cloud_info.hosts.ali?.file_id,
-                        sdk_domain: this.params.sdk_domain,
-                        port: port1,
-                    };
-                    const result = await piscina.run(params, {
-                        name: 'uploadAli',
-                        signal: abortController.signal,
-                        transferList: [port1],
-                    });
-                    if (result) {
-                        await this.partRepository.update(part.id, {
-                            upload_result: {
-                                ali: result,
-                            },
-                            upload_status: types_1.UploadStatus.SUCCESS,
-                        });
-                        await this.onPartSuccess({
-                            file_id,
-                            host_type: types_1.HostType.ALI,
-                        });
-                    }
-                }
-                catch (error) {
-                    if (error.name == 'AbortError') {
-                        this.partRepository.update(part.id, {
-                            upload_status: types_1.UploadStatus.NULL,
-                        });
-                    }
-                    else {
-                        await this.onPartError({
-                            error,
-                            part_id: part.id,
-                            file_id: part.file_id,
-                        });
-                    }
-                }
-                finally {
-                    this.threads = this.threads.filter((thread) => thread.part_id != part.id);
-                    port2.close();
-                    await this.start();
-                }
-            },
-            stop: async () => {
-                abortController.abort();
-                this.threads = this.threads.filter((thread) => thread.part_id !== part.id);
-            },
-        };
-    }
-}
-exports.AliUploader = AliUploader;
-const getAliUploader = async (params) => {
-    const { uid, host_id } = params;
-    const key = `${uid}_${host_id}`;
-    let uploader = uploaders.get(key);
-    if (!uploader) {
-        uploader = new AliUploader(params);
-        uploader.setThreads(3);
-        uploaders.set(key, uploader);
-    }
-    return uploader;
-};
-exports.getAliUploader = getAliUploader;

package/dist/host-uploader/baidu.d.ts

@@ -1,9 +0,0 @@
-import { ThreadType, UploaderParams } from '../types/main';
-import { BaseUploader } from './base';
-import { UploadPart } from '@soga/entities';
-export declare class BaiduUploader extends BaseUploader {
-    private params;
-    constructor(params: UploaderParams);
-    getThread(part: UploadPart): ThreadType;
-}
-export declare const getBaiduUploader: (params: UploaderParams) => Promise<BaiduUploader>;

package/dist/host-uploader/baidu.js

@@ -1,106 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.getBaiduUploader = exports.BaiduUploader = void 0;
-const base_1 = require("./base");
-const types_1 = require("@soga/types");
-const piscina_1 = __importDefault(require("piscina"));
-const node_worker_threads_1 = require("node:worker_threads");
-const piscina = new piscina_1.default({
-    filename: require.resolve('@soga/part-uploader'),
-});
-const uploaders = new Map();
-class BaiduUploader extends base_1.BaseUploader {
-    params;
-    constructor(params) {
-        super(params, types_1.HostType.BAIDU);
-        this.params = params;
-    }
-    getThread(part) {
-        const abortController = new AbortController();
-        const { port1, port2 } = new node_worker_threads_1.MessageChannel();
-        return {
-            file_id: part.file_id,
-            part_id: part.id,
-            uid: part.uid,
-            start: async () => {
-                try {
-                    const { file_id, host_id, id: part_id, info } = part;
-                    const file = await this.fileRepository.findOneBy({ id: file_id });
-                    const { output_root } = file;
-                    port2.on('message', async (event) => {
-                        await this.onPartProgress(event);
-                    });
-                    port2.on('error', (evt) => {
-                        throw new Error('Baidu upload part error');
-                    });
-                    const params = {
-                        file_id,
-                        host_id,
-                        part_id,
-                        output_root,
-                        part: info,
-                        cloud_folder_path: file.cloud_info.hosts.baidu?.path,
-                        sdk_domain: this.params.sdk_domain,
-                        port: port1,
-                    };
-                    const result = await piscina.run(params, {
-                        name: 'uploadBaidu',
-                        signal: abortController.signal,
-                        transferList: [port1],
-                    });
-                    if (result) {
-                        await this.partRepository.update(part.id, {
-                            upload_result: {
-                                baidu: result,
-                            },
-                            upload_status: types_1.UploadStatus.SUCCESS,
-                        });
-                        await this.onPartSuccess({
-                            file_id,
-                            host_type: types_1.HostType.BAIDU,
-                        });
-                    }
-                }
-                catch (error) {
-                    if (error.name == 'AbortError') {
-                        await this.partRepository.update(part.id, {
-                            upload_status: types_1.UploadStatus.NULL,
-                        });
-                    }
-                    else {
-                        await this.onPartError({
-                            error,
-                            part_id: part.id,
-                            file_id: part.file_id,
-                        });
-                    }
-                }
-                finally {
-                    this.threads = this.threads.filter((thread) => thread.part_id != part.id);
-                    port2.close();
-                    await this.start();
-                }
-            },
-            stop: async () => {
-                abortController.abort();
-                this.threads = this.threads.filter((thread) => thread.part_id !== part.id);
-            },
-        };
-    }
-}
-exports.BaiduUploader = BaiduUploader;
-const getBaiduUploader = async (params) => {
-    const { uid, host_id } = params;
-    const key = `${uid}_${host_id}`;
-    let uploader = uploaders.get(key);
-    if (!uploader) {
-        uploader = new BaiduUploader(params);
-        uploader.setThreads(6);
-        uploaders.set(key, uploader);
-    }
-    return uploader;
-};
-exports.getBaiduUploader = getBaiduUploader;

package/dist/host-uploader/base.d.ts

@@ -1,40 +0,0 @@
-import { Repository } from 'typeorm';
-import { UploadFile, UploadPart, UploadSuccess } from '@soga/entities';
-import { HostType, UploadWorkerPercent } from '@soga/types';
-import { UploaderParams, ThreadType } from '../types/main';
-export declare abstract class BaseUploader {
-    protected hostType: HostType;
-    protected onProgress: (file: UploadFile) => Promise<void>;
-    protected onSuccess: (file: UploadFile) => Promise<void>;
-    protected onError: (err: Error, file: UploadFile) => Promise<void>;
-    protected successRepository: Repository<UploadSuccess>;
-    protected fileRepository: Repository<UploadFile>;
-    protected partRepository: Repository<UploadPart>;
-    private thread_count;
-    private maxThreads;
-    protected threads: ThreadType[];
-    baseParams: UploaderParams;
-    abstract getThread(part: UploadPart): ThreadType;
-    protected getValidThreads(threads: number): number;
-    protected onPartProgress(params: UploadWorkerPercent): Promise<void>;
-    protected onPartSuccess({ file_id, host_type, }: {
-        file_id: number;
-        host_type: HostType;
-    }): Promise<void>;
-    protected onFileSuccess(file: UploadFile): Promise<void>;
-    protected onPartError({ error, part_id, file_id, }: {
-        error: Error;
-        part_id: number;
-        file_id: number;
-    }): Promise<void>;
-    constructor(baseParams: UploaderParams, host_type: HostType);
-    setThreads(threads: number): Promise<void>;
-    start(): Promise<void>;
-    stopFiles(ids: number[]): Promise<void>;
-    deleteFiles(ids: number[]): Promise<void>;
-    stopAll(): Promise<void>;
-    protected dequeueOneFile(): Promise<void>;
-    private checkProcessingFile;
-    private run;
-    private checkAllHostsUploaded;
-}