@nger/fk-upload 1.0.8 → 1.0.9

package/dist/templates/task.service.d.ts

@@ -0,0 +1,66 @@
+import { Injector } from '@nger/core';
+import { RabbitMqService } from '@nger/rabbitmq';
+import { Client } from '@nger/redis';
+import { Db } from '@nger/typeorm';
+import { Observable } from 'rxjs';
+import { WsService } from '@nger/ws';
+export interface DownloadTask {
+    filename: string;
+    path: string;
+    size: number;
+    url: string;
+    totalSize: number;
+}
+export interface UploadTask {
+    filename: string;
+    path: string;
+    uploadUrl: string;
+    start: number;
+    end: number;
+    total: number;
+    cookies: string;
+    aid: number;
+    folderId: number;
+    fileMd5: string;
+    index: number;
+    splitSize: number;
+    totalChunks: number;
+    complete: boolean;
+    topicId: number;
+}
+import { ScheduleService } from '@nger/schedule';
+import { W7DataSource } from '@nger/w7';
+export interface EffectTask {
+    filename: string;
+    uploadUrl: string;
+    topicId: number;
+}
+export interface ScheduleTask {
+    name: string;
+    rule: string;
+}
+export declare class TaskService {
+    private rabbit;
+    private redis;
+    private db;
+    private ws;
+    private schedule;
+    private w7;
+    private injector;
+    constructor(rabbit: RabbitMqService, redis: Client, db: Db, ws: WsService, schedule: ScheduleService, w7: W7DataSource, injector: Injector);
+    addDownloadTask(task: DownloadTask): Promise<void>;
+    addUploadTask(task: UploadTask): Promise<void>;
+    beginUploadTask(filename: string): Promise<void>;
+    addEffectTask(task: EffectTask): Promise<void>;
+    addScheduleTask(task: ScheduleTask): Promise<void>;
+    createDownLoadTask(url: string, fkLoginId: number, topicId: number): Promise<void>;
+    receiveScheduleTask(): Promise<void>;
+    receiveEffectTask(): Promise<void>;
+    receiveUploadTask(): Promise<void>;
+    receiveDownloadTask(): Promise<void>;
+    private getUploadInfourl;
+    private encodeURIComponent;
+    private getUploadUrl;
+    startUploadTask(filename: string): Promise<void>;
+    download(task: DownloadTask): Observable<DownloadTask>;
+}

package/dist/templates/task.service.js

@@ -0,0 +1,451 @@
+"use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TaskService = void 0;
+const core_1 = require("@nger/core");
+const rabbitmq_1 = require("@nger/rabbitmq");
+const redis_1 = require("@nger/redis");
+const typeorm_1 = require("@nger/typeorm");
+const fs_extra_1 = require("fs-extra");
+const entities_1 = require("../entities");
+const request_1 = require("request");
+const rxjs_1 = require("rxjs");
+const ws_1 = require("@nger/ws");
+const md5_file_1 = __importDefault(require("md5-file"));
+const typeorm_2 = require("typeorm");
+const cids_1 = __importDefault(require("cids"));
+const multihashing_async_1 = __importDefault(require("multihashing-async"));
+const form_data_1 = __importDefault(require("form-data"));
+const axios_1 = __importDefault(require("axios"));
+const schedule_1 = require("@nger/schedule");
+const w7_1 = require("@nger/w7");
+const path_1 = require("path");
+let TaskService = class TaskService {
+    rabbit;
+    redis;
+    db;
+    ws;
+    schedule;
+    w7;
+    injector;
+    constructor(rabbit, redis, db, ws, schedule, w7, injector) {
+        this.rabbit = rabbit;
+        this.redis = redis;
+        this.db = db;
+        this.ws = ws;
+        this.schedule = schedule;
+        this.w7 = w7;
+        this.injector = injector;
+    }
+    async addDownloadTask(task) {
+        const content = Buffer.from(JSON.stringify(task));
+        await this.rabbit.send('@nger/fk-upload/download-task', content);
+    }
+    async addUploadTask(task) {
+        const content = Buffer.from(JSON.stringify(task));
+        await this.rabbit.send('@nger/fk-upload/upload-task', content);
+    }
+    async beginUploadTask(filename) {
+        this.redis.set(filename, `0`);
+    }
+    async addEffectTask(task) {
+        const content = Buffer.from(JSON.stringify(task));
+        await this.rabbit.send('@nger/fk-upload/effect-task', content);
+    }
+    async addScheduleTask(task) {
+        const content = Buffer.from(JSON.stringify(task));
+        await this.rabbit.send('@nger/fk-upload/schedule-task', content);
+    }
+    async createDownLoadTask(url, fkLoginId, topicId) {
+        const hash = await (0, multihashing_async_1.default)(Buffer.from(url), 'sha1', 16);
+        const cid = new cids_1.default(1, 'raw', hash, 'base64');
+        const filename = cid.toString('base64');
+        const _url = new URL(url);
+        const filePath = _url.pathname;
+        const ext = (0, path_1.extname)(filePath);
+        const task = new entities_1.FkDownloadTaskEntity();
+        const root = this.injector.get(core_1.APP_ROOT);
+        (0, fs_extra_1.ensureDirSync)((0, path_1.join)(root, 'attachments/fk-upload'));
+        const path = (0, path_1.join)(root, 'attachments/fk-upload', encodeURIComponent(filename));
+        task.filename = filename + ext;
+        task.url = url;
+        task.loginId = fkLoginId;
+        task.totalSize = 0;
+        task.path = path;
+        task.status = 0;
+        task.size = 0;
+        task.topicId = topicId;
+        await this.db.manager.save(task);
+        return this.addDownloadTask(task);
+    }
+    async receiveScheduleTask() {
+        const obs = await this.rabbit.receive(`@nger/fk-upload/schedule-task`);
+        obs.subscribe({
+            next: ([data, complete, fail]) => {
+                const task = JSON.parse(data);
+                this.redis.get(`setting`, (err, reply) => {
+                    if (err) {
+                        fail();
+                    }
+                    else {
+                        if (reply) {
+                            const setting = JSON.parse(reply);
+                            if (task.name === setting.name) {
+                                this.schedule.scheduleJob(task.name, task.rule, async (time) => {
+                                    const accounts = await this.db.manager.find(entities_1.FkLoginEntity, {});
+                                    const w7Usernames = accounts.map(account => account.w7Username);
+                                    const users = await this.w7.manager.find(w7_1.W7UsersEntity, { where: { username: (0, typeorm_2.In)(w7Usernames) } });
+                                    const uids = users.map(user => user.uid);
+                                    const uniAccounts = await this.w7.manager.find(w7_1.W7UniAccountEntity, { where: { createUid: (0, typeorm_2.In)(uids) } });
+                                    const uniacids = uniAccounts.map(a => a.uniacid);
+                                    const tasks = await this.db.manager.find(entities_1.FkDownloadTaskEntity, { select: ['topicId'] });
+                                    const ids = tasks.map(task => task.topicId);
+                                    const topics = await this.w7.manager.find(w7_1.W7ChatTopicEntity, { where: { uniacid: (0, typeorm_2.In)(uniacids), id: (0, typeorm_2.Not)((0, typeorm_2.In)(ids)) } });
+                                    topics.map(topic => {
+                                        if (topic.thirdUrl) {
+                                            const uniacid = topic.uniacid;
+                                            const account = uniAccounts.find(a => a.uniacid === uniacid);
+                                            const user = users.find(u => u.uid === account.createUid);
+                                            const login = accounts.find(a => a.w7Username === user.username);
+                                            this.createDownLoadTask(topic.thirdUrl, login.fkLoginId, topic.id);
+                                        }
+                                    });
+                                });
+                            }
+                            else {
+                                complete();
+                            }
+                        }
+                        else {
+                            complete();
+                        }
+                    }
+                });
+            }
+        });
+    }
+    async receiveEffectTask() {
+        const obs = await this.rabbit.receive(`@nger/fk-upload/effect-task`);
+        obs.subscribe({
+            next: async ([data, complete, fail]) => {
+                const task = JSON.parse(data);
+                if (task.topicId) {
+                    this.w7.manager.update(w7_1.W7ChatTopicEntity, task.topicId, { thirdUrl: task.uploadUrl }).then(() => complete()).catch(() => fail());
+                }
+                else {
+                    complete();
+                }
+            }
+        });
+    }
+    async receiveUploadTask() {
+        const obs = await this.rabbit.receive(`@nger/fk-upload/upload-task`);
+        obs.subscribe({
+            next: async ([data, complete, fail]) => {
+                const task = JSON.parse(data);
+                this.redis.get(task.filename, async (err, reply) => {
+                    if (err)
+                        return fail();
+                    if (reply) {
+                        const taskIndex = Number(reply); // next
+                        const isComplete = task.complete && task.index === taskIndex;
+                        const isShouldRunning = isComplete || (!isComplete && task.index == taskIndex);
+                        if (isShouldRunning) {
+                            // should handler 
+                            const fileStream = (0, fs_extra_1.createReadStream)(task.path, { start: task.start, end: task.end });
+                            const formdata = new form_data_1.default();
+                            formdata.append('ctrl', fileStream);
+                            formdata.append('isFreeVer', 'false');
+                            formdata.append('aid', `${task.aid}`);
+                            formdata.append('folderId', `${task.folderId}`);
+                            formdata.append('fileName', task.filename);
+                            formdata.append(`totalSize`, `${task.total}`);
+                            formdata.append(`fileMd5`, task.fileMd5);
+                            formdata.append('index', `${task.index}`);
+                            formdata.append('chunkSize', `${task.splitSize}`);
+                            formdata.append('totalChunks', `${task.totalChunks}`);
+                            formdata.append(`complete`, `${task.complete}`);
+                            const bssInfo = {
+                                fromSite: true, siteId: 0, groupId: 0, fileSizeLimit: 1024
+                            };
+                            formdata.append('bssInfo', JSON.stringify(bssInfo));
+                            const headers = formdata.getHeaders();
+                            const uploadResult = await axios_1.default.post(task.uploadUrl, formdata, {
+                                headers: {
+                                    ...headers,
+                                    Cookie: task.cookies
+                                }
+                            }).then(res => res.data);
+                            if (uploadResult.code === 200) {
+                                const data = uploadResult.data;
+                                this.ws.send({
+                                    action: `@nger/fk-upload/uploading`,
+                                    data: {
+                                        uploadSize: task.end,
+                                        filename: task.filename,
+                                        total: task.total
+                                    }
+                                });
+                                this.redis.set(task.filename, `${task.index + 1}`);
+                                if (data) {
+                                    const downUrl = data.downUrl;
+                                    if (downUrl) {
+                                        this.redis.del(task.filename);
+                                        this.ws.send({
+                                            action: `@nger/fk-upload/uploading`,
+                                            data: {
+                                                uploadSize: task.end,
+                                                filename: task.filename,
+                                                total: task.total,
+                                                uploadUrl: `https://` + downUrl
+                                            }
+                                        });
+                                        this.addEffectTask({
+                                            filename: task.filename,
+                                            uploadUrl: `https://` + downUrl,
+                                            topicId: task.topicId
+                                        });
+                                        this.db.manager.update(entities_1.FkDownloadTaskEntity, task.filename, {
+                                            status: 2,
+                                            uploadUrl: `https://` + downUrl,
+                                            uploadSize: task.total
+                                        }).then(() => complete()).catch(() => fail());
+                                    }
+                                    else {
+                                        this.db.manager.update(entities_1.FkDownloadTaskEntity, task.filename, {
+                                            uploadSize: task.end
+                                        }).then(() => complete()).catch(() => fail());
+                                    }
+                                }
+                                else {
+                                    fail();
+                                }
+                            }
+                            else {
+                                fail();
+                            }
+                        }
+                        else {
+                            fail();
+                        }
+                    }
+                    else {
+                        return complete();
+                    }
+                });
+            }
+        });
+    }
+    async receiveDownloadTask() {
+        const obs = await this.rabbit.receive(`@nger/fk-upload/download-task`);
+        obs.subscribe({
+            next: async ([data, complete, fail]) => {
+                const task = JSON.parse(data);
+                const download = await this.db.manager.findOne(entities_1.FkDownloadTaskEntity, { where: { filename: task.filename } });
+                if (download) {
+                    let total = 0;
+                    download.url = decodeURIComponent(download.url);
+                    this.download(download).pipe((0, rxjs_1.throttleTime)(1000), (0, rxjs_1.switchMap)(async (res) => {
+                        const size = res.size;
+                        total = res.totalSize;
+                        this.ws.send({
+                            action: `@nger/fk-upload/downloading`,
+                            data: { total, size, filename: res.filename }
+                        });
+                        await this.db.manager.update(entities_1.FkDownloadTaskEntity, download.filename, { size: size, totalSize: total });
+                        return res;
+                    })).subscribe({
+                        next: (task) => { },
+                        complete: async () => {
+                            const md5 = await (0, md5_file_1.default)(task.path);
+                            this.db.manager.update(entities_1.FkDownloadTaskEntity, download.filename, { status: 1, size: total, fileMd5: md5 }).then(async () => {
+                                this.ws.send({
+                                    action: `@nger/fk-upload/downloading`,
+                                    data: { total, size: total, filename: download.filename }
+                                });
+                                this.redis.del(task.filename);
+                                await this.startUploadTask(task.filename);
+                                complete();
+                            }).catch(() => fail());
+                        },
+                        error: () => {
+                            fail();
+                        }
+                    });
+                }
+            }
+        });
+    }
+    getUploadInfourl(token) {
+        return `https://${token.url}/${token.visitType}/${token.app}/advance/info?cmd=${token.cmd}&token=${token.token}`;
+    }
+    encodeURIComponent(data) {
+        return Object.keys(data).map(key => {
+            const value = Reflect.get(data, key);
+            return `${key}=${encodeURIComponent(value)}`;
+        }).join('&');
+    }
+    getUploadUrl(token, forceDirect = true) {
+        if (forceDirect) {
+            return `https://${token.url}/${token.visitType}/${token.app}/upload?cmd=${token.cmd}&token=${token.token}`;
+        }
+        else {
+            return `https://${token.url}/${token.visitType}/${token.app}/advance?cmd=${token.cmd}&token=${token.token}`;
+        }
+    }
+    async startUploadTask(filename) {
+        const task = await this.db.manager.findOne(entities_1.FkDownloadTaskEntity, { where: { filename } });
+        if (task && task.status === 1) {
+            const loginEntity = await this.db.manager.findOne(entities_1.FkLoginEntity, { where: { fkLoginId: task.loginId } });
+            const cookies = await this.db.manager.find(entities_1.FkLoginCookieEntity, { where: { fkLoginId: task.loginId } });
+            const cookie = cookies.map(cookie => `${cookie.key}=${cookie.value}`).join(';');
+            // 00 take token
+            const url = `https://i.vip.webportal.top/`;
+            const iVipWebPortal = await axios_1.default.get(url, {
+                headers: {
+                    Cookie: cookie
+                }
+            }).then(res => res.data);
+            const items = iVipWebPortal.match(/<meta id='_TOKEN' value='(.*?)'\/>/);
+            let token = ``;
+            if (items && items.length === 2) {
+                token = items[1];
+            }
+            if (token.length > 0) {
+                // 01
+                const url = `https://smr00.vip.webportal.top/cn/api/manage/advanceUpload/genAccessKey?_v=1651232099799&_TOKEN=${token}`;
+                const accessKey = await axios_1.default.get(url).then(res => res.data);
+                if (accessKey.success) {
+                    const accessTokenString = decode(accessKey.accessKey);
+                    if (accessTokenString) {
+                        const accessToken = JSON.parse(accessTokenString);
+                        const uploadInfoUrl = this.getUploadInfourl(accessToken);
+                        const bssInfo = {
+                            fromSite: true, siteId: 0, groupId: 0, fileSizeLimit: 1024
+                        };
+                        const info = await axios_1.default.post(uploadInfoUrl, this.encodeURIComponent({
+                            fileMd5: task.fileMd5,
+                            fileSize: task.totalSize,
+                            isFreeVer: false,
+                            aid: loginEntity.aid,
+                            folderId: loginEntity.uploadGroupId,
+                            bssInfo: JSON.stringify(bssInfo),
+                            fileName: task.filename
+                        }), { headers: { cookie: cookie } }).then(res => res.data);
+                        if (info.code === 200) {
+                            const data = info.data;
+                            const { delayTime, splitSize, uploadedSize } = data;
+                            const uploadUrl = this.getUploadUrl(accessToken, false);
+                            const totalChunks = Math.ceil((task.totalSize) / splitSize);
+                            let start = 0;
+                            let index = 0;
+                            let complete = false;
+                            this.beginUploadTask(task.filename);
+                            do {
+                                const maxEndSize = start + splitSize;
+                                if (index >= totalChunks - 1 || maxEndSize >= task.totalSize) {
+                                    complete = true;
+                                }
+                                const endSize = maxEndSize > task.totalSize ? task.totalSize : maxEndSize;
+                                await this.addUploadTask({
+                                    filename: task.filename,
+                                    path: task.path,
+                                    uploadUrl: uploadUrl,
+                                    cookies: cookie,
+                                    aid: loginEntity?.aid || 0,
+                                    folderId: loginEntity?.uploadGroupId || 0,
+                                    start,
+                                    end: endSize,
+                                    total: task.totalSize,
+                                    index,
+                                    fileMd5: task.fileMd5,
+                                    totalChunks: totalChunks,
+                                    splitSize,
+                                    complete,
+                                    topicId: task.topicId
+                                });
+                                index = index + 1;
+                                start = endSize;
+                            } while (!complete);
+                        }
+                    }
+                }
+            }
+        }
+    }
+    download(task) {
+        console.log(`download ${task.url}`);
+        return new rxjs_1.Observable((sub) => {
+            // go on
+            task.size = 0;
+            let length = task.size;
+            const writeStream = (0, fs_extra_1.createWriteStream)(task.path, { start: task.size });
+            const req = (0, request_1.get)(task.url, {
+                headers: {
+                    'Content-Type': 'application/octet-stream'
+                }
+            });
+            req.on('data', (buf) => {
+                length = length + buf.length;
+                task.size = length;
+                writeStream.write(buf, (err) => {
+                    if (err)
+                        sub.error(err);
+                });
+                sub.next(task);
+            });
+            req.on('end', () => {
+                sub.complete();
+                writeStream.end();
+            });
+            req.on('response', (resp) => {
+                const headers = resp.headers;
+                task.totalSize = Number(Reflect.get(headers, 'content-length'));
+                sub.next(task);
+            });
+        });
+    }
+};
+TaskService = __decorate([
+    (0, core_1.Injectable)(),
+    __metadata("design:paramtypes", [rabbitmq_1.RabbitMqService,
+        redis_1.Client,
+        typeorm_1.Db,
+        ws_1.WsService,
+        schedule_1.ScheduleService,
+        w7_1.W7DataSource,
+        core_1.Injector])
+], TaskService);
+exports.TaskService = TaskService;
+function decode(base64Str) {
+    if (!base64Str)
+        return;
+    const t = base64Str.replace(/_/g, "/").replace(/-/g, "+");
+    const f = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=";
+    let l = 0;
+    let p = 0;
+    let h = [];
+    do {
+        const o = f.indexOf(t.charAt(l++));
+        const u = f.indexOf(t.charAt(l++));
+        const a = f.indexOf(t.charAt(l++));
+        const c = f.indexOf(t.charAt(l++));
+        const s = o << 18 | u << 12 | a << 6 | c;
+        const e = s >> 16 & 255;
+        const r = s >> 8 & 255;
+        const n = 255 & s;
+        h[p++] = 64 === a ? String.fromCharCode(e) : 64 === c ? String.fromCharCode(e, r) : String.fromCharCode(e, r, n);
+    } while (l < t.length);
+    return h.join('');
+}

package/dist/templates/upload.service.d.ts

@@ -1,20 +1,12 @@
-import { FkDownloadTaskEntity } from '../entities';
 import { Db } from '@nger/typeorm';
-import { Observable } from 'rxjs';
 import { Injector } from '@nger/core';
+import { WsService } from '@nger/ws';
+import { TaskService } from './task.service';
 export declare class UploadService {
     private db;
     private injector;
+    get ws(): WsService;
+    get task(): TaskService;
     constructor(db: Db, injector: Injector);
-    createDownLoadTask(url: string, fkLoginId: number): Promise<import("rxjs").Subscription>;
-    getUploadUrl(token: any, forceDirect?: boolean): string;
-    getUploadInfourl(token: any): string;
-    encodeURIComponent(data: any): string;
-    createUplaodTask(filename: string): Promise<{
-        msg: string;
-    } | undefined>;
-    createObservable(url: string): Promise<{
-        listen: Observable<FkDownloadTaskEntity>;
-        filename: string;
-    }>;
+    continueTask(filename: string): Promise<void>;
 }