octoparse-cli 0.1.14

package/node_modules/@octopus/engine/dist/types/browserManager.d.ts

@@ -0,0 +1,41 @@
+import IBrowser from './browser';
+export default interface IBrowserManager extends IBrowser {
+    /**
+     * 切换页面
+     * @param index 页面索引
+     */
+    switchPage: (index: number) => void;
+    /**
+     * 新增页面
+     * @param url 页面地址
+     * @param loadOptions 页面加载参数
+     */
+    addPage: (url: string, loadOptions?: any) => void;
+    /** 获取当前页面的索引 */
+    getCurrentPageIndex: () => number;
+    /** 获取总页面数 */
+    getPageCount: () => number;
+    /**
+     * 设置浏览器屏蔽广告
+     * @param disable 是否禁用
+     */
+    setDisableAD: (disable: boolean) => void;
+    /**
+     * 设置浏览器禁止加载图片
+     * @param disable 是否禁用
+     */
+    setDisableImage: (disable: boolean) => void;
+    /**
+     * 设置浏览器禁止加载样式
+     * @param disable 是否禁用
+     */
+    setDisableStylesheet: (disable: boolean) => void;
+    /** 注册页面加载中状态回调 */
+    onPageLoading: (func: (...args: any[]) => void) => void;
+    /** 注册页面加载失败状态回调 */
+    onPageFailLoad: (func: (...args: any[]) => void) => void;
+    /** 注册页面加载完成状态回调 */
+    onPageLoaded: (func: (...args: any[]) => void) => void;
+    /** 注册页面标题发生更新回调 */
+    onPageTitleUpdated: (func: (...args: any[]) => void) => void;
+}

package/node_modules/@octopus/engine/dist/types/browserManager.js

@@ -0,0 +1 @@
+'use strict';Object['defineProperty'](exports,'__esModule',{'value':!![]});

package/node_modules/@octopus/engine/dist/types/index.d.ts

@@ -0,0 +1,40 @@
+import type BaseAction from '../actions/BaseAction';
+export interface Variables {
+    /** 全局变量 */
+    globalContext: any;
+    /** 所有步骤实例 */
+    actions: Map<string, BaseAction>;
+    index?: number;
+}
+export interface ProxyMessage {
+    ip: string;
+    port?: number;
+    account?: string;
+    password?: string;
+    user?: string;
+    host?: string;
+}
+export interface TurnstileParams {
+    websiteKey: string;
+    websiteUrl: string;
+    taskId: string;
+    captchaType: number;
+    pageAction: string;
+    pageData: string;
+    userAgent: string;
+    data: string;
+    cloudflareTaskType: string;
+    lotno: string;
+}
+declare global {
+    interface Window {
+        turnstileFreezeParams?: any;
+        turnstileCallback?: (token: string) => void;
+        _turnstileInternal?: any;
+        turnstileErrorCallback?: (error: any) => void;
+        turnstile?: {
+            render: (a: string, b: any) => string;
+            [key: string]: any;
+        };
+    }
+}

package/node_modules/@octopus/engine/dist/types/index.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/node_modules/@octopus/engine/dist/types/plugin.d.ts

@@ -0,0 +1,29 @@
+import EnterCaptchaAction from '../actions/EnterCaptchaAction';
+import { CaptchaAvailableStatus, LocalCaptchaType } from '../enums';
+import IBrowserManager from './browserManager';
+export interface ExtraPlugin {
+    getImageCaptcha(image: string, url: string): Promise<string>;
+    getReCaptcha(key: string, url: string): Promise<string>;
+    getHCaptcha(key: string, url: string): Promise<string>;
+    getReCaptchaV3(key: string, action: string, url: string): Promise<string>;
+    getTurnstileToken(params: any): Promise<string>;
+    getOpenCloudflareStatus(): Promise<boolean>;
+    switchUA(browser: IBrowserManager, userAgents: Array<string>): void;
+    switchIP(proxyType: number, browser: IBrowserManager): Promise<string>;
+    /** 滑块点选专用  */
+    checkIsAvailable?(imageBase64: string): Promise<boolean>;
+    /** 获取滑块距离 */
+    getSlideCaptcha(image: string): Promise<{
+        id: string;
+        status: CaptchaAvailableStatus;
+        distance: number;
+    }>;
+    /** 获取点选坐标 */
+    getClickCaptcha(image: string, subImage: string): Promise<{
+        id: string;
+        status: CaptchaAvailableStatus;
+        clickArea: string[];
+    }>;
+    /** 开始滑块，点选打码 */
+    startCaptcha(type: LocalCaptchaType, action: EnterCaptchaAction): Promise<void>;
+}

package/node_modules/@octopus/engine/dist/types/plugin.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/node_modules/@octopus/engine/dist/utils/AsyncEmitter.d.ts

@@ -0,0 +1,15 @@
+import EventEmitter from 'node:events';
+type AsyncEventMap = Record<string | symbol, any[]>;
+type AsyncEventKey<Events extends AsyncEventMap> = Extract<keyof Events, string | symbol>;
+/**
+ * 扩展 EventEmitter 支持异步事件发射
+ */
+export declare class AsyncEmitter<Events extends AsyncEventMap = AsyncEventMap> extends EventEmitter {
+    on<E extends AsyncEventKey<Events>>(eventName: E, listener: (...args: Events[E]) => void): this;
+    emit<E extends AsyncEventKey<Events>>(eventName: E, ...args: Events[E]): boolean;
+    /**
+     * 异步发射事件，执行所有监听器并等待异步监听器完成
+     */
+    emitAsync<E extends AsyncEventKey<Events>>(eventName: E, ...args: Events[E]): Promise<boolean>;
+}
+export {};

package/node_modules/@octopus/engine/dist/utils/AsyncEmitter.js

@@ -0,0 +1 @@
+'use strict';var __importDefault=this&&this['__importDefault']||function(_0x4523a8){return _0x4523a8&&_0x4523a8['__esModule']?_0x4523a8:{'default':_0x4523a8};};Object['defineProperty'](exports,'__esModule',{'value':!![]}),exports['AsyncEmitter']=void 0x0;const node_events_1=__importDefault(require('node:events'));class AsyncEmitter extends node_events_1['default']{['on'](_0x13760e,_0x4a4ea1){return super['on'](_0x13760e,_0x4a4ea1);}['emit'](_0xe64b17,..._0x2c169a){return super['emit'](_0xe64b17,..._0x2c169a);}async['emitAsync'](_0xf8e38c,..._0x2b8580){const _0x16557a=super['listeners'](_0xf8e38c);if(_0x16557a['length']===0x0)return![];const _0x156d98=[];for(const _0x324f55 of _0x16557a){try{const _0x2dc48e=_0x324f55,_0x46d1b4=_0x2dc48e(..._0x2b8580);_0x46d1b4 instanceof Promise&&_0x156d98['push'](_0x46d1b4);}catch(_0x39dc33){console['error']('Error\x20in\x20listener\x20for\x20event\x20'+String(_0xf8e38c)+':',_0x39dc33);}}return _0x156d98['length']>0x0&&await Promise['allSettled'](_0x156d98),!![];}}exports['AsyncEmitter']=AsyncEmitter;

package/node_modules/@octopus/engine/dist/utils/DataStore.d.ts

@@ -0,0 +1,58 @@
+interface RowData {
+    /** 行号 */
+    '#': number;
+    /** 行数据的唯一标识 */
+    $id: string;
+    /** 标记是否已执行保存数据库 */
+    $isSaved: boolean;
+    /** 字段 */
+    [index: string]: string | number | boolean;
+}
+export default class DataStore {
+    /** 数据表头 */
+    columns: any[];
+    /** 数据内容 */
+    dataSource: Array<RowData>;
+    /** 当前页码 */
+    page: number;
+    /** 数据总数 */
+    total: number;
+    /** 重复数据数量 */
+    repeatCount: number;
+    /** 加速模式下此时运行的子任务编号 */
+    subTaskID: number;
+    /** 固定的表格每页行数 */
+    readonly pageSize: number;
+    /** 采集暂停时临时存储当前数据 */
+    private tempDataSource;
+    /** 所有行数据 MD5 值，用来判断是否有重复数据 */
+    rowMD5s: Set<string>;
+    /** 最后一个提取步骤的序号 */
+    lastExtractIndex: number;
+    /** 上一次循环列表的执行次数 */
+    lastLoopTimes: number;
+    constructor();
+    /** 更新表头字段数据 */
+    updateColumns(fieldNames: Array<string>): void;
+    /** 生成一条空行数据，包含序号和全部字段 */
+    createEmptyRowData(): RowData;
+    /** 提取字段数据更新行数据 */
+    extractRowData(data: object, extractIndex: number, loopTimes: number, isRowAppend: boolean): RowData;
+    /** 增加一条子任务的行数据 */
+    addSubRowData(subRowData: RowData, subMD5: string): void;
+    /**
+     * 持久化当前最后一行数据，转换 XML、计算 MD5、压缩内容、写入数据库
+     * 为什么是最后一行？因为当前行在后面的提取步骤可能还会继续写入，所以需要在出现新行时落地上一行的数据，并且在采集完成时落地最后一条数据
+     * @todo 存储为 XML 格式是老版本的做法，后面可以改为 JSON 格式，更加方便读写
+     * PS: 改不了 XML 格式，因为本地数据需要云备份到服务端，那边的数据格式时固定的，改了格式就解析不了，改不动
+     */
+    persistentRowData(): void;
+    /**
+     * 计算重复数据量，通过计算行数据的 MD5 存储起来来判断是否重复
+     * PS: 这里如果采 100 万条数据，就会存 100w 的 MD5，内存会不会占用太多？
+     */
+    calcRepeatCount(md5: string): void;
+    /** 更新数据源，当数量超过预设的页面个数则截断，并增加页码 */
+    updateDataSource(): void;
+}
+export {};

package/node_modules/@octopus/engine/dist/utils/DataStore.js

@@ -0,0 +1 @@
+'use strict';Object['defineProperty'](exports,'__esModule',{'value':!![]});const lodash_1=require('lodash'),uuid_1=require('uuid'),defaultColumns=[{'dataIndex':'#'}];class DataStore{constructor(){this['columns']=[],this['dataSource']=[],this['page']=0x1,this['total']=0x0,this['repeatCount']=0x0,this['subTaskID']=0x0,this['pageSize']=0x14,this['tempDataSource']=[],this['rowMD5s']=new Set(),this['lastExtractIndex']=-0x1,this['lastLoopTimes']=0x0;}['updateColumns'](_0x38174e){const _0x1a4d38=_0x38174e['map'](_0x14275f=>({'title':_0x14275f,'dataIndex':_0x14275f}));this['columns']=defaultColumns['concat'](_0x1a4d38);}['createEmptyRowData'](){const _0x725d94={'#':this['dataSource']['length']+0x1,'$id':(0x0,uuid_1['v4'])(),'$isSaved':![]};for(const _0x98b5ad of this['columns']){const _0x1aba58=_0x98b5ad['dataIndex'];_0x1aba58!=='#'&&(_0x725d94[_0x1aba58]='');}return this['total']++,_0x725d94;}['extractRowData'](_0x3eaf17,_0x33eaa7,_0x4ca133,_0x3d9b55){let _0x3dbe2b;if(this['lastExtractIndex']===-0x1)_0x3dbe2b=this['createEmptyRowData']();else{if(_0x33eaa7>this['lastExtractIndex']||_0x3d9b55)_0x3dbe2b=this['dataSource']['pop']();else{if(_0x33eaa7<=this['lastExtractIndex']){_0x3dbe2b=this['createEmptyRowData']();const _0x13081b=(0x0,lodash_1['findIndex'])(this['columns'],{'dataIndex':Object['keys'](_0x3eaf17)[0x0]});for(let _0x5c592d=0x1;_0x5c592d<_0x13081b;_0x5c592d++){const _0x29c3c3=this['columns'][_0x5c592d]['dataIndex'];_0x3dbe2b[_0x29c3c3]=this['dataSource'][this['dataSource']['length']-0x1][_0x29c3c3];}}}}for(const _0x47bbbf of Object['keys'](_0x3eaf17)){const {value:_0x38f363,isAppend:_0x1b6d18}=_0x3eaf17[_0x47bbbf];_0x4ca133===this['lastLoopTimes']&&_0x1b6d18?_0x3dbe2b[_0x47bbbf]+='\x0d\x0a'+_0x38f363:_0x3dbe2b[_0x47bbbf]=_0x38f363;}return this['dataSource']['push'](_0x3dbe2b),this['lastExtractIndex']=_0x33eaa7,this['lastLoopTimes']=_0x4ca133,this['updateDataSource'](),_0x3dbe2b;}['addSubRowData'](_0x1e46fe,_0x3d8904){const _0x55382c={..._0x1e46fe,'#':this['dataSource']['length']+0x1};this['dataSource']['push'](_0x55382c),this['calcRepeatCount'](_0x3d8904),this['updateDataSource'](),this['total']++;}['persistentRowData'](){const _0xbe99e6=this['dataSource'][this['dataSource']['length']-0x1];if(_0xbe99e6&&!_0xbe99e6['$isSaved']){_0xbe99e6['$isSaved']=!![];const _0xfda7d1=_0xbe99e6['$id'];}}['calcRepeatCount'](_0xae9b78){this['rowMD5s']['has'](_0xae9b78)?this['repeatCount']++:this['rowMD5s']['add'](_0xae9b78);}['updateDataSource'](){if(this['dataSource']['length']>this['pageSize']){const _0x3ac513=this['dataSource']['pop']();_0x3ac513['#']=0x1,this['dataSource']=[_0x3ac513],this['page']++;}}}exports['default']=DataStore;

package/node_modules/@octopus/engine/dist/utils/DateTimeFormatHelper.d.ts

@@ -0,0 +1,22 @@
+export default class DateTimeFormatHelper {
+    private static weekDesMap;
+    private static monthDesMap;
+    private static weekPlaceholder;
+    private static monthPlaceholder;
+    static personality: string;
+    static formatChinese(str: string, fmt?: string): string | number;
+    static format(input: Date, fmt: string): string;
+    private static dateTimeFormat;
+    /**
+     * JS Date()对象, 传入的参数在不指定具体年份的时候会设为默认的2001年,
+     * 这个方法用于调整默认年份，并返回添加年份后的字符串
+     * @param {string}
+     */
+    static adjustDefaultYear(str: string): string | number;
+    /**
+     * 时区转换
+     * @param utc 时区偏移量
+     * @param input 需要转换的时间
+     */
+    static timeZone(input: string, utc: string | number): string;
+}

package/node_modules/@octopus/engine/dist/utils/DateTimeFormatHelper.js

@@ -0,0 +1,173 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const moment_1 = __importDefault(require("moment"));
+class DateTimeFormatHelper {
+    //处理日期格式为2019年12月20日情况
+    static formatChinese(str, fmt = '-') {
+        if (str.includes('年')) {
+            const formatedStr = str.replace('年', fmt).replace('月', fmt).replace('日', '');
+            return this.adjustDefaultYear(formatedStr);
+        }
+        return this.adjustDefaultYear(str);
+    }
+    static format(input, fmt) {
+        let reFmt = fmt;
+        let weekDes = this.weekDesMap[input.getDay()];
+        let monthDes = this.monthDesMap[input.getMonth()];
+        const noon = input.getHours() > 12 ? 'PM' : 'AM';
+        switch (fmt) {
+            case 'yyyy-MM-dd':
+            case 'yyyy/MM/dd':
+                reFmt = fmt;
+                break;
+            case 'd':
+                reFmt = 'MM/dd/yyyy';
+                break;
+            case 'D':
+                reFmt = `${this.weekPlaceholder}，dd ${this.monthPlaceholder} yyyy`;
+                break;
+            case 'e':
+                reFmt = `${this.personality} ${noon}`;
+                break;
+            case 'E':
+                reFmt = 'yyyy年MM月dd日 hh時mm分';
+                break;
+            case 'f':
+                reFmt = `${this.weekPlaceholder}，dd ${this.monthPlaceholder} yyyy hh:mm`;
+                break;
+            case 'F':
+                reFmt = `${this.weekPlaceholder}，dd ${this.monthPlaceholder} yyyy hh:mm:ss`;
+                break;
+            case 'g':
+                reFmt = 'MM/yyyy/dd hh:mm';
+                break;
+            case 'G':
+                reFmt = 'MM/yyyy/dd hh:mm:ss';
+                break;
+            case 'm':
+                reFmt = `${this.monthPlaceholder} dd`;
+                break;
+            case 'r':
+                return input.toString();
+            case 's':
+                reFmt = 'yyyy-MM-ddThh:mm:ss';
+                break;
+            case 't':
+                reFmt = 'hh:mm';
+                break;
+            case 'T':
+                reFmt = 'hh:mm:ss';
+                break;
+            case 'u':
+                reFmt = 'yyyy-MM-dd hh:mm:ssZ';
+                break;
+            case 'U':
+                reFmt = `${this.weekPlaceholder}，dd ${this.monthPlaceholder} yyyy hh:mm:ss`;
+                break;
+            case 'y':
+                reFmt = `yyyy ${this.monthPlaceholder}`;
+                break;
+            case 'M':
+                reFmt = `${this.monthPlaceholder} dd`;
+                break;
+            case 'Y':
+                reFmt = `yyyy ${this.monthPlaceholder}`;
+                break;
+            default:
+                reFmt = fmt;
+                break;
+        }
+        let formatOutput = this.dateTimeFormat(input, reFmt);
+        let output1Reg = new RegExp(this.weekPlaceholder, 'g');
+        let output2Reg = new RegExp(this.monthPlaceholder, 'g');
+        let output1 = formatOutput.replace(output1Reg, weekDes);
+        let output2 = output1.replace(output2Reg, monthDes);
+        return output2;
+    }
+    static dateTimeFormat(input, fmt) {
+        let o = {
+            'M+': input.getMonth() + 1, //月份
+            'd+': input.getDate(), //日
+            'H+': input.getHours(),
+            'h+': input.getHours() == 12 ? 12 : input.getHours() % 12, //小时
+            'm+': input.getMinutes(), //分
+            's+': input.getSeconds(), //秒
+            'q+': Math.floor((input.getMonth() + 3) / 3), //季度
+            S: input.getMilliseconds(), //毫秒
+        };
+        if (/(y+)/.test(fmt))
+            fmt = fmt.replace(RegExp.$1, (input.getFullYear() + '').substr(4 - RegExp.$1.length));
+        for (var k in o)
+            if (new RegExp('(' + k + ')').test(fmt))
+                fmt = fmt.replace(RegExp.$1, RegExp.$1.length == 1 ? o[k] : ('00' + o[k]).substr(('' + o[k]).length));
+        return fmt;
+    }
+    /**
+     * JS Date()对象, 传入的参数在不指定具体年份的时候会设为默认的2001年,
+     * 这个方法用于调整默认年份，并返回添加年份后的字符串
+     * @param {string}
+     */
+    static adjustDefaultYear(str) {
+        // 判断2001年是参数自带的还是new Date()默认提供的
+        if (new Date(str) && new Date(str).getFullYear() === 2001) {
+            if (str.indexOf('2001') > 0) {
+                return str;
+            }
+            // 2001是new Date()自动设置的默认值，需要替换为当前时间
+            let year = new Date().getFullYear();
+            // 如果加上今年的年份大于现在的时间，需要变成去年的年份
+            if ((0, moment_1.default)(new Date(str).setFullYear(year)).isAfter((0, moment_1.default)())) {
+                year = year - 1;
+            }
+            // 把年份设置成当前年,并返回日期字符串
+            return new Date(str).setFullYear(year);
+            // 补充一句：new Date()里面传入的参数为时间戳时必须为number类型，否则会输出invalid Date，所以这个函数的返回值类型保持string和number
+        }
+        return str;
+    }
+    /**
+     * 时区转换
+     * @param utc 时区偏移量
+     * @param input 需要转换的时间
+     */
+    static timeZone(input, utc) {
+        try {
+            // const date = new Date(input);
+            const utcDate = (0, moment_1.default)(input).add(Number(utc), 'hour').format('YYYY-MM-DD HH:mm:ss');
+            return moment_1.default.utc(new Date(utcDate)).format('YYYY-MM-DD HH:mm:ss');
+        }
+        catch (error) {
+            return 'Invalid date';
+        }
+    }
+}
+DateTimeFormatHelper.weekDesMap = {
+    0: 'Sunday',
+    1: 'Monday',
+    2: 'Tuesday',
+    3: 'Wednesday',
+    4: 'Thursday',
+    5: 'Friday',
+    6: 'Saturday',
+};
+DateTimeFormatHelper.monthDesMap = {
+    0: 'January',
+    1: 'February',
+    2: 'March',
+    3: 'April',
+    4: 'May',
+    5: 'June',
+    6: 'July',
+    7: 'August',
+    8: 'September',
+    9: 'October',
+    10: 'November',
+    11: 'December',
+};
+DateTimeFormatHelper.weekPlaceholder = '@###@';
+DateTimeFormatHelper.monthPlaceholder = '@#@';
+DateTimeFormatHelper.personality = 'yyyy/MM/dd hh:mm:ss';
+exports.default = DateTimeFormatHelper;

package/node_modules/@octopus/engine/dist/utils/FileDownloader.d.ts

@@ -0,0 +1,108 @@
+import { Page } from 'puppeteer-core';
+import { ExtractItem } from '../models/extractItem';
+export interface DownloadOptions {
+    /** 下载目录路径 (对应旧版 threeFolderPath: 根目录/任务名/时间/) */
+    downloadPath: string;
+    /** 字段名 */
+    fieldName?: string;
+    /** 日志回调函数 */
+    logCallback?: (message: string) => void;
+}
+/**
+ * 文件下载工具类
+ * 通过 Puppeteer Response 拦截实现文件下载
+ */
+export declare class FileDownloader {
+    private page;
+    private options;
+    /** MIME 类型到文件扩展名的映射表 */
+    private static readonly MIME_TO_EXTENSION;
+    constructor(page: Page, options: DownloadOptions);
+    /**
+     * 下载文件
+     * @param url 下载链接
+     * @param extractItem 提取项配置
+     * @returns 下载后的文件路径和文件大小
+     */
+    downloadFile(url: string, extractItem: ExtractItem): Promise<{
+        filePath: string;
+        fileSize: number;
+    }>;
+    /**
+     * 使用 Response 拦截下载单个文件
+     * @param url 下载链接
+     * @param extractItem 提取项配置
+     * @returns 下载后的文件路径和文件大小
+     */
+    private downloadSingleFile;
+    /**
+     * 触发下载请求
+     * @param url 下载链接
+     */
+    private triggerDownloadRequest;
+    /**
+     * 判断两个 URL 是否指向同一资源
+     * @param url1 URL 1
+     * @param url2 URL 2
+     * @returns 是否相同
+     */
+    private isSameResource;
+    /**
+     * 从响应中提取文件名和扩展名
+     * @param response Puppeteer Response 对象
+     * @param url 原始 URL
+     * @returns { name: 文件名, ext: 扩展名 }
+     */
+    private extractFileNameParts;
+    /**
+     * 根据 Content-Type 获取文件扩展名
+     * @param contentType MIME 类型
+     * @returns 文件扩展名
+     */
+    private getExtensionByContentType;
+    /**
+     * 日志输出
+     */
+    private log;
+    /**
+     * 格式化文件名，去除非法字符
+     * @param name 原始名称
+     * @returns 格式化后的名称
+     */
+    private formatName;
+    /**
+     * 格式化扩展名，确保以点开头
+     * @param ext 扩展名
+     * @returns 格式化后的扩展名
+     */
+    private formatExtension;
+    /**
+     * 构建下载目录路径
+     * 对应旧版逻辑: threeFolderPath/fieldName
+     * @returns 下载目录路径
+     */
+    private buildThreeFolderPath;
+    /**
+     * 计算 Buffer 的 MD5 值
+     * @param buffer 文件内容
+     * @returns MD5 哈希值
+     */
+    private calculateMD5;
+    /**
+     * 根据配置生成文件名
+     * @param originalName 原始文件名
+     * @param extension 文件扩展名
+     * @param buffer 文件内容
+     * @param extractItem 提取项配置
+     * @returns 格式化后的文件名（含扩展名）
+     */
+    private generateFileName;
+    /**
+     * 处理重名文件
+     * @param filePath 目标文件路径
+     * @param buffer 文件内容
+     * @param extractItem 提取项配置
+     * @returns 最终保存的文件路径
+     */
+    private handleDuplicateFile;
+}

package/node_modules/@octopus/engine/dist/utils/FileDownloader.js

@@ -0,0 +1 @@
+'use strict';var __createBinding=this&&this['__createBinding']||(Object['create']?function(_0xdc1bc6,_0x50eeea,_0x44f73e,_0x35dbe6){if(_0x35dbe6===undefined)_0x35dbe6=_0x44f73e;var _0x5389b6=Object['getOwnPropertyDescriptor'](_0x50eeea,_0x44f73e);(!_0x5389b6||('get'in _0x5389b6?!_0x50eeea['__esModule']:_0x5389b6['writable']||_0x5389b6['configurable']))&&(_0x5389b6={'enumerable':!![],'get':function(){return _0x50eeea[_0x44f73e];}}),Object['defineProperty'](_0xdc1bc6,_0x35dbe6,_0x5389b6);}:function(_0x22e204,_0x1d627e,_0x2d0508,_0x38c8f5){if(_0x38c8f5===undefined)_0x38c8f5=_0x2d0508;_0x22e204[_0x38c8f5]=_0x1d627e[_0x2d0508];}),__setModuleDefault=this&&this['__setModuleDefault']||(Object['create']?function(_0x7c1f4f,_0xc7b1a3){Object['defineProperty'](_0x7c1f4f,'default',{'enumerable':!![],'value':_0xc7b1a3});}:function(_0x512225,_0x59697f){_0x512225['default']=_0x59697f;}),__importStar=this&&this['__importStar']||(function(){var _0x24e10f=function(_0x59a622){return _0x24e10f=Object['getOwnPropertyNames']||function(_0x346869){var _0x16bb7f=[];for(var _0x2e8d94 in _0x346869)if(Object['prototype']['hasOwnProperty']['call'](_0x346869,_0x2e8d94))_0x16bb7f[_0x16bb7f['length']]=_0x2e8d94;return _0x16bb7f;},_0x24e10f(_0x59a622);};return function(_0x1a39cf){if(_0x1a39cf&&_0x1a39cf['__esModule'])return _0x1a39cf;var _0x51b6a3={};if(_0x1a39cf!=null){for(var _0x2ff7d4=_0x24e10f(_0x1a39cf),_0x50794e=0x0;_0x50794e<_0x2ff7d4['length'];_0x50794e++)if(_0x2ff7d4[_0x50794e]!=='default')__createBinding(_0x51b6a3,_0x1a39cf,_0x2ff7d4[_0x50794e]);}return __setModuleDefault(_0x51b6a3,_0x1a39cf),_0x51b6a3;};}()),__importDefault=this&&this['__importDefault']||function(_0xfabaa8){return _0xfabaa8&&_0xfabaa8['__esModule']?_0xfabaa8:{'default':_0xfabaa8};};Object['defineProperty'](exports,'__esModule',{'value':!![]}),exports['FileDownloader']=void 0x0;const path=__importStar(require('path')),fs=__importStar(require('fs')),crypto=__importStar(require('crypto')),utils_1=require('../utils'),moment_1=__importDefault(require('moment')),enums_1=require('../enums');class FileDownloader{constructor(_0x3162b5,_0x2fdb6f){this['page']=_0x3162b5,this['options']=_0x2fdb6f;}async['downloadFile'](_0x1eac04,_0x8912ed){if((0x0,utils_1['isNullOrEmpty'])(_0x1eac04))throw new Error('下载链接为空');const _0x482afd=_0x8912ed['DownloadFileConfig'];console['log']('|config',_0x482afd);let _0x3235bd=[_0x1eac04];if(_0x482afd['mutipleUrlConfig']['isIncludeMutipleUrl']){const _0x413d0d=_0x482afd['mutipleUrlConfig']['urlSplitType'],_0x21a03f=['\x0a',',','\x20'],_0x38fb67=_0x21a03f[_0x413d0d]||'\x0a',_0x457064=_0x1eac04['includes'](_0x38fb67);if(_0x457064)_0x3235bd=_0x1eac04['split'](_0x38fb67)['map'](_0x6af2d6=>_0x6af2d6['trim']())['filter'](_0x2fe2b3=>!(0x0,utils_1['isNullOrEmpty'])(_0x2fe2b3));else throw new Error('URL拆分失败：未找到分隔符\x20\x22'+_0x38fb67+'\x22');}const _0x26908c=[];let _0x517fa3=0x0;for(const _0x979f69 of _0x3235bd){if(_0x482afd['excludeUrls']&&_0x482afd['excludeUrls']['includes'](_0x979f69)){this['log']('跳过下载排除的链接:\x20'+_0x979f69);continue;}const _0x5318be=await this['downloadSingleFile'](_0x979f69,_0x8912ed);_0x26908c['push'](_0x5318be['filePath']),_0x517fa3+=_0x5318be['fileSize'];}return{'filePath':_0x26908c['join'](';\x20'),'fileSize':_0x517fa3};}async['downloadSingleFile'](_0x4b44ff,_0x36157e){const _0x101441=_0x4b44ff['replace'](/&amp;/g,'&')['replace'](/&lt;/g,'<')['replace'](/&gt;/g,'>')['replace'](/&quot;/g,'\x22');let _0x3a92e2='',_0x3374f4=0x0,_0x2798a9=![],_0x8b14af=null;const _0x1deb96=async _0x18aae3=>{try{const _0x4d3a67=_0x18aae3['url'](),_0x5df09e=_0x18aae3['status']();if(!this['isSameResource'](_0x4d3a67,_0x101441))return;if(_0x5df09e>=0x12c&&_0x5df09e<0x190)return;const _0x5eafc6=await _0x18aae3['buffer']();_0x3374f4=_0x5eafc6['length'];const {name:_0x452367,ext:_0x212323}=this['extractFileNameParts'](_0x18aae3,_0x101441),_0x284b31=this['buildThreeFolderPath'](),_0x11c5e9=this['generateFileName'](_0x452367,_0x212323,_0x5eafc6,_0x36157e),_0x1f4ed6=path['join'](_0x284b31,_0x11c5e9);_0x3a92e2=await this['handleDuplicateFile'](_0x1f4ed6,_0x5eafc6,_0x36157e),_0x2798a9=!![];}catch(_0x350068){_0x8b14af=_0x350068 instanceof Error?_0x350068:new Error(String(_0x350068)),_0x2798a9=!![];}};this['page']['on']('response',_0x1deb96);try{await this['triggerDownloadRequest'](_0x101441);const _0x26497b=Date['now'](),_0x4d7996=0xea60;while(!_0x2798a9&&Date['now']()-_0x26497b<_0x4d7996){await new Promise(_0x4b4372=>setTimeout(_0x4b4372,0x64));}if(!_0x2798a9)throw new Error('下载超时：未收到响应');if(_0x8b14af)throw _0x8b14af;if(!_0x3a92e2)throw new Error('下载失败：未能保存文件');return{'filePath':_0x3a92e2,'fileSize':_0x3374f4};}finally{this['page']['off']('response',_0x1deb96);}}async['triggerDownloadRequest'](_0x2a08b0){try{await this['page']['evaluate'](_0x3a63ef=>{return fetch(_0x3a63ef,{'method':'GET','credentials':'include'});},_0x2a08b0);}catch(_0x28bf99){console['log']('error',_0x28bf99),await this['page']['evaluate'](_0x145f62=>{return new Promise(_0x262a0f=>{console['log']('使用图片元素触发下载请求');const _0x443824=document['createElement']('img');_0x443824['style']['display']='none',_0x443824['src']=_0x145f62,document['body']['appendChild'](_0x443824),setTimeout(()=>_0x262a0f(undefined),0x64);});},_0x2a08b0);}}['isSameResource'](_0x57a9cb,_0x4a20f8){try{const _0x1e7325=new URL(_0x57a9cb),_0x5725ce=new URL(_0x4a20f8);if(_0x1e7325['origin']!==_0x5725ce['origin']||_0x1e7325['pathname']!==_0x5725ce['pathname'])return![];return!![];}catch(_0x402017){return _0x57a9cb===_0x4a20f8;}}['extractFileNameParts'](_0x2755fe,_0x3f5305){var _0x37fb31;const _0xdfac5c=_0x2755fe['headers'](),_0x376ac0=_0xdfac5c['content-disposition'];if(_0x376ac0){const _0x1e2944=[_0x376ac0['match'](/filename\*=UTF-8''([^;]+)/i),_0x376ac0['match'](/filename="([^"]+)"/i),_0x376ac0['match'](/filename=([^;]+)/i)];for(const _0x165a20 of _0x1e2944){if(_0x165a20&&_0x165a20[0x1])try{const _0x2b8176=decodeURIComponent(_0x165a20[0x1]['trim']());if(_0x2b8176){const _0x5cd276=path['parse'](_0x2b8176);return{'name':_0x5cd276['name'],'ext':_0x5cd276['ext']};}}catch(_0x234b57){}}}try{const _0x3e6c25=new URL(_0x3f5305),_0x13e142=_0x3e6c25['pathname'],_0x3190dd=path['basename'](_0x13e142);if(_0x3190dd&&_0x3190dd['includes']('.')&&_0x3190dd['length']>0x1){const _0x568667=decodeURIComponent(_0x3190dd),_0xb1881=path['parse'](_0x568667);return{'name':_0xb1881['name'],'ext':_0xb1881['ext']};}}catch(_0x251440){}const _0x4c12ab=(_0x37fb31=_0xdfac5c['content-type'])===null||_0x37fb31===void 0x0?void 0x0:_0x37fb31['split'](';')[0x0]['trim'](),_0x4ae646=this['getExtensionByContentType'](_0x4c12ab);return{'name':'download_'+Date['now'](),'ext':'.'+_0x4ae646};}['getExtensionByContentType'](_0x31a22c){return FileDownloader['MIME_TO_EXTENSION'][_0x31a22c]||'bin';}['log'](_0x1572ab){this['options']['logCallback']&&this['options']['logCallback'](_0x1572ab);}['formatName'](_0x5aa5d3){return _0x5aa5d3['replace'](/[\\/:*?"<>|]/g,'_');}['formatExtension'](_0xd89576){return _0xd89576['startsWith']('.')?_0xd89576:'.'+_0xd89576;}['buildThreeFolderPath'](){const _0x3dca25=[this['options']['downloadPath']];this['options']['fieldName']&&_0x3dca25['push'](this['formatName'](this['options']['fieldName']));const _0x5ab516=path['join'](..._0x3dca25);return!fs['existsSync'](_0x5ab516)&&fs['mkdirSync'](_0x5ab516,{'recursive':!![]}),_0x5ab516;}['calculateMD5'](_0xc402af){return crypto['createHash']('md5')['update'](_0xc402af)['digest']('hex');}['generateFileName'](_0x83bd3,_0x557535,_0xa2e18e,_0x2c7299){const _0x297381=_0x2c7299['DownloadFileConfig'];let _0x276513=_0x83bd3;switch(_0x297381['downloadFileRenameType']){case enums_1['DownloadFileRenameType']['SourceFileMD5']:_0x276513=this['calculateMD5'](_0xa2e18e);break;case enums_1['DownloadFileRenameType']['AcquisitionTime']:_0x276513=(0x0,moment_1['default'])()['format']('HHmmssSSS');break;case enums_1['DownloadFileRenameType']['FieldValue']:_0x297381['field']&&(_0x276513=_0x297381['field']);break;case enums_1['DownloadFileRenameType']['SourceFileName']:default:_0x276513=_0x83bd3;break;}return this['formatName'](_0x276513)+this['formatExtension'](_0x557535);}async['handleDuplicateFile'](_0xe3c4fc,_0x122d46,_0xaeba0a){const _0x295556=_0xaeba0a['DownloadFileConfig'];if(!fs['existsSync'](_0xe3c4fc))return await fs['promises']['writeFile'](_0xe3c4fc,_0x122d46),_0xe3c4fc;switch(_0x295556['repeatValue']){case enums_1['DownloadFileNameRepeatType']['Cover']:await fs['promises']['writeFile'](_0xe3c4fc,_0x122d46),this['log']('文件已覆盖:\x20'+_0xe3c4fc);return _0xe3c4fc;case enums_1['DownloadFileNameRepeatType']['SkipNonDownload']:this['log']('文件已存在，跳过下载:\x20'+_0xe3c4fc);throw new Error('文件已存在，跳过下载:\x20'+_0xe3c4fc);case enums_1['DownloadFileNameRepeatType']['AutoRename']:default:const _0x3c9304=path['parse'](_0xe3c4fc);let _0x8fbb15=0x2,_0x307210=path['join'](_0x3c9304['dir'],_0x3c9304['name']+'\x20('+_0x8fbb15+')'+_0x3c9304['ext']);while(fs['existsSync'](_0x307210)){_0x8fbb15++,_0x307210=path['join'](_0x3c9304['dir'],_0x3c9304['name']+'\x20('+_0x8fbb15+')'+_0x3c9304['ext']);}await fs['promises']['writeFile'](_0x307210,_0x122d46),this['log']('文件自动重命名:\x20'+_0x307210);return _0x307210;}}}exports['FileDownloader']=FileDownloader,FileDownloader['MIME_TO_EXTENSION']={'image/jpeg':'jpg','image/png':'png','image/gif':'gif','image/webp':'webp','image/svg+xml':'svg','image/bmp':'bmp','image/tiff':'tiff','application/pdf':'pdf','application/msword':'doc','application/vnd.openxmlformats-officedocument.wordprocessingml.document':'docx','application/vnd.ms-excel':'xls','application/vnd.openxmlformats-officedocument.spreadsheetml.sheet':'xlsx','application/vnd.ms-powerpoint':'ppt','application/vnd.openxmlformats-officedocument.presentationml.presentation':'pptx','text/plain':'txt','text/csv':'csv','application/zip':'zip','application/x-rar-compressed':'rar','application/x-7z-compressed':'7z','application/x-tar':'tar','application/gzip':'gz','video/mp4':'mp4','video/mpeg':'mpeg','video/webm':'webm','video/quicktime':'mov','video/x-msvideo':'avi','audio/mpeg':'mp3','audio/wav':'wav','audio/ogg':'ogg','audio/webm':'weba','application/json':'json','application/xml':'xml','text/html':'html'};

package/node_modules/@octopus/engine/dist/utils/HttpRequester.d.ts

@@ -0,0 +1,43 @@
+import { EventEmitter } from 'events';
+import { CustomizeCookie } from '../models/customizeCookie';
+export declare class HttpRequester extends EventEmitter {
+    static xForwardFor: string;
+    static userAgent: string;
+    static cookies?: CustomizeCookie[];
+    isRequesting: boolean;
+    onRequested?: (response: any) => void;
+    onRequesting?: (args: any) => void;
+    onRequestError?: (error: any) => void;
+    onRequestStopped?: () => void;
+    constructor();
+    startRequest(url: string, method: string, headers: Map<string, string>, bodyContent: string, timeout: number, requestCallback?: (response?: unknown) => void): Promise<void>;
+    /**
+     * 判断网页地址是否包含中文字符
+     * @param str
+     */
+    static checkChinese(str: string): boolean;
+}
+export declare class RequestingArgs {
+    url: string;
+    method: string;
+    requestHeaders: Map<string, string>;
+    bodyContent: string;
+    cancel: boolean;
+    constructor(url: string, method: string, requestHeaders: Map<string, string>, bodyContent: string);
+}
+export declare class RequestedArgs {
+    url: string;
+    statusCode: number;
+    encoding: string;
+    responseString: string;
+    response: any;
+    constructor(url: string, statusCode: number, responseString: string, encoding: string, response: any);
+}
+export declare class RequestErrorArgs {
+    url: string;
+    error: any;
+    constructor(url: string, error: any);
+    statusCode: number;
+    responseString: string;
+    get ErrorSummary(): string;
+}