octoparse-cli 0.1.14

package/node_modules/@octopus/engine/dist/utils/HttpRequester.js

@@ -0,0 +1,174 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RequestErrorArgs = exports.RequestedArgs = exports.RequestingArgs = exports.HttpRequester = void 0;
+const events_1 = require("events");
+const https_1 = require("https");
+const axios_1 = __importDefault(require("axios"));
+// eslint-disable-next-line
+const iconv_lite_1 = __importDefault(require("iconv-lite"));
+const _1 = require(".");
+// Axios 会自动根据环境选择合适的 adapter
+// Node.js 环境自动使用 http/https adapter，浏览器环境使用 xhr adapter
+// 移除手动设置以避免 ESM 兼容性问题
+const agent = new https_1.Agent({ rejectUnauthorized: false });
+class HttpRequester extends events_1.EventEmitter {
+    constructor() {
+        super();
+        this.isRequesting = false;
+    }
+    startRequest(url, method, headers, bodyContent, timeout, requestCallback) {
+        //兼容v7版本的客户端生成的规则的问题，如果v7版本不再使用，即可删除该逻辑
+        if (bodyContent &&
+            bodyContent.startsWith('{}') &&
+            !bodyContent.endsWith('{}')) {
+            bodyContent = bodyContent.slice(2, bodyContent.length);
+        }
+        this.isRequesting = true;
+        return new Promise((resolve, reject) => {
+            try {
+                const requestingArgs = new RequestingArgs(url, method, headers, bodyContent);
+                // if(this.onRequesting){
+                //     this.onRequesting(requestingArgs);
+                // }
+                this.emit('onRequesting', requestingArgs);
+                if (requestingArgs.cancel) {
+                    // if(this.onRequested){
+                    //     this.onRequestStopped();
+                    //     resolve();
+                    //     return;
+                    // }
+                    this.emit('onRequestStopped');
+                    return;
+                }
+            }
+            catch (error) {
+                console.error('🔴 请求前处理错误:', error);
+            }
+            /**
+             * 切换到axios的原因：可以请求到更多JSON接口，支持timeout, 避免系统长时间的无效等待
+             */
+            const config = {
+                url: HttpRequester.checkChinese(url) ? encodeURI(url) : url,
+                method,
+                headers: {
+                    Accept: 'application/json, text/plain, */*',
+                    'User-Agent': HttpRequester.userAgent || 'Mozilla/5.0',
+                    Cookie: ''
+                },
+                data: bodyContent,
+                timeout,
+                responseType: 'arraybuffer',
+                httpsAgent: agent
+            };
+            if (headers && headers.size > 0) {
+                for (const k of headers) {
+                    config.headers[k[0]] = k[1];
+                }
+            }
+            if (!(0, _1.isNullOrEmpty)(HttpRequester.xForwardFor)) {
+                config.headers['X-Forwarded-For'] = HttpRequester.xForwardFor;
+            }
+            if (HttpRequester.cookies) {
+                let cookiesStr = '';
+                for (const cookie of HttpRequester.cookies) {
+                    cookiesStr += cookie.name + '=' + cookie.value + ';';
+                }
+                config.headers.Cookie = cookiesStr;
+            }
+            (0, axios_1.default)(config)
+                .then((response) => {
+                var _a;
+                this.isRequesting = false;
+                try {
+                    const charset = (response.headers &&
+                        ((_a = response.headers['content-type']) === null || _a === void 0 ? void 0 : _a.toLowerCase().split('charset=')[1])) ||
+                        'utf8';
+                    const body = iconv_lite_1.default.decode(response.data, charset);
+                    const responseCode = response.status;
+                    const requestedArgs = new RequestedArgs((0, _1.isNullOrEmpty)(response.config.url) ? url : response.config.url, responseCode, body, '', response);
+                    this.emit('onRequested', requestedArgs);
+                    if (requestCallback)
+                        requestCallback();
+                    resolve();
+                }
+                catch (error) {
+                    const errorArgs = new RequestErrorArgs(url, error);
+                    this.emit('onRequestError', errorArgs);
+                    reject(error);
+                }
+            })
+                .catch((error) => {
+                this.isRequesting = false;
+                const errorArgs = new RequestErrorArgs(url, error);
+                this.emit('onRequestError', errorArgs);
+                reject(error);
+            });
+        });
+    }
+    /**
+     * 判断网页地址是否包含中文字符
+     * @param str
+     */
+    static checkChinese(str) {
+        const reg = /[\u4E00-\u9FFF]+/g;
+        return reg.test(str);
+    }
+}
+exports.HttpRequester = HttpRequester;
+class RequestingArgs {
+    constructor(url, method, requestHeaders, bodyContent) {
+        this.url = url;
+        this.method = method;
+        this.requestHeaders = requestHeaders;
+        this.bodyContent = bodyContent;
+        this.cancel = false;
+    }
+}
+exports.RequestingArgs = RequestingArgs;
+class RequestedArgs {
+    constructor(url, statusCode, responseString, encoding, response) {
+        this.url = url;
+        this.statusCode = statusCode;
+        this.responseString = responseString;
+        this.encoding = encoding;
+        this.response = response;
+    }
+}
+exports.RequestedArgs = RequestedArgs;
+class RequestErrorArgs {
+    constructor(url, error) {
+        this.statusCode = 0;
+        this.responseString = '';
+        this.url = url;
+        this.error = error;
+    }
+    // public SocketException SocketException { get { return Exception?.InnerException as SocketException; } }
+    // public WebException WebException { get { return Exception as WebException; } }
+    get ErrorSummary() {
+        let summary = '';
+        // if (WebException != null)
+        // {
+        //     summary += ($"{(int)StatusCode}: {StatusCode}") + Environment.NewLine;
+        //     summary += ("Status: " + WebException.Status) + Environment.NewLine;
+        //     if (!string.IsNullOrEmpty(ResponseString))
+        //     {
+        //         summary += "Server Message: " + ResponseString + Environment.NewLine;
+        //     }
+        //     summary += "Message: " + Exception + Environment.NewLine;
+        // }
+        // else
+        {
+            summary += 'Message: ' + this.error;
+        }
+        // if (SocketException != null)
+        // {
+        //     summary += "SocketException: " + SocketException.SocketErrorCode.ToString() + Environment.NewLine;
+        //     summary += "Message: " + SocketException + Environment.NewLine;
+        // }
+        return summary;
+    }
+}
+exports.RequestErrorArgs = RequestErrorArgs;

package/node_modules/@octopus/engine/dist/utils/JsonParser.d.ts

@@ -0,0 +1,95 @@
+import { EventEmitter } from 'events';
+import xmlbuilder from 'xmlbuilder';
+import { ElementNotFoundArgs } from '../models/elementNotFoundArgs';
+import { ExtractItem } from '../models/extractItem';
+import { ExtractTemplate } from '../models/extractTemplate';
+import { ExtractTextItem } from '../models/extractTextItem';
+import { RequestedArgs, RequestErrorArgs, RequestingArgs } from './HttpRequester';
+export default class JsonParser extends EventEmitter {
+    private jsonObject;
+    private httpRequester?;
+    onElementNotFound?: (args: ElementNotFoundArgs) => void;
+    onReport?: (message: string) => void;
+    private jsonpath;
+    /** 当前任务源网址 */
+    private taskSourceUrl;
+    constructor();
+    onRequested(func: (args: RequestedArgs) => void): void;
+    onRequestError(func: (args: RequestErrorArgs) => void): void;
+    onRequesting(func: (args: RequestingArgs) => void): void;
+    onRequestStopped(func: () => void): void;
+    onJsonParsed(func: (args: JsonParsedArgs) => void): void;
+    get json(): string;
+    set json(str: string);
+    originText: string;
+    currentUrl: string;
+    get isRequesting(): boolean;
+    startRequest(url: string, method: string, headers: Map<string, string>, bodyContent: string, timeout: number, requestCallback?: (resp?: unknown) => void): Promise<void>;
+    getRoot(): any;
+    private readonly UNICODE_REGEX;
+    /**
+     * 获取路径末尾的[.*]
+     * @param path
+     */
+    private getLastBracketString;
+    /**
+     * 将jsonpath转为8可用的格式
+     */
+    private fixPathToV8;
+    getNode(absPath: string, relativePath?: string): JsonNode | undefined;
+    loadFromJson(json: string): void;
+    httpJsonParser_OnRequested(args: RequestedArgs): void;
+    extractText(textItem: ExtractTextItem, baseJPath: string): Promise<string>;
+    extractItem(extractItem: ExtractItem | undefined, writer: xmlbuilder.XMLElement, baseJPath: string, data: any): Promise<boolean>;
+    extractGroupItem(extractTemplate: ExtractTemplate, extractItem: ExtractItem, writer: xmlbuilder.XMLElement, baseJPath: string, data: any): Promise<boolean>;
+    /** 当前任务源网址 */
+    extractFromTemplate(extractTemplate: ExtractTemplate, baseJPath: string, data: any, taskSourceUrl: any): Promise<string>;
+    isNormalJson(text: string): boolean;
+    stripJsonpToJson(text: string): string;
+}
+export declare class JsonRequest {
+    method: HttpMethod;
+    url: string;
+    private headers;
+    private body;
+    addHeader(name: string, value: string): void;
+    removeHeader(name: string): void;
+    getHeaders(): Map<string, string>;
+    setBody(body: string): void;
+    getBody(): string;
+}
+export declare enum HttpMethod {
+    Get = "GET",
+    Post = "POST"
+}
+export declare class JsonNode {
+    private _name;
+    private _value;
+    private _pathItems;
+    private _jsonpath;
+    selected: boolean;
+    childs: JsonNode[];
+    parent: JsonNode | undefined;
+    constructor(jsonpath: any, name: string, value: any, pathItems: (string | number)[]);
+    /** 获取json节点的value文本 */
+    getValueText(): string;
+    /** 在json browser上展示的文本 */
+    get text(): string;
+    get path(): string;
+    get pathItems(): (string | number)[];
+    get type(): JsonNodeType;
+    get value(): any;
+}
+export declare enum JsonNodeType {
+    Object = "Object",
+    Array = "Array",
+    Value = "Value"
+}
+export declare class JsonParsedArgs {
+    url: string;
+    originText: string;
+    json: string;
+    parseSuccess: boolean;
+    parseException: any;
+    constructor(url: string, originText: string, json: string, parseSuccess: boolean, parseException?: any);
+}