octoparse-cli 0.1.14

package/node_modules/@octopus/engine/dist/utils/JsonParser.js

@@ -0,0 +1,439 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.JsonParsedArgs = exports.JsonNodeType = exports.JsonNode = exports.HttpMethod = exports.JsonRequest = void 0;
+const events_1 = require("events");
+const jsonpath_plus_1 = require("jsonpath-plus");
+const moment_1 = __importDefault(require("moment"));
+const xmlbuilder_1 = __importDefault(require("xmlbuilder"));
+const enums_1 = require("../enums");
+const elementNotFoundArgs_1 = require("../models/elementNotFoundArgs");
+const elementNotFoundError_1 = require("../models/elementNotFoundError");
+const _1 = require(".");
+const HttpRequester_1 = require("./HttpRequester");
+const lodash_1 = require("lodash");
+const xmlValueRegex = /[\0-\x08\x0B\f\x0E-\x1F\uFFFE\uFFFF]|[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?:[^\uD800-\uDBFF]|^)[\uDC00-\uDFFF]/g;
+class JsonParser extends events_1.EventEmitter {
+    constructor() {
+        super();
+        this.originText = '';
+        this.currentUrl = '';
+        this.UNICODE_REGEX = /[\u4e00-\u9fa5]/;
+        this.jsonpath = jsonpath_plus_1.JSONPath;
+        this.httpRequester = new HttpRequester_1.HttpRequester();
+        this.httpRequester.on('onRequested', (args) => {
+            this.httpJsonParser_OnRequested(args);
+        });
+        this.httpRequester.on('onRequestError', (args) => {
+            this.json = '';
+        });
+    }
+    onRequested(func) {
+        this.httpRequester.on('onRequested', (args) => {
+            func(args);
+        });
+    }
+    onRequestError(func) {
+        this.httpRequester.on('onRequestError', (args) => {
+            func(args);
+        });
+    }
+    onRequesting(func) {
+        this.httpRequester.on('onRequesting', (args) => {
+            func(args);
+        });
+    }
+    onRequestStopped(func) {
+        this.httpRequester.on('onRequestStopped', () => {
+            func();
+        });
+    }
+    onJsonParsed(func) {
+        this.on('onJsonParsed', (args) => {
+            func(args);
+        });
+    }
+    get json() {
+        return JSON.stringify(this.jsonObject);
+    }
+    set json(str) {
+        try {
+            this.jsonObject = JSON.parse(str);
+        }
+        catch (error) {
+            this.jsonObject = {};
+        }
+    }
+    get isRequesting() {
+        return this.httpRequester.isRequesting;
+    }
+    async startRequest(url, method, headers, bodyContent, timeout, requestCallback) {
+        await this.httpRequester.startRequest(url, method, headers, bodyContent, timeout, requestCallback);
+    }
+    getRoot() {
+        return this.jsonObject;
+    }
+    /**
+     * 获取路径末尾的[.*]
+     * @param path
+     */
+    getLastBracketString(path) {
+        if (path.endsWith(']')) {
+            return path.substring(path.lastIndexOf('['));
+        }
+        return '';
+    }
+    /**
+     * 将jsonpath转为8可用的格式
+     */
+    fixPathToV8(path, relativePath) {
+        if (this.UNICODE_REGEX.test(path)) {
+            const end = this.getLastBracketString(path);
+            path = path.substr(0, path.length - end.length);
+            if (!path.includes('.')) {
+                if (!path.includes('["')) {
+                    path = '$["' + path + '"]';
+                }
+            }
+            else {
+                const arrs = path.split('.');
+                arrs.forEach((a) => {
+                    if (!a.includes('["') && a.match(this.UNICODE_REGEX)) {
+                        path = path.replace('.' + a, `["${a}"]`);
+                    }
+                });
+            }
+            if (path.startsWith('[')) {
+                path = '$' + path;
+            }
+            path += end;
+        }
+        if (relativePath) {
+            if (this.UNICODE_REGEX.test(relativePath)) {
+                if (relativePath.startsWith('[') && relativePath.endsWith(']')) {
+                    path = path + relativePath;
+                }
+                else {
+                    path = `${path}["${relativePath}"]`;
+                }
+            }
+            else {
+                if (relativePath.startsWith('["')) {
+                    path = path + relativePath;
+                }
+                else {
+                    const absPathItems = jsonpath_plus_1.JSONPath.toPathArray(path);
+                    const relativePathItems = jsonpath_plus_1.JSONPath.toPathArray(relativePath);
+                    path = jsonpath_plus_1.JSONPath.toPathString(absPathItems.concat(relativePathItems));
+                }
+            }
+        }
+        return path;
+    }
+    getNode(absPath, relativePath) {
+        const root = this.getRoot();
+        if (root) {
+            const path = this.fixPathToV8(absPath, relativePath);
+            if (!path)
+                return undefined;
+            const results = (0, jsonpath_plus_1.JSONPath)({
+                path,
+                json: root,
+                resultType: 'all'
+            });
+            for (const result of results) {
+                const pathArray = jsonpath_plus_1.JSONPath.toPathArray(result.path);
+                const nodeName = pathArray[pathArray.length - 1].toString();
+                const n = new JsonNode(this.jsonpath, nodeName, result.value, pathArray);
+                return n;
+            }
+            return undefined;
+        }
+    }
+    loadFromJson(json) {
+        this.originText = json;
+        try {
+            if (!this.isNormalJson(json)) {
+                this.json = this.stripJsonpToJson(json);
+            }
+            else {
+                this.json = json;
+            }
+        }
+        catch (error) {
+            this.json = json;
+        }
+    }
+    httpJsonParser_OnRequested(args) {
+        let parseArgs;
+        try {
+            this.currentUrl = args.url;
+            this.loadFromJson(args.responseString);
+            parseArgs = new JsonParsedArgs(args.url, args.responseString, args.responseString, true);
+        }
+        catch (error) {
+            parseArgs = new JsonParsedArgs(args.url, '', '', false, error);
+        }
+        finally {
+            this.emit('onJsonParsed', parseArgs);
+        }
+    }
+    async extractText(textItem, baseJPath) {
+        let resultText = '';
+        let node;
+        switch (textItem.ExtractType) {
+            case "ExtractPageSource" /* ExtractTextType.ExtractPageSource */:
+                resultText = this.json;
+                break;
+            case "ExtractPageURL" /* ExtractTextType.ExtractPageURL */:
+                resultText = this.currentUrl;
+                break;
+            case "ExtractPageTitle" /* ExtractTextType.ExtractPageTitle */:
+                resultText = '';
+                break;
+            case "GenerateFixedValue" /* ExtractTextType.GenerateFixedValue */:
+                resultText = textItem.FixedValue;
+                break;
+            case "GenerateCurrentDateTime" /* ExtractTextType.GenerateCurrentDateTime */:
+                resultText = (0, moment_1.default)(new Date()).format('YYYY-MM-DD HH:mm:ss.SSSS');
+                break;
+            case "ExtractAddTaskSourceUrl" /* ExtractTextType.ExtractAddTaskSourceUrl */:
+                resultText = this.taskSourceUrl;
+                break;
+            default:
+                if (textItem.useRelativeXPath) {
+                    node = this.getNode(baseJPath, textItem.relativeXpath);
+                }
+                else {
+                    node = this.getNode(textItem.absXpath);
+                }
+                if (!node && textItem.backUpAbsXPath) {
+                    node = this.getNode(textItem.backUpAbsXPath);
+                }
+                if (node) {
+                    resultText = node.getValueText();
+                }
+                else {
+                    throw new elementNotFoundError_1.ElementNotFoundError('element not found, extraction failed');
+                }
+                break;
+        }
+        if (!(0, _1.isNullOrEmpty)(resultText)) {
+            if (textItem.operations != null && textItem.operations.length > 0) {
+                for (const operation of textItem.operations) {
+                    resultText = await operation.caculate(resultText);
+                }
+            }
+        }
+        return resultText;
+    }
+    async extractItem(extractItem, writer, baseJPath, data) {
+        if (extractItem == undefined) {
+            return false;
+        }
+        let resultText = '';
+        try {
+            resultText = await this.extractText(extractItem, baseJPath);
+        }
+        catch (error) {
+            if (error instanceof elementNotFoundError_1.ElementNotFoundError) {
+                const e = new elementNotFoundArgs_1.ElementNotFoundArgs(baseJPath, extractItem);
+                // TODO
+                // RaiseElementNotFoundEvent(e)
+                switch (e.HandleType) {
+                    case enums_1.ElementNotFoundHandleType.AllowSkip:
+                        return false;
+                        break;
+                    case enums_1.ElementNotFoundHandleType.SetBackup:
+                    case enums_1.ElementNotFoundHandleType.ModifyXPath:
+                        break;
+                    case enums_1.ElementNotFoundHandleType.NotHandle:
+                        if (extractItem.allowSkip) {
+                            return false;
+                        }
+                        else if (extractItem.allowDefaultValue) {
+                            resultText = extractItem.NullValue;
+                        }
+                        break;
+                    case enums_1.ElementNotFoundHandleType.AllowEmpty:
+                    default:
+                        break;
+                }
+            }
+            console.error(error);
+        }
+        finally {
+            data[extractItem.header] = resultText;
+            /** 去除字段内容中的 XML 的非法字符 */
+            resultText = resultText.replace(xmlValueRegex, '');
+            writer.ele(extractItem.header, { IsAppend: extractItem.IsAppend }, resultText);
+        }
+        return true;
+    }
+    async extractGroupItem(extractTemplate, extractItem, writer, baseJPath, data) {
+        if ((0, lodash_1.isNil)(extractItem)) {
+            return false;
+        }
+        if (extractItem.header != 'Root') {
+            writer.ele(extractItem.header);
+        }
+        const childs = extractTemplate.getChildren(extractItem.id);
+        if (childs.length > 0) {
+            for (const item of childs) {
+                if (item.type == 'ExtractGroupItem') {
+                    return await this.extractGroupItem(extractTemplate, extractItem, writer, baseJPath, data);
+                }
+                else {
+                    if (!(await this.extractItem(item, writer, baseJPath, data))) {
+                        return true;
+                    }
+                }
+            }
+        }
+        return false;
+    }
+    /** 当前任务源网址 */
+    async extractFromTemplate(extractTemplate, baseJPath, data, taskSourceUrl) {
+        // let resultXmlFile: string = path.join(foldername, uuid.v1() + '.xml');
+        this.taskSourceUrl = taskSourceUrl;
+        let skipExtractAction = false;
+        const writer = xmlbuilder_1.default.create('Root');
+        const root = extractTemplate.getRoot();
+        skipExtractAction = await this.extractGroupItem(extractTemplate, root, writer, baseJPath, data);
+        if (skipExtractAction) {
+            for (const item of extractTemplate.items) {
+                writer.ele(item.name);
+            }
+        }
+        const xmlStr = writer.end({ pretty: true });
+        return xmlStr;
+    }
+    isNormalJson(text) {
+        if ((0, _1.isNullOrEmpty)(text)) {
+            return false;
+        }
+        const trimText = text.trim();
+        if ((0, _1.isNullOrEmpty)(trimText)) {
+            return false;
+        }
+        return trimText[0] == '[' || trimText[0] == '{';
+    }
+    stripJsonpToJson(text) {
+        const start = text.indexOf('(');
+        const end = text.lastIndexOf(')');
+        const length = text.length;
+        const newLen = end - start - 1;
+        if (newLen > 1) {
+            const str = text.substring(start + 1, end);
+            return str;
+        }
+        return text;
+    }
+}
+exports.default = JsonParser;
+class JsonRequest {
+    constructor() {
+        this.method = HttpMethod.Get;
+        this.url = '';
+        this.headers = new Map();
+        this.body = '';
+    }
+    addHeader(name, value) {
+        if (!(0, _1.isNullOrEmpty)(name)) {
+            this.headers.set(name, value === undefined ? '' : value);
+        }
+    }
+    removeHeader(name) {
+        if (!(0, _1.isNullOrEmpty)(name) && this.headers.has(name)) {
+            this.headers.delete(name);
+        }
+    }
+    getHeaders() {
+        return this.headers;
+    }
+    setBody(body) {
+        this.body = body;
+    }
+    getBody() {
+        return this.body;
+    }
+}
+exports.JsonRequest = JsonRequest;
+var HttpMethod;
+(function (HttpMethod) {
+    HttpMethod["Get"] = "GET";
+    HttpMethod["Post"] = "POST";
+})(HttpMethod || (exports.HttpMethod = HttpMethod = {}));
+class JsonNode {
+    constructor(jsonpath, name, value, pathItems) {
+        this.selected = false;
+        this.childs = [];
+        this._jsonpath = jsonpath;
+        this._name = name;
+        this._value = value;
+        this._pathItems = pathItems;
+    }
+    /** 获取json节点的value文本 */
+    getValueText() {
+        if (typeof this._value == 'object')
+            return JSON.stringify(this._value);
+        else
+            return this._value.toString();
+    }
+    /** 在json browser上展示的文本 */
+    get text() {
+        const type = this.type;
+        let val = this._value;
+        switch (type) {
+            case JsonNodeType.Value:
+                if (val == null)
+                    val = '<null>';
+                else if (val)
+                    val = `"${val}"`;
+                return `${this._name} : ${val}`;
+            case JsonNodeType.Array:
+                return `${this._name} [${val.length}]`;
+            case JsonNodeType.Object:
+                return `${this._name} {${Object.keys(val).length}}`;
+        }
+        return '';
+    }
+    get path() {
+        return jsonpath_plus_1.JSONPath.toPathString(this._pathItems.map(String));
+    }
+    get pathItems() {
+        return this._pathItems;
+    }
+    get type() {
+        if (Array.isArray(this._value))
+            return JsonNodeType.Array;
+        if (typeof this._value === 'boolean' ||
+            typeof this._value === 'string' ||
+            typeof this._value === 'number' ||
+            this._value === null)
+            return JsonNodeType.Value;
+        return JsonNodeType.Object;
+    }
+    get value() {
+        return this._value;
+    }
+}
+exports.JsonNode = JsonNode;
+var JsonNodeType;
+(function (JsonNodeType) {
+    JsonNodeType["Object"] = "Object";
+    JsonNodeType["Array"] = "Array";
+    JsonNodeType["Value"] = "Value";
+})(JsonNodeType || (exports.JsonNodeType = JsonNodeType = {}));
+class JsonParsedArgs {
+    constructor(url, originText, json, parseSuccess, parseException) {
+        this.url = url;
+        this.originText = originText;
+        this.json = json;
+        this.parseSuccess = parseSuccess;
+        this.parseException = parseException;
+    }
+}
+exports.JsonParsedArgs = JsonParsedArgs;

package/node_modules/@octopus/engine/dist/utils/Operations.d.ts

@@ -0,0 +1,27 @@
+import { Page } from 'puppeteer-core';
+import { ActionItem } from '../models/actionItem';
+/**
+ * 判定是否属于“页面上下文暂时不可用”的瞬时错误。
+ * 这类错误通常不是 XPath 本身有问题，而是查询发生在 context 重建窗口期。
+ */
+export declare function isTransientContextError(error: unknown): boolean;
+/**
+ * 根据 ActionItem 获取元素句柄。
+ * 这个方法只负责“找元素”，不执行页面内脚本。
+ * 如果目标在 iframe 中，会先定位 iframe，再进入 frame 内查找真实元素。
+ */
+export declare const getElement: (actionItem: ActionItem, page: Page) => Promise<import("puppeteer-core").ElementHandle<Element>>;
+/**
+ * 在目标元素上执行 Puppeteer 的 `$eval` / 其他同类调用。
+ * 这里把主文档和 iframe 的分支统一收口，同时把 context 重试放在公共层，
+ * 让提取文本、读取属性、页面内局部滚动等调用都复用同一套容错逻辑。
+ */
+export declare const processElement: <TFnEle, TFnParam, TFnReturn>(actionItem: ActionItem, page: Page, $type: string, Fn: (ele: TFnEle, value?: TFnParam) => TFnReturn, FnParam?: TFnParam) => Promise<TFnReturn>;
+/**
+ * 查询元素，并在需要时先做一次显式等待。
+ *
+ * 和 getElement 的区别：
+ * - getElement：立即查找，偏向“当前时刻有没有”
+ * - queryElement：可附带 timeout，偏向“在一小段时间内等它出现”
+ */
+export declare const queryElement: (actionItem: ActionItem, page: Page, timeout?: number) => Promise<import("puppeteer-core").ElementHandle<Element>>;

package/node_modules/@octopus/engine/dist/utils/Operations.js

@@ -0,0 +1,115 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.queryElement = exports.processElement = exports.getElement = void 0;
+exports.isTransientContextError = isTransientContextError;
+// rebrowser / Puppeteer 在页面跳转、局部刷新、iframe 重建时，
+// 偶发会出现 execution context 短暂失效的问题。
+// 这里统一做一次轻量重试，避免上层把这种瞬时抖动误判成“元素不存在”。
+const CONTEXT_RETRY_COUNT = 3;
+const CONTEXT_RETRY_DELAY_MS = 120;
+/**
+ * 判定是否属于“页面上下文暂时不可用”的瞬时错误。
+ * 这类错误通常不是 XPath 本身有问题，而是查询发生在 context 重建窗口期。
+ */
+function isTransientContextError(error) {
+    const message = error instanceof Error
+        ? `${error.message}\n${error.stack || ''}`
+        : String(error);
+    return (message.includes('acquireContextId failed') ||
+        message.includes('Execution context was destroyed') ||
+        message.includes('Cannot find context with specified id') ||
+        message.includes('Inspected target navigated or closed'));
+}
+async function sleep(delay) {
+    await new Promise((resolve) => setTimeout(resolve, delay));
+}
+/**
+ * 对底层 DOM 访问做短重试。
+ * 只重试瞬时 context 错误；真正的业务错误、XPath 错误仍然直接抛出。
+ *
+ */
+async function withContextRetry(task) {
+    let lastError;
+    for (let tryCount = 0; tryCount < CONTEXT_RETRY_COUNT; tryCount++) {
+        try {
+            return await task();
+        }
+        catch (error) {
+            lastError = error;
+            if (!isTransientContextError(error) || tryCount === CONTEXT_RETRY_COUNT - 1) {
+                throw error;
+            }
+            await sleep(CONTEXT_RETRY_DELAY_MS * (tryCount + 1));
+        }
+    }
+    throw lastError;
+}
+/**
+ * 根据 ActionItem 获取元素句柄。
+ * 这个方法只负责“找元素”，不执行页面内脚本。
+ * 如果目标在 iframe 中，会先定位 iframe，再进入 frame 内查找真实元素。
+ */
+const getElement = async (actionItem, page) => {
+    let element = await withContextRetry(() => page.$(`::-p-xpath(${actionItem.AbsXpath})`));
+    if (actionItem.IsIFrame) {
+        const iframeElement = await withContextRetry(() => page.$(`::-p-xpath(${actionItem.IFrameAbsXPath})`));
+        if (iframeElement) {
+            const iframe = await iframeElement.contentFrame();
+            element = await withContextRetry(() => iframe.$(`::-p-xpath(${actionItem.AbsXpath})`));
+        }
+    }
+    return element;
+};
+exports.getElement = getElement;
+/**
+ * 在目标元素上执行 Puppeteer 的 `$eval` / 其他同类调用。
+ * 这里把主文档和 iframe 的分支统一收口，同时把 context 重试放在公共层，
+ * 让提取文本、读取属性、页面内局部滚动等调用都复用同一套容错逻辑。
+ */
+const processElement = async (actionItem, page, $type, Fn, FnParam) => {
+    let attributeValue;
+    if (actionItem.IsIFrame) {
+        const iframeElement = await withContextRetry(() => page.$(`::-p-xpath(${actionItem.IFrameAbsXPath})`));
+        if (iframeElement) {
+            const iframe = await iframeElement.contentFrame();
+            attributeValue = await withContextRetry(() => iframe[$type](`::-p-xpath(${actionItem.AbsXpath})`, Fn, FnParam));
+        }
+    }
+    else {
+        attributeValue = await withContextRetry(() => page[$type](`::-p-xpath(${actionItem.AbsXpath})`, Fn, FnParam));
+    }
+    return attributeValue;
+};
+exports.processElement = processElement;
+/**
+ * 查询元素，并在需要时先做一次显式等待。
+ *
+ * 和 getElement 的区别：
+ * - getElement：立即查找，偏向“当前时刻有没有”
+ * - queryElement：可附带 timeout，偏向“在一小段时间内等它出现”
+ */
+const queryElement = async (actionItem, page, timeout = 0) => {
+    const { IsIFrame, AbsXpath, IFrameAbsXPath } = actionItem;
+    if (timeout) {
+        // 等待阶段仍然只等最外层目标：
+        // iframe 场景先确保 iframe 本身可用，后面的 frame.$ 再去定位内部元素。
+        await page
+            .locator(`::-p-xpath(${IsIFrame ? IFrameAbsXPath : AbsXpath})`)
+            .setTimeout(timeout)
+            .wait()
+            .catch((er) => {
+            console.error('等待元素超时', er.message);
+        });
+    }
+    if (IsIFrame) {
+        const iframe = await withContextRetry(() => page.$(`::-p-xpath(${IFrameAbsXPath})`));
+        const frame = await (iframe === null || iframe === void 0 ? void 0 : iframe.contentFrame());
+        const element = frame
+            ? await withContextRetry(() => frame.$(`::-p-xpath(${AbsXpath})`))
+            : undefined;
+        return element;
+    }
+    const element = await withContextRetry(() => page.$(`::-p-xpath(${AbsXpath})`));
+    return element;
+};
+exports.queryElement = queryElement;

package/node_modules/@octopus/engine/dist/utils/index.d.ts

@@ -0,0 +1,28 @@
+export declare const parseXML: (xml: string, cb: (er: any, data: any) => void) => void;
+/** 解析 XML 转为对象 */
+export declare const parseXMLToObj: (xml: string) => {
+    data: {};
+    isAppend: boolean;
+};
+export declare const sleep: (delay?: number) => Promise<void>;
+/** 生成任务流程的全部【提取步骤】的序号映射表 */
+export declare const generateExtractIndexMap: (xml: string) => Map<string, number>;
+export declare const getGlobalCookie: (xml: string) => {
+    globalCookie: string;
+    isSetGlobalCookie: boolean;
+};
+export declare const isNullOrEmpty: (value?: string) => boolean;
+export declare const correctUrl: (url: string) => string;
+/**
+ * 在 Worker 线程中执行耗时的 JavaScript 代码，可以设置超时时间自动结束
+ * PS：这个功能目前主要给执行正则表达式时使用，因为 string.replace() 等方法是同步执行，当执行一些用户输入的不可预期的正则表达式时会卡死的当前 UI 线程
+ * @see https://skieer.feishu.cn/docs/doccncVezuTFcky9wboyAkCPLuc?sidebarOpen=1
+ * @param code 需要执行的代码
+ * @param timeout 超时时间，默认为 10s
+ */
+export declare const executeJavaScriptInWorker: (code: string, timeout?: number) => Promise<any>;
+export declare const hashCode: (str: string) => number;
+export declare const random: (min: number, max: number) => number;
+/** 转换时间描述文本为标准时间对象 */
+export declare const transformDate: (input: string) => Date | undefined;
+export declare const repairURL: (url: string) => string;