octoparse-cli 0.1.14

package/node_modules/@octopus/engine/dist/configs/ExtractDataConfig.js

@@ -0,0 +1 @@
+'use strict';var __importDefault=this&&this['__importDefault']||function(_0x1c7bdd){return _0x1c7bdd&&_0x1c7bdd['__esModule']?_0x1c7bdd:{'default':_0x1c7bdd};};Object['defineProperty'](exports,'__esModule',{'value':!![]});const extractItem_1=require('../models/extractItem'),operation_1=require('../models/operation'),trigger_1=require('../models/trigger'),downloadFileConfig_1=require('../models/downloadFileConfig'),BaseConfig_1=__importDefault(require('./BaseConfig'));class ExtractDataConfig extends BaseConfig_1['default']{constructor(_0x3ca109,_0x35d49d){super(_0x3ca109),this['extractTriggerStatus']=0x0,this['url']=_0x3ca109['URL'],this['extractTriggerStatus']=parseInt(_0x3ca109['ExtractTriggerStatus']),this['filePath']=_0x3ca109['FilePath'],this['extractTemplate']=new Array(),this['triggers']=new Array(),this['uniqueGroup']=new Array();for(let _0x38c3e7 in _0x35d49d['values']){let _0x3ed068=_0x35d49d['values'][_0x38c3e7];if(_0x3ed068['$type']=='xml:ExtractDataAction.ExtractTemplate'&&_0x3ed068['$children'])for(let _0x4eb4ad of _0x3ed068['$children']){let _0x140cd8=new extractItem_1['ExtractItem']();_0x140cd8['absXpath']=_0x4eb4ad['AbsXpath'],_0x140cd8['id']=parseInt(_0x4eb4ad['Id']),_0x140cd8['parentId']=parseInt(_0x4eb4ad['ParentId']),_0x140cd8['alertWhenNotFound']=_0x4eb4ad['AlertWhenNotFound']=='true',_0x140cd8['allowDefaultValue']=_0x4eb4ad['AllowDefaultValue']=='true',_0x140cd8['allowNull']=_0x4eb4ad['AllowNull']=='true',_0x140cd8['allowSkip']=_0x4eb4ad['AllowSkip']=='true',_0x140cd8['backUpAbsXPath']=_0x4eb4ad['BackUpAbsXPath'],_0x140cd8['backUpRelativeXPath']=_0x4eb4ad['BackUpRelativeXPath'],_0x140cd8['useBackupIframePath']=_0x4eb4ad['UseBackupIframePath']=='true',_0x140cd8['backUpIframeXPath']=_0x4eb4ad['BackUpIframeXPath'],_0x140cd8['useBackupPath']=_0x4eb4ad['UseBackupPath']=='true',_0x140cd8['useBackupRelativePath']=_0x4eb4ad['UseBackupRelativePath']=='true',_0x140cd8['header']=_0x4eb4ad['Header'],_0x140cd8['name']=_0x4eb4ad['Name'],_0x140cd8['relativeXpath']=_0x4eb4ad['RelativeXpath'],_0x140cd8['useRelativeXPath']=_0x4eb4ad['UseRelativeXPath']=='true',_0x140cd8['operations']=[],_0x140cd8['ExtractType']=_0x4eb4ad['ExtractType'],_0x140cd8['type']=_0x4eb4ad['type'],_0x140cd8['CustomizeField']=_0x4eb4ad['CustomizeField'];if(_0x4eb4ad['$children'])for(let _0x211a94 of _0x4eb4ad['$children']){if(_0x211a94['$type']=='xml:Operation'){let _0x1a64b5=new operation_1['Operation']();_0x1a64b5['formatType']=_0x211a94['FormatType'],_0x1a64b5['index']=_0x211a94['Index'],_0x1a64b5['input']=_0x211a94['Input'],_0x1a64b5['name']=_0x211a94['Name'],_0x1a64b5['output']=_0x211a94['Output'],_0x1a64b5['parameters']=[];if(_0x211a94['$children'])for(let _0x464b04 of _0x211a94['$children']){_0x464b04['$type']=='xml:Parameter'&&_0x1a64b5['parameters']['push'](_0x464b04['Value']);}_0x140cd8['operations']['push'](_0x1a64b5);}}_0x140cd8['IsAppend']=_0x4eb4ad['IsAppend']=='true',_0x140cd8['MatchAll']=_0x4eb4ad['MatchAll']=='true',_0x140cd8['FixedValue']=_0x4eb4ad['FixedValue']?_0x4eb4ad['FixedValue']:'',_0x140cd8['NullValue']=_0x4eb4ad['NullValue']?_0x4eb4ad['NullValue']:'',_0x140cd8['isIFrame']=_0x4eb4ad['IsIFrame']=='true';_0x140cd8['isIFrame']&&(_0x140cd8['iFrameXpath']=_0x4eb4ad['IFrameAbsXPath']);_0x140cd8['IsUniqueGroup']=_0x4eb4ad['IsUniqueGroup']=='true';_0x140cd8['IsUniqueGroup']&&this['uniqueGroup']['push'](_0x140cd8['name']);_0x140cd8['IsDownloadFile']=_0x4eb4ad['IsDownloadFile']=='true';if(_0x4eb4ad['DownloadFileConfig'])try{const _0x419dfe=typeof _0x4eb4ad['DownloadFileConfig']==='string'?JSON['parse'](_0x4eb4ad['DownloadFileConfig']):_0x4eb4ad['DownloadFileConfig'],_0x6a0b34=new downloadFileConfig_1['DownloadFileConfig']();_0x419dfe['downloadFileRenameType']!==undefined&&(_0x6a0b34['downloadFileRenameType']=_0x419dfe['downloadFileRenameType']);_0x419dfe['repeatValue']!==undefined&&(_0x6a0b34['repeatValue']=_0x419dfe['repeatValue']);_0x419dfe['field']!==undefined&&(_0x6a0b34['field']=_0x419dfe['field']);_0x419dfe['excludeUrls']!==undefined&&(_0x6a0b34['excludeUrls']=_0x419dfe['excludeUrls']);if(_0x419dfe['mutipleUrlConfig']!==undefined){const _0x2c5633=new downloadFileConfig_1['MutipleUrlConfig']();_0x419dfe['mutipleUrlConfig']['isIncludeMutipleUrl']!==undefined&&(_0x2c5633['isIncludeMutipleUrl']=_0x419dfe['mutipleUrlConfig']['isIncludeMutipleUrl']),_0x419dfe['mutipleUrlConfig']['urlSplitType']!==undefined&&(_0x2c5633['urlSplitType']=_0x419dfe['mutipleUrlConfig']['urlSplitType']),_0x6a0b34['mutipleUrlConfig']=_0x2c5633;}_0x140cd8['DownloadFileConfig']=_0x6a0b34;}catch(_0x26a39c){_0x140cd8['DownloadFileConfig']=new downloadFileConfig_1['DownloadFileConfig']();}this['extractTemplate'][this['extractTemplate']['length']]=_0x140cd8;}else{if(_0x3ed068['$type']=='xml:ExtractDataAction.Triggers'&&_0x3ed068['$children'])for(let _0x214ba1 of _0x3ed068['$children']){for(let _0x2012d4 of _0x214ba1['$children']){let _0x4b1c04=new trigger_1['Trigger']();_0x4b1c04['Status']=Number(_0x2012d4['Status']),_0x4b1c04['Priority']=Number(_0x2012d4['Priority']),_0x4b1c04['Name']=_0x2012d4['Name'],_0x4b1c04['LoopLevel']=Number(_0x2012d4['LoopLevel']),_0x4b1c04['Enabled']=_0x2012d4['Enabled']=='true'||_0x2012d4['Enabled']=='True';for(let _0x4cdc67 of _0x2012d4['$children']){let _0x3674c6=new trigger_1['TriggerAdditionGroup'](),_0x52ddce=[];_0x3674c6['Id']=_0x3674c6['Id'];for(let _0x5cd118 of _0x4cdc67['$children']){let _0x39f257=new trigger_1['TriggerAddition']();_0x39f257['Content']=_0x5cd118['Content'],_0x39f257['DateTimeType']=_0x5cd118['DateTimeType'],_0x39f257['Day']=Number(_0x5cd118['Day']),_0x39f257['FieldName']=_0x5cd118['FieldName'],_0x39f257['FormatDateTimeType']=_0x5cd118['FormatDateTimeType'],_0x39f257['FormatTemplate']=_0x5cd118['FormatTemplate'],_0x39f257['Hour']=Number(_0x5cd118['Hour']),_0x39f257['Id']=Number(_0x5cd118['Id']),_0x39f257['Increase']=_0x5cd118['Increase']=='true'||_0x5cd118['Increase']=='True',_0x39f257['Minute']=Number(_0x5cd118['Minute']),_0x39f257['Month']=Number(_0x5cd118['Month']),_0x39f257['OperatingSymbol']=_0x5cd118['OperatingSymbol'],_0x39f257['Second']=Number(_0x5cd118['Second']),_0x39f257['Year']=Number(_0x5cd118['Year']);if(_0x5cd118['$children']!=undefined)for(let _0x38ac21 of _0x5cd118['$children']){if(_0x38ac21['$type']=='xml:Operation'){let _0x1a5eb9=new operation_1['Operation']();_0x1a5eb9['formatType']=_0x38ac21['FormatType'],_0x1a5eb9['index']=Number(_0x38ac21['Index']),_0x1a5eb9['input']=_0x38ac21['Input'],_0x1a5eb9['name']=_0x38ac21['Name'],_0x1a5eb9['output']=_0x38ac21['Output'],_0x1a5eb9['parameters']=[];for(let _0x592f21 of _0x38ac21['$children']){_0x592f21['$type']=='xml:Parameter'&&_0x1a5eb9['parameters']['push'](_0x592f21['Value']);}_0x39f257['Operations']['push'](_0x1a5eb9);}}_0x52ddce['push'](_0x39f257);}_0x3674c6['TriggerAdditions']=_0x52ddce,_0x4b1c04['TriggerAdditionGroups']['push'](_0x3674c6);}this['triggers']['push'](_0x4b1c04);}}}}}}exports['default']=ExtractDataConfig;

package/node_modules/@octopus/engine/dist/configs/LoopConfig.d.ts

@@ -0,0 +1,25 @@
+import { ScrollType, ScrollScope, LoopType } from '../enums';
+import { ActionItem } from '../models/actionItem';
+import BaseConfig from './BaseConfig';
+export default class LoopConfig extends BaseConfig {
+    url: string;
+    loopType: LoopType;
+    quitLoopWhenExecutedTimesEqual: boolean;
+    executedTimesLimitation: number;
+    fixedItem?: ActionItem;
+    waitItem?: string;
+    variableList?: ActionItem;
+    scrollDown: boolean;
+    scrollTime: number;
+    scrollInterval: number;
+    scrollIntervalUnit?: string;
+    scrollType: ScrollType;
+    scrollScope?: ScrollScope;
+    scrollXPath?: ActionItem;
+    IfStopScroll: boolean;
+    /** 是否勾选跳过无效循环点击 */
+    checkedJumpLoopClick: boolean;
+    /** 是否勾选滚动时循环列表不自动去重 */
+    disabledScrollAutoRemoveDuplication: boolean;
+    constructor(config: any);
+}

package/node_modules/@octopus/engine/dist/configs/LoopConfig.js

@@ -0,0 +1,40 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const utils_1 = require("../utils");
+const BaseConfig_1 = __importDefault(require("./BaseConfig"));
+class LoopConfig extends BaseConfig_1.default {
+    constructor(config) {
+        super(config);
+        this.executedTimesLimitation = 0;
+        this.url = (0, utils_1.repairURL)(config.URL);
+        this.loopType = config.LoopType;
+        this.quitLoopWhenExecutedTimesEqual = config.QuitLoopWhenExecutedTimesEqual == 'true';
+        this.executedTimesLimitation = parseInt(config.ExecutedTimesLimitation);
+        if (this.loopType == "FixedItem" /* LoopType.FixedItem */) {
+            this.fixedItem = this.getElementXPath(config.FixedItem);
+        }
+        this.waitItem = config.WaitItem;
+        this.variableList = this.getElementXPath(config.VariableList);
+        this.scrollDown = config.ScrollDown;
+        this.scrollTime = parseInt(config.ScrollTime);
+        if (config.ScrollIntervalUnit == 'ms') {
+            this.scrollInterval = parseInt(config.ScrollInterval);
+        }
+        else {
+            this.scrollInterval = parseInt(config.ScrollInterval) * 1000;
+        }
+        this.scrollIntervalUnit = config.ScrollIntervalUnit;
+        this.scrollType = parseInt(config.ScrollType);
+        this.scrollScope = config.ScrollScope === '1' ? 1 /* ScrollScope.Local */ : 0 /* ScrollScope.AllDocument */;
+        if (this.scrollScope === 1 /* ScrollScope.Local */) {
+            this.scrollXPath = this.getElementXPath(config.ScrollXPath);
+        }
+        this.IfStopScroll = config.IfStopScroll === 'true';
+        this.checkedJumpLoopClick = config.CheckedJumpLoopClick !== 'false';
+        this.disabledScrollAutoRemoveDuplication = config.DisabledScrollAutoRemoveDuplication === 'true';
+    }
+}
+exports.default = LoopConfig;

package/node_modules/@octopus/engine/dist/configs/LoopStartConfig.d.ts

@@ -0,0 +1,4 @@
+import LoopConfig from './LoopConfig';
+export default class LoopStartConfig extends LoopConfig {
+    constructor(config: any);
+}

package/node_modules/@octopus/engine/dist/configs/LoopStartConfig.js

@@ -0,0 +1,12 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const LoopConfig_1 = __importDefault(require("./LoopConfig"));
+class LoopStartConfig extends LoopConfig_1.default {
+    constructor(config) {
+        super(config);
+    }
+}
+exports.default = LoopStartConfig;

package/node_modules/@octopus/engine/dist/configs/MouseOverConfig.d.ts

@@ -0,0 +1,8 @@
+import { ActionItem } from '../models/actionItem';
+import BaseConfig from './BaseConfig';
+export default class MouseOverConfig extends BaseConfig {
+    elementXPath?: ActionItem;
+    ajaxLoad: boolean;
+    ajaxTimeout: number;
+    constructor(config: any);
+}

package/node_modules/@octopus/engine/dist/configs/MouseOverConfig.js

@@ -0,0 +1,15 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const BaseConfig_1 = __importDefault(require("./BaseConfig"));
+class MouseOverConfig extends BaseConfig_1.default {
+    constructor(config) {
+        super(config);
+        this.ajaxLoad = config.AjaxLoad == 'true';
+        this.ajaxTimeout = parseInt(config.AjaxTimeout);
+        this.elementXPath = this.getElementXPath(config.ElementXPath);
+    }
+}
+exports.default = MouseOverConfig;

package/node_modules/@octopus/engine/dist/configs/NavigateConfig.d.ts

@@ -0,0 +1,41 @@
+import { ScrollType, ScrollScope, NavigateType } from '../enums';
+import { ActionItem } from '../models/actionItem';
+import { CustomizeCookie } from '../models/customizeCookie';
+import { HttpHeader } from '../models/httpHeader';
+import { RetryCondition } from '../models/retryCondition';
+import BaseConfig from './BaseConfig';
+export default class NavigateConfig extends BaseConfig {
+    url: string;
+    timeOut: number;
+    scrollDown: boolean;
+    scrollTime: number;
+    scrollInterval: number;
+    scrollIntervalUnit?: string;
+    scrollType: ScrollType;
+    scrollXPath?: ActionItem;
+    scrollScope?: ScrollScope;
+    IfStopScroll: boolean;
+    useCustomizeCookie: boolean;
+    customizeCookies: CustomizeCookie[];
+    retryConditions: RetryCondition[];
+    maxRetry: number;
+    enableRetry: boolean;
+    retryInterval: number;
+    enableSwitchIp: boolean;
+    enableSwitchUserAgent: boolean;
+    clearCache: boolean;
+    autoRetry: boolean;
+    textContain?: string;
+    urlContain?: string;
+    textNotContain?: string;
+    iPType: number;
+    /** json 请求体内容 */
+    requestBodyContent: string;
+    /** json 请求方法 */
+    requestMethod: string;
+    /** json 请求头部 */
+    requestHeaders: HttpHeader[];
+    navigateType: NavigateType;
+    userAgents: Array<string>;
+    constructor(config: any, extensionElements: any);
+}

package/node_modules/@octopus/engine/dist/configs/NavigateConfig.js

@@ -0,0 +1,121 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const utils_1 = require("../utils");
+const customizeCookie_1 = require("../models/customizeCookie");
+const httpHeader_1 = require("../models/httpHeader");
+const retryCondition_1 = require("../models/retryCondition");
+const BaseConfig_1 = __importDefault(require("./BaseConfig"));
+const utils_2 = require("../utils");
+// import xml2js from 'xml2js'
+// const parser = new xml2js.Parser({ explicitArray: false, ignoreAttrs: true })
+class NavigateConfig extends BaseConfig_1.default {
+    constructor(config, extensionElements) {
+        super(config);
+        this.timeOut = 120;
+        /** json 请求体内容 */
+        this.requestBodyContent = '';
+        /** json 请求方法 */
+        this.requestMethod = 'GET';
+        /** json 请求头部 */
+        this.requestHeaders = [];
+        this.navigateType = "None" /* NavigateType.None */;
+        this.userAgents = [];
+        this.url = (0, utils_2.repairURL)(config.URL);
+        this.timeOut = parseInt(config.TimeOut);
+        // 读取滚动配置
+        this.scrollDown = config.ScrollDown == 'true';
+        this.scrollTime = parseInt(config.ScrollTime);
+        this.IfStopScroll = config.IfStopScroll === 'true';
+        if (config.ScrollIntervalUnit == 'ms') {
+            this.scrollInterval = parseInt(config.ScrollInterval);
+        }
+        else {
+            this.scrollInterval = parseInt(config.ScrollInterval) * 1000;
+        }
+        this.scrollIntervalUnit = config.ScrollIntervalUnit;
+        this.scrollType = parseInt(config.ScrollType);
+        this.scrollScope = config.ScrollScope === '1' ? 1 /* ScrollScope.Local */ : 0 /* ScrollScope.AllDocument */;
+        if (this.scrollScope === 1 /* ScrollScope.Local */) {
+            this.scrollXPath = this.getElementXPath(config.ScrollXPath);
+        }
+        // cookies
+        this.useCustomizeCookie = config.UseCustomizeCookie == 'true';
+        if (this.useCustomizeCookie && config.CustomizeCookie) {
+            this.customizeCookies = [];
+            (0, utils_1.parseXML)(config.CustomizeCookie, (err, result) => {
+                if (err)
+                    return;
+                const parseCookie = (cookie) => {
+                    const customizeCookie = new customizeCookie_1.CustomizeCookie();
+                    customizeCookie.creationTime = parseInt(cookie.CreationTime);
+                    customizeCookie.expiry = parseInt(cookie.Expiry);
+                    customizeCookie.host = cookie.Host;
+                    customizeCookie.isDomain = cookie.IsDomain == 'true';
+                    customizeCookie.isHttpOnly = cookie.IsHttpOnly == 'true';
+                    customizeCookie.isSecure = cookie.IsSecure == 'true';
+                    customizeCookie.isSession = cookie.IsSession == 'true';
+                    customizeCookie.lastAccessed = cookie.LastAccessed;
+                    customizeCookie.name = cookie.Name;
+                    customizeCookie.path = cookie.Path;
+                    customizeCookie.rawHost = cookie.RawHost;
+                    customizeCookie.value = cookie.Value;
+                    return customizeCookie;
+                };
+                const cookies = result.ArrayOfCustomizeCookie.CustomizeCookie;
+                if (Array.isArray(cookies)) {
+                    for (const cookie of cookies) {
+                        const customizeCookie = parseCookie(cookie);
+                        this.customizeCookies.push(customizeCookie);
+                    }
+                }
+                else {
+                    const customizeCookie = parseCookie(cookies);
+                    this.customizeCookies.push(customizeCookie);
+                }
+            });
+        }
+        // 读取重试配置
+        this.maxRetry = parseInt(config.MaxRetry);
+        this.enableRetry = config.EnableRetry == 'true';
+        this.retryInterval = parseInt(config.RetryInterval);
+        this.enableSwitchIp = config.EnableSwitchIp == 'true';
+        this.enableSwitchUserAgent = config.EnableSwitchUserAgent == 'true';
+        this.clearCache = config.ClearCache == 'true';
+        this.iPType = parseInt(config.IPType);
+        if (this.enableRetry) {
+            this.retryConditions = new Array();
+            this.getEntityFromExtensionElements("xml:NavigateAction.RetryConditions" /* ActionTags.NavigateRetryConditions */, extensionElements, (result) => {
+                const rc = new retryCondition_1.RetryCondition();
+                rc.conditionType = result.ConditionType;
+                rc.containsOrNot = result.ContainsOrNot == 'true';
+                rc.mainValue = result.MainValue;
+                rc.minorValue = result.MinorValue;
+                this.retryConditions[this.retryConditions.length] = rc;
+            });
+            this.getEntityFromExtensionElements("xml:NavigateAction.UserAgents" /* ActionTags.NavigateRetryUserAgent */, extensionElements, (result) => {
+                this.userAgents[this.userAgents.length] = result.userAgent;
+            });
+        }
+        this.textContain = config.TextContain;
+        this.urlContain = config.UrlContain;
+        this.textNotContain = config.TextNotContain;
+        this.autoRetry = config.AutoRetry == 'true';
+        this.navigateType = config.NavigateType;
+        if (!this.navigateType) {
+            this.navigateType = "OpenWebpage" /* NavigateType.OpenWebpage */;
+        }
+        this.requestBodyContent = config.RequestBodyContent;
+        this.requestMethod = config.RequestMethod;
+        this.requestHeaders = [];
+        this.getEntityFromExtensionElements("xml:NavigateAction.RequestHeaders" /* ActionTags.NavigateRequestHeaders */, extensionElements, (result) => {
+            const header = new httpHeader_1.HttpHeader();
+            header.name = result.Name;
+            header.value = result.Value;
+            this.requestHeaders.push(header);
+        });
+    }
+}
+exports.default = NavigateConfig;

package/node_modules/@octopus/engine/dist/configs/SwitchComboConfig.d.ts

@@ -0,0 +1,8 @@
+import { ActionItem } from '../models/actionItem';
+import BaseConfig from './BaseConfig';
+export default class SwitchComboConfig extends BaseConfig {
+    elementXPath?: ActionItem;
+    ajaxLoad: boolean;
+    ajaxTimeout: number;
+    constructor(config: any);
+}

package/node_modules/@octopus/engine/dist/configs/SwitchComboConfig.js

@@ -0,0 +1,15 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const BaseConfig_1 = __importDefault(require("./BaseConfig"));
+class SwitchComboConfig extends BaseConfig_1.default {
+    constructor(config) {
+        super(config);
+        this.ajaxLoad = config.AjaxLoad == 'true';
+        this.ajaxTimeout = parseInt(config.AjaxTimeout);
+        this.elementXPath = this.getElementXPath(config.ElementXPath);
+    }
+}
+exports.default = SwitchComboConfig;