octoparse-cli 0.1.14

package/node_modules/@octopus/engine/dist/actions/BaseAction.d.ts

@@ -0,0 +1,339 @@
+import { EventEmitter } from 'events';
+import { BrowserProxy } from '../browserProxy';
+import BaseConfig from '../configs/BaseConfig';
+import { ActionType, FuncRetryType, LocalCaptchaType, NavigateType } from '../enums';
+import { ActionItem } from '../models/actionItem';
+import { ConditionCheckArgs } from '../models/conditionCheckArgs';
+import { RetryCondition } from '../models/retryCondition';
+import { ClickCaptchaResult, SlideCaptchaResult } from '@/solvers/captcha/types';
+import type { ProxyMessage, Variables } from '../types';
+import type { Page, Target } from 'puppeteer-core';
+export interface CaptchaParams {
+    url: string;
+    captchaType: LocalCaptchaType;
+    image?: string;
+    key?: string;
+    action?: string;
+}
+export default class BaseAction {
+    protected browserProxy: BrowserProxy;
+    protected activity: any;
+    protected execution: any;
+    protected variables: Variables;
+    parent?: BaseAction;
+    config: BaseConfig;
+    actionTarget: Target;
+    /** 单个步骤最长执行时间，超过默认最大时间自动结束当前步骤 (毫秒) */
+    executeTimeMax: number;
+    /** 步骤类型 */
+    type: ActionType;
+    protected navigateType: NavigateType;
+    get id(): any;
+    /** 步骤名 */
+    get name(): string;
+    get isOpenProxyIp(): boolean;
+    get proxyIpSwitchPeriod(): number;
+    event: EventEmitter;
+    constructor(browserProxy: BrowserProxy, activity: any, execution: any, parent?: BaseAction);
+    emit(eventName: string, ...args: any[]): void;
+    on(eventName: any, listener: any): void;
+    once(eventName: any, listener: any): void;
+    log(key: string, ...args: any[]): void;
+    error(key: string, ...args: any[]): void;
+    resource(key: string, ...args: any[]): void;
+    debug(key: string, ...args: any[]): void;
+    removeListeners(): void;
+    checkPause(): Promise<void>;
+    screenshot(): Promise<Uint8Array<ArrayBufferLike> | Buffer<ArrayBuffer>>;
+    getCurrentPageUrl(): Promise<string>;
+    waitForTarget(timeout: number, options?: {
+        signal?: AbortSignal;
+    }): Promise<boolean>;
+    private setProxy;
+    protected collectProxyLog(reason?: string, success?: boolean): Promise<void>;
+    protected get shouldWait(): boolean;
+    protected set shouldWait(flag: boolean);
+    /** 当前可见页面 */
+    protected getCurrentPage(): Promise<Page>;
+    private waitForDocumentReady;
+    private waitForReady;
+    protected getContext(): import("puppeteer-core").BrowserContext;
+    getProxyIp(): Promise<ProxyMessage>;
+    /** 解析步骤配置 */
+    protected parseConfig(config: any): BaseConfig;
+    /** 首次解析成功后缓存 Page → tabId，后续无需再查 URL */
+    private tabIdCache;
+    /** 是否应优先使用插件桥接执行操作 */
+    protected get useExtensionBridge(): boolean;
+    /**
+     * 获取当前页面对应的 Extension tabId
+     * 首次通过 URL 解析，之后从 WeakMap 缓存读取
+     */
+    protected getTabId(): Promise<number | undefined>;
+    protected getTabIdForPage(page?: Page): Promise<number | undefined>;
+    private runBridgeReadyCheck;
+    protected bridgeElementExists(page: Page, actionItem: ActionItem, timeoutMs?: number): Promise<boolean | undefined>;
+    locator(page: Page, actionItem: ActionItem): Promise<import("puppeteer-core").Locator<Element>>;
+    private getBridgeScrollMode;
+    private getScrollLogKey;
+    private buildScrollDebugContext;
+    private normalizeBridgeScrollReason;
+    /** 休眠指定时间 */
+    protected sleep(delay?: number): Promise<void>;
+    /** 结束当前步骤 */
+    protected finish(): void;
+    /** 切换代理 IP */
+    protected switchIP(page: Page): Promise<void>;
+    /** 切换浏览器 UA */
+    protected switchUA(): void;
+    /** 等待当前网页还在加载的请求完成，每隔 100ms 判定一次，最多判定 100 次，最长等待 10s */
+    protected waitRequest(): Promise<void>;
+    /** 等待 Ajax 请求完成 */
+    ajax(): Promise<void>;
+    /** 执行滚动网页 */
+    protected scroll(): Promise<void>;
+    onPageLoading(url: string): void;
+    onPageLoaded(url: string, isTimeout?: boolean): Promise<void>;
+    onPageFailLoad(url: string): void;
+    onJsonResponse(url: string): Promise<void>;
+    onJsonError(url: string): void;
+    enter(): void;
+    start(): void;
+    wait(skipUseLoop?: boolean): Promise<void>;
+    waitAfter(): Promise<void>;
+    /** 根据验证码类型获取超时时间（毫秒） */
+    private getCaptchaTimeout;
+    getCaptchaToken(data: {
+        url?: string;
+        captchaType: LocalCaptchaType;
+        image?: string;
+        key?: string;
+        action?: string;
+    }): Promise<string | SlideCaptchaResult | ClickCaptchaResult>;
+    end(): void;
+    leave(): void;
+    executeCaptcha(data: {
+        token: string;
+        captchaType: LocalCaptchaType;
+    }): Promise<void>;
+    /** @todo 以下是旧的代码直接迁移过来，待重构 */
+    protected getCurrentLoopValue(): any;
+    /**
+     * 获取指定循环的当前值
+     * @param nodeId 这个属性的名称
+     */
+    protected getLoopValue(nodeId: string): any;
+    /**
+     * 获取设置当前loop的url
+     * @param nodeId
+     * @param val
+     */
+    protected setCurrentLoopUrl(nodeId: string, val: any): void;
+    /**
+     * 获取当前loop所在的url
+     * @param nodeId
+     * @param val
+     */
+    protected getCurrentLoopUrl(nodeId: string): any;
+    /**
+     * 设置指定循环的当前值
+     * @param nodeId
+     * @param val
+     */
+    protected setCurrentLoopValue(nodeId: string, val: any): void;
+    /**
+     * 从循环中取出一个对象
+     * @param nodeId
+     */
+    protected popLoopValue(nodeId: string): any;
+    /**
+     * 设置该循环数组的值
+     * @param nodeId
+     * @param dic
+     */
+    protected setLoopValues(nodeId: string, dic: any[]): void;
+    /**
+     * 获取指定循环队列中，已推入的所有内容
+     * @param nodeId 指定循环的节点 id
+     */
+    protected getLoopValueArray(nodeId: string): any;
+    protected pushLoopValue(nodeId: string, val: any): void;
+    protected saveSampedLoopRecord(info: any): void;
+    protected initSampedLoopRecord(): void;
+    protected getSampedLoopRecord(): any;
+    /**
+     * 统计指定循环列表提取数据步骤，当前已导出的数据条目
+     * @param nodeId
+     */
+    protected countSomeLoopExtractDataNum(nodeId: string): void;
+    /**
+     * 初始化指定循环列表提取数据步骤，当前已导出的数据条目为0
+     * @param nodeId
+     */
+    protected initSomeLoopExtractDataCount(nodeId: string): void;
+    /**
+     * 获取指定循环列表提取数据步骤，当前已导出的数据条目
+     * @param nodeId
+     */
+    protected getSomeLoopExtractDataCount(nodeId: string): number;
+    /**
+     * 设置指定循环列表提取数据步骤，存储的历史采样数据
+     * @param nodeId
+     * @param val
+     */
+    protected setSamplingElementsMap(nodeId: string, val: any): void;
+    /**
+     * 读取指定循环列表提取数据步骤，存储的历史采样数据
+     * @param nodeId
+     * @param val
+     */
+    protected getSamplingElementsMap(nodeId: string): Map<number, string>;
+    /**
+     * 初始化指定循环列表提取数据步骤，存储的历史采样数据
+     * @param nodeId
+     * @param val
+     */
+    protected initSamplingElementsMap(nodeId: string): void;
+    /**
+     * 初始化指定循环列表提取数据步骤，当轮循环提取数据步骤完成后，已采集到的数据条目，用于生成特定的Xpath偏移量
+     * @param nodeId
+     * @param val
+     */
+    protected initIdentifiedEachLoopExtractDataCount(nodeId: string): void;
+    /**
+     * 设置指定循环列表提取数据步骤，当轮循环提取数据步骤完成后，已采集到的数据条目，用于生成特定的Xpath偏移量
+     * @param nodeId
+     * @param val
+     */
+    protected setIdentifiedEachLoopExtractDataCount(nodeId: string): void;
+    /**
+     * 读取指定循环列表提取数据步骤，当轮循环提取数据步骤完成后，已采集到的数据条目，用于生成特定的Xpath偏移量
+     * @param nodeId
+     * @param val
+     */
+    protected getIdentifiedEachLoopExtractDataCount(nodeId: string): number;
+    protected setNextLoopOffset(nodeId: string, val: number): void;
+    protected getNextLoopOffset(nodeId: string): number;
+    protected resetNextLoopOffset(nodeId: string): void;
+    /**
+     * 初始化循环次数
+     * @param nodeId 节点id
+     */
+    protected initLoopCount(nodeId: string): void;
+    /**
+     * 累加循环次数
+     * @param nodeId 节点id
+     */
+    protected addLoopCount(nodeId: string): void;
+    /**
+     * 获取循环次数
+     * @param nodeId 节点id
+     */
+    protected getLoopCount(nodeId: string): number;
+    /**
+     * 初始化连续无效翻页计数
+     * @param nodeId 节点id
+     */
+    protected initContinuousInvalidPageCount(nodeId: string): void;
+    /**
+     * 添加连续无效翻页计数
+     * @param nodeId 节点id
+     */
+    protected addContinuousInvalidPageCount(nodeId: string): void;
+    /**
+     * 获取连续无效翻页计数
+     * @param nodeId 节点id
+     */
+    protected getContinuousInvalidPageCount(nodeId: string): number;
+    /**
+     * 初始化，连续循环点击加载更多按钮，一直没有新数据的循环次数
+     * @param nodeId 节点id
+     */
+    protected initContinuousClickLoadNoNewDataCount(nodeId: string): void;
+    /**
+     * 累加，连续循环点击加载更多按钮，一直没有新数据的循环次数
+     * @param nodeId 节点id
+     */
+    protected addContinuousClickLoadNoNewDataCount(nodeId: string): void;
+    /**
+     * 获取，连续循环点击加载更多按钮，一直没有新数据的循环次数
+     * @param nodeId 节点id
+     */
+    protected getContinuousClickLoadNoNewDataCount(nodeId: string): number;
+    /**
+     * 初始化，连续滚动网页，滚动条高度一只没有变化的循环次数
+     * @param nodeId 节点id
+     */
+    protected initContinuousScrollTopFixedWebCount(nodeId: string): void;
+    /**
+     * 累加，连续滚动网页，滚动条高度一只没有变化的循环次数
+     * @param nodeId 节点id
+     */
+    protected addContinuousScrollTopFixedWebCount(nodeId: string, num?: number): void;
+    /**
+     * 获取，连续滚动网页，滚动条高度一只没有变化的循环次数
+     * @param nodeId 节点id
+     */
+    protected getContinuousScrollTopFixedWebCount(nodeId: string): number;
+    /**
+     * 缓存网页滚动条距离顶部的高度
+     * @param nodeId 节点id
+     * @param top 网页滚动条距离顶部的高度
+     */
+    protected setWebScrollTopTemp(nodeId: string, top: number): void;
+    /**
+     * 获取网页滚动条距离顶部的高度
+     * @param nodeId 节点id
+     */
+    protected getWebScrollTopTemp(nodeId: string): number;
+    /**
+     *
+     * @param nodeId
+     */
+    protected setWebDocumentHeightTemp(nodeId: string, height: number): void;
+    /**
+     * 获取到网页高度
+     * @param nodeId 节点id
+     */
+    protected getWebDocumentHeightTemp(nodeId: string): number;
+    /**
+     * 设置当前循环是否需要跳过
+     * @param nodeId
+     * @param isSkip
+     * @param isOnlySkipOneLoop
+     * @param isClickNotFound
+     */
+    protected setIsSkipCurrentLoop(nodeId: string, isSkip: boolean, isOnlySkipOneLoop?: boolean, isClickNotFound?: boolean): void;
+    protected getIsClickNotFound(): boolean;
+    protected deleteIsSkipCurrentLoop(nodeId: string): void;
+    protected setIsSkipActionUtilNextLoop(nodeId: string, isSkip?: boolean): void;
+    protected getIsSkipActionUtilNextLoop(nodeId: string): any;
+    protected getIsSkiptLoop(nodeId: string): boolean;
+    protected getIsSkipCurrentLoop(): boolean;
+    protected getIsOnlySkipOneLoop(): boolean;
+    protected resetIsOnlySkipOneLoop(): void;
+    protected getNodeId(): string;
+    /**
+     * 加载新页面后设置值（非ajax），为了后面为加载超时做判断依据
+     * @param type 当前的状态
+     */
+    protected setRetryTypeState(type: FuncRetryType): void;
+    /**
+     * 获取是否加载的状态，获取后删除存储的状态
+     */
+    protected popIsLoadedState(): FuncRetryType;
+    /**
+     * @param waitItemXpath 等待的xpath
+     * @param waitTimeMS 等待的时间 默认20秒
+     */
+    protected setPageActionWaitXpathTime(waitItemXpath: string, waitTimeMS?: number): void;
+    protected getPageActionWaitXpathTime(waitItemXpath: string): void;
+    private loopXpathStrategyMap;
+    protected getXpathStrategy(key: string): number;
+    protected setXpathStratey(key: string, xpathItem: number): void;
+    protected awaitElement(elementXPath: ActionItem): Promise<boolean>;
+    protected conditionCheck(e: ConditionCheckArgs, page: Page, skip?: boolean): Promise<boolean>;
+    protected extractElementData(actionItem: ActionItem, extractFn: (ele: HTMLElement) => string, page: Page): Promise<any>;
+    protected checkPageContains({ conditionType, containsOrNot, mainValue, minorValue }: RetryCondition, page: Page): Promise<boolean>;
+    protected checkJSONContains(retryCondition: RetryCondition): boolean;
+}