@rpascene/core 0.30.8

package/dist/types/image/index.d.ts

@@ -0,0 +1 @@
+export { imageInfo, imageInfoOfBase64, localImg2Base64, httpImg2Base64, resizeAndConvertImgBuffer, saveBase64Image, zoomForGPT4o, } from '@rpascene/shared/img';

package/dist/types/index.d.ts

@@ -0,0 +1,12 @@
+import { z } from 'zod';
+import { Executor } from './ai-model/action-executor';
+import Insight from './insight/index';
+import { getVersion } from './utils';
+export { plan, describeUserPage, AiLocateElement, getRpasceneLocationSchema, type RpasceneLocationResultType, PointSchema, SizeSchema, RectSchema, TMultimodalPromptSchema, TUserPromptSchema, type TMultimodalPrompt, type TUserPrompt, } from './ai-model/index';
+export { RPASCENE_MODEL_NAME } from '@rpascene/shared/env';
+export type * from './types';
+export { z };
+export default Insight;
+export { Executor, Insight, getVersion };
+export type { RpasceneYamlScript, RpasceneYamlTask, RpasceneYamlFlowItem, RpasceneYamlConfigResult, RpasceneYamlConfig, RpasceneYamlScriptWebEnv, RpasceneYamlScriptAndroidEnv, RpasceneYamlScriptIOSEnv, RpasceneYamlScriptEnv, LocateOption, DetailedLocateParam, } from './yaml';
+export { Agent, type AgentOpt, createAgent } from './agent';

package/dist/types/insight/index.d.ts

@@ -0,0 +1,31 @@
+import { callAIWithObjectResponse } from '../ai-model/index';
+import type { AIDescribeElementResponse, AIUsageInfo, BaseElement, DetailedLocateParam, DumpSubscriber, InsightAction, InsightExtractOption, InsightExtractParam, InsightTaskInfo, LocateResult, Rect, UIContext } from '../types';
+import { type IModelConfig } from '@rpascene/shared/env';
+import type { TMultimodalPrompt } from '../ai-model/common';
+export interface LocateOpts {
+    context?: UIContext<BaseElement>;
+}
+export type AnyValue<T> = {
+    [K in keyof T]: unknown extends T[K] ? any : T[K];
+};
+interface InsightOptions {
+    taskInfo?: Omit<InsightTaskInfo, 'durationMs'>;
+    aiVendorFn?: typeof callAIWithObjectResponse;
+}
+export default class Insight<ElementType extends BaseElement = BaseElement, ContextType extends UIContext<ElementType> = UIContext<ElementType>> {
+    contextRetrieverFn: (action: InsightAction) => Promise<ContextType> | ContextType;
+    aiVendorFn: Exclude<InsightOptions['aiVendorFn'], undefined>;
+    onceDumpUpdatedFn?: DumpSubscriber;
+    taskInfo?: Omit<InsightTaskInfo, 'durationMs'>;
+    constructor(context: ContextType | ((action: InsightAction) => Promise<ContextType> | ContextType), opt?: InsightOptions);
+    locate(query: DetailedLocateParam, opt: LocateOpts, modelConfig: IModelConfig): Promise<LocateResult>;
+    extract<T>(dataDemand: InsightExtractParam, modelConfig: IModelConfig, opt?: InsightExtractOption, multimodalPrompt?: TMultimodalPrompt): Promise<{
+        data: T;
+        thought?: string;
+        usage?: AIUsageInfo;
+    }>;
+    describe(target: Rect | [number, number], modelConfig: IModelConfig, opt?: {
+        deepThink?: boolean;
+    }): Promise<Pick<AIDescribeElementResponse, 'description'>>;
+}
+export {};

package/dist/types/insight/utils.d.ts

@@ -0,0 +1,2 @@
+import type { DumpSubscriber, PartialInsightDumpFromSDK } from '../types';
+export declare function emitInsightDump(data: PartialInsightDumpFromSDK, dumpSubscriber?: DumpSubscriber): void;

package/dist/types/report.d.ts

@@ -0,0 +1,12 @@
+import type { ReportFileWithAttributes } from './types';
+export declare class ReportMergingTool {
+    private reportInfos;
+    append(reportInfo: ReportFileWithAttributes): void;
+    clear(): void;
+    private extractScriptContent;
+    mergeReports(reportFileName?: 'AUTO' | string, // user custom report filename, save into rpascene report dir if undefined
+    opts?: {
+        rmOriginalReports?: boolean;
+        overwrite?: boolean;
+    }): string | null;
+}

package/dist/types/tree.d.ts

@@ -0,0 +1 @@
+export { truncateText, trimAttributes, descriptionOfTree, } from '@rpascene/shared/extractor';

package/dist/types/types.d.ts

@@ -0,0 +1,414 @@
+import type { NodeType } from '@rpascene/shared/constants';
+import type { TModelConfigFn } from '@rpascene/shared/env';
+import type { BaseElement, ElementTreeNode, Rect, Size } from '@rpascene/shared/types';
+import type { z } from 'zod';
+import type { TUserPrompt } from './ai-model/common';
+import type { DetailedLocateParam, RpasceneYamlFlowItem } from './yaml';
+export type { ElementTreeNode, BaseElement, Rect, Size, Point, } from '@rpascene/shared/types';
+export * from './yaml';
+export type AIUsageInfo = Record<string, any> & {
+    prompt_tokens: number | undefined;
+    completion_tokens: number | undefined;
+    total_tokens: number | undefined;
+    time_cost: number | undefined;
+    model_name: string | undefined;
+    model_description: string | undefined;
+    intent: string | undefined;
+};
+/**
+ * openai
+ *
+ */
+export declare enum AIResponseFormat {
+    JSON = "json_object",
+    TEXT = "text"
+}
+export type AISingleElementResponseById = {
+    id: string;
+    reason?: string;
+    text?: string;
+    xpaths?: string[];
+};
+export type AISingleElementResponseByPosition = {
+    position?: {
+        x: number;
+        y: number;
+    };
+    bbox?: [number, number, number, number];
+    reason: string;
+    text: string;
+};
+export type AISingleElementResponse = AISingleElementResponseById;
+export interface AIElementLocatorResponse {
+    elements: {
+        id: string;
+        reason?: string;
+        text?: string;
+        xpaths?: string[];
+    }[];
+    bbox?: [number, number, number, number];
+    isOrderSensitive?: boolean;
+    errors?: string[];
+}
+export interface AIElementCoordinatesResponse {
+    bbox: [number, number, number, number];
+    isOrderSensitive?: boolean;
+    errors?: string[];
+}
+export type AIElementResponse = AIElementLocatorResponse | AIElementCoordinatesResponse;
+export interface AIDataExtractionResponse<DataDemand> {
+    data: DataDemand;
+    errors?: string[];
+    thought?: string;
+}
+export interface AISectionLocatorResponse {
+    bbox: [number, number, number, number];
+    references_bbox?: [number, number, number, number][];
+    error?: string;
+}
+export interface AIAssertionResponse {
+    pass: boolean;
+    thought: string;
+}
+export interface AIDescribeElementResponse {
+    description: string;
+    error?: string;
+}
+export interface LocatorValidatorOption {
+    centerDistanceThreshold?: number;
+}
+export interface LocateValidatorResult {
+    pass: boolean;
+    rect: Rect;
+    center: [number, number];
+    centerDistance?: number;
+}
+export interface AgentDescribeElementAtPointResult {
+    prompt: string;
+    deepThink: boolean;
+    verifyResult?: LocateValidatorResult;
+}
+/**
+ * context
+ */
+export declare abstract class UIContext<ElementType extends BaseElement = BaseElement> {
+    abstract screenshotBase64: string;
+    abstract tree: ElementTreeNode<ElementType>;
+    abstract size: Size;
+    abstract _isFrozen?: boolean;
+}
+export type EnsureObject<T> = {
+    [K in keyof T]: any;
+};
+export type InsightAction = 'locate' | 'extract' | 'assert' | 'describe';
+export type InsightExtractParam = string | Record<string, string>;
+export type ElementCacheFeature = Record<string, unknown>;
+export type LocateResultElement = {
+    center: [number, number];
+    rect: Rect;
+    id: string;
+    indexId?: number;
+    xpaths: string[];
+    attributes: {
+        nodeType: NodeType;
+        [key: string]: string;
+    };
+    isOrderSensitive?: boolean;
+    containerPaths?: any[];
+    allPaths?: any[];
+};
+export interface LocateResult {
+    element: LocateResultElement | null;
+    rect?: Rect;
+}
+export interface InsightTaskInfo {
+    durationMs: number;
+    formatResponse?: string;
+    rawResponse?: string;
+    usage?: AIUsageInfo;
+    searchArea?: Rect;
+    searchAreaRawResponse?: string;
+    searchAreaUsage?: AIUsageInfo;
+}
+export interface DumpMeta {
+    logTime: number;
+}
+export interface ReportDumpWithAttributes {
+    dumpString: string;
+    attributes?: Record<string, any>;
+}
+export interface InsightDump extends DumpMeta {
+    type: 'locate' | 'extract' | 'assert';
+    logId: string;
+    userQuery: {
+        element?: TUserPrompt;
+        dataDemand?: InsightExtractParam;
+        assertion?: TUserPrompt;
+    };
+    matchedElement: BaseElement[];
+    matchedRect?: Rect;
+    deepThink?: boolean;
+    data: any;
+    assertionPass?: boolean;
+    assertionThought?: string;
+    taskInfo: InsightTaskInfo;
+    error?: string;
+    output?: any;
+}
+export type PartialInsightDumpFromSDK = Omit<InsightDump, 'logTime' | 'logId' | 'model_name'>;
+export type DumpSubscriber = (dump: InsightDump) => Promise<void> | void;
+export interface LiteUISection {
+    name: string;
+    description: string;
+    sectionCharacteristics: string;
+    textIds: string[];
+}
+export type ElementById = (id: string) => BaseElement | null;
+export type InsightAssertionResponse = AIAssertionResponse & {
+    usage?: AIUsageInfo;
+};
+/**
+ * agent
+ */
+export type OnTaskStartTip = (tip: string) => Promise<void> | void;
+export interface AgentWaitForOpt {
+    checkIntervalMs?: number;
+    timeoutMs?: number;
+}
+export interface AgentAssertOpt {
+    keepRawResponse?: boolean;
+}
+/**
+ * planning
+ *
+ */
+export interface PlanningLocateParam extends DetailedLocateParam {
+    id?: string;
+    bbox?: [number, number, number, number];
+}
+export interface PlanningAction<ParamType = any> {
+    thought?: string;
+    type: string;
+    param: ParamType;
+    locate?: PlanningLocateParam | null;
+}
+export interface PlanningAIResponse {
+    action?: PlanningAction;
+    actions?: PlanningAction[];
+    more_actions_needed_by_instruction: boolean;
+    log: string;
+    sleep?: number;
+    error?: string;
+    usage?: AIUsageInfo;
+    rawResponse?: string;
+    yamlFlow?: RpasceneYamlFlowItem[];
+    yamlString?: string;
+}
+export type PlanningActionParamTap = null;
+export type PlanningActionParamHover = null;
+export type PlanningActionParamRightClick = null;
+export interface PlanningActionParamInputOrKeyPress {
+    value: string;
+    autoDismissKeyboard?: boolean;
+}
+export interface PlanningActionParamSleep {
+    timeMs: number;
+}
+export interface PlanningActionParamError {
+    thought: string;
+}
+export type PlanningActionParamWaitFor = AgentWaitForOpt & {};
+export interface AndroidLongPressParam {
+    duration?: number;
+}
+export interface AndroidPullParam {
+    direction: 'up' | 'down';
+    distance?: number;
+    duration?: number;
+}
+/**
+ * misc
+ */
+export interface Color {
+    name: string;
+    hex: string;
+}
+export interface BaseAgentParserOpt {
+    selector?: string;
+}
+export interface PuppeteerParserOpt extends BaseAgentParserOpt {
+}
+export interface PlaywrightParserOpt extends BaseAgentParserOpt {
+}
+export interface ExecutionTaskProgressOptions {
+    onTaskStart?: (task: ExecutionTask) => Promise<void> | void;
+}
+export interface ExecutionRecorderItem {
+    type: 'screenshot';
+    ts: number;
+    screenshot?: string;
+    timing?: string;
+}
+export type ExecutionTaskType = 'Planning' | 'Insight' | 'Action' | 'Assertion' | 'Log';
+export interface ExecutorContext {
+    task: ExecutionTask;
+    element?: LocateResultElement | null;
+}
+export interface ExecutionTaskApply<Type extends ExecutionTaskType = any, TaskParam = any, TaskOutput = any, TaskLog = any> {
+    type: Type;
+    subType?: string;
+    param?: TaskParam;
+    thought?: string;
+    locate?: PlanningLocateParam | null;
+    uiContext?: UIContext;
+    executor: (param: TaskParam, context: ExecutorContext) => Promise<ExecutionTaskReturn<TaskOutput, TaskLog> | undefined | void> | undefined | void;
+}
+export interface ExecutionTaskHitBy {
+    from: string;
+    context: Record<string, any>;
+}
+export interface ExecutionTaskReturn<TaskOutput = unknown, TaskLog = unknown> {
+    output?: TaskOutput;
+    log?: TaskLog;
+    recorder?: ExecutionRecorderItem[];
+    hitBy?: ExecutionTaskHitBy;
+}
+export type ExecutionTask<E extends ExecutionTaskApply<any, any, any> = ExecutionTaskApply<any, any, any>> = E & ExecutionTaskReturn<E extends ExecutionTaskApply<any, any, infer TaskOutput, any> ? TaskOutput : unknown, E extends ExecutionTaskApply<any, any, any, infer TaskLog> ? TaskLog : unknown> & {
+    status: 'pending' | 'running' | 'finished' | 'failed' | 'cancelled';
+    error?: Error;
+    errorMessage?: string;
+    errorStack?: string;
+    timing?: {
+        start: number;
+        end?: number;
+        cost?: number;
+    };
+    usage?: AIUsageInfo;
+    searchAreaUsage?: AIUsageInfo;
+};
+export interface ExecutionDump extends DumpMeta {
+    name: string;
+    description?: string;
+    tasks: ExecutionTask[];
+    aiActionContext?: string;
+}
+export type ExecutionTaskInsightLocateParam = PlanningLocateParam;
+export interface ExecutionTaskInsightLocateOutput {
+    element: LocateResultElement | null;
+}
+export type ExecutionTaskInsightDump = InsightDump;
+export type ExecutionTaskInsightLocateApply = ExecutionTaskApply<'Insight', ExecutionTaskInsightLocateParam, ExecutionTaskInsightLocateOutput, ExecutionTaskInsightDump>;
+export type ExecutionTaskInsightLocate = ExecutionTask<ExecutionTaskInsightLocateApply>;
+export interface ExecutionTaskInsightQueryParam {
+    dataDemand: InsightExtractParam;
+}
+export interface ExecutionTaskInsightQueryOutput {
+    data: any;
+}
+export type ExecutionTaskInsightQueryApply = ExecutionTaskApply<'Insight', ExecutionTaskInsightQueryParam, any, ExecutionTaskInsightDump>;
+export type ExecutionTaskInsightQuery = ExecutionTask<ExecutionTaskInsightQueryApply>;
+export interface ExecutionTaskInsightAssertionParam {
+    assertion: string;
+}
+export type ExecutionTaskInsightAssertionApply = ExecutionTaskApply<'Insight', ExecutionTaskInsightAssertionParam, InsightAssertionResponse, ExecutionTaskInsightDump>;
+export type ExecutionTaskInsightAssertion = ExecutionTask<ExecutionTaskInsightAssertionApply>;
+export type ExecutionTaskActionApply<ActionParam = any> = ExecutionTaskApply<'Action', ActionParam, void, void>;
+export type ExecutionTaskAction = ExecutionTask<ExecutionTaskActionApply>;
+export type ExecutionTaskLogApply<LogParam = {
+    content: string;
+}> = ExecutionTaskApply<'Log', LogParam, void, void>;
+export type ExecutionTaskLog = ExecutionTask<ExecutionTaskLogApply>;
+export type ExecutionTaskPlanningApply = ExecutionTaskApply<'Planning', {
+    userInstruction: string;
+}, PlanningAIResponse>;
+export type ExecutionTaskPlanning = ExecutionTask<ExecutionTaskPlanningApply>;
+export interface GroupedActionDump {
+    sdkVersion: string;
+    groupName: string;
+    groupDescription?: string;
+    modelBriefs: string[];
+    executions: ExecutionDump[];
+}
+export type InterfaceType = 'puppeteer' | 'playwright' | 'static' | 'chrome-extension-proxy' | 'android' | string;
+export interface StreamingCodeGenerationOptions {
+    /** Whether to enable streaming output */
+    stream?: boolean;
+    /** Callback function to handle streaming chunks */
+    onChunk?: StreamingCallback;
+    /** Callback function to handle streaming completion */
+    onComplete?: (finalCode: string) => void;
+    /** Callback function to handle streaming errors */
+    onError?: (error: Error) => void;
+}
+export type StreamingCallback = (chunk: CodeGenerationChunk) => void;
+export interface CodeGenerationChunk {
+    /** The incremental content chunk */
+    content: string;
+    /** The reasoning content */
+    reasoning_content: string;
+    /** The accumulated content so far */
+    accumulated: string;
+    /** Whether this is the final chunk */
+    isComplete: boolean;
+    /** Token usage information if available */
+    usage?: AIUsageInfo;
+}
+export interface StreamingAIResponse {
+    /** The final accumulated content */
+    content: string;
+    /** Token usage information */
+    usage?: AIUsageInfo;
+    /** Whether the response was streamed */
+    isStreamed: boolean;
+}
+export interface DeviceAction<T = any> {
+    name: string;
+    description?: string;
+    interfaceAlias?: string;
+    paramSchema?: z.ZodType<T>;
+    call: (param: T, context: ExecutorContext) => Promise<void> | void;
+}
+/**
+ * Web-specific types
+ */
+export interface WebElementInfo extends BaseElement {
+    id: string;
+    attributes: {
+        nodeType: NodeType;
+        [key: string]: string;
+    };
+}
+export type WebUIContext = UIContext<WebElementInfo>;
+/**
+ * Agent
+ */
+export type CacheConfig = {
+    strategy?: 'read-only' | 'read-write' | 'write-only';
+    id: string;
+};
+export type Cache = false | true | CacheConfig;
+export interface AgentOpt {
+    testId?: string;
+    cacheId?: string;
+    groupName?: string;
+    groupDescription?: string;
+    generateReport?: boolean;
+    autoPrintReportMsg?: boolean;
+    onTaskStartTip?: OnTaskStartTip;
+    aiActionContext?: string;
+    reportFileName?: string;
+    modelConfig?: TModelConfigFn;
+    cache?: Cache;
+    replanningCycleLimit?: number;
+}
+export type TestStatus = 'passed' | 'failed' | 'timedOut' | 'skipped' | 'interrupted';
+export interface ReportFileWithAttributes {
+    reportFilePath: string;
+    reportAttributes: {
+        testDuration: number;
+        testStatus: TestStatus;
+        testTitle: string;
+        testId: string;
+        testDescription: string;
+    };
+}

package/dist/types/utils.d.ts

@@ -0,0 +1,40 @@
+import type { Cache, Rect, ReportDumpWithAttributes } from './types';
+export { appendFileSync } from 'node:fs';
+export declare const groupedActionDumpFileExt = "web-dump.json";
+/**
+ * Process cache configuration with environment variable support and backward compatibility.
+ *
+ * @param cache - The original cache configuration
+ * @param cacheId - The cache ID to use as:
+ *   1. Fallback ID when cache is true or cache object has no ID
+ *   2. Legacy cacheId when cache is undefined (requires RPASCENE_CACHE env var)
+ * @returns Processed cache configuration
+ */
+export declare function processCacheConfig(cache: Cache | undefined, cacheId: string): Cache | undefined;
+export declare function getReportTpl(): string;
+/**
+ * high performance, insert script before </html> in HTML file
+ * only truncate and append, no temporary file
+ */
+export declare function insertScriptBeforeClosingHtml(filePath: string, scriptContent: string): void;
+export declare function reportHTMLContent(dumpData: string | ReportDumpWithAttributes, reportPath?: string, appendReport?: boolean, withTpl?: boolean): string;
+export declare function writeDumpReport(fileName: string, dumpData: string | ReportDumpWithAttributes, appendReport?: boolean): string | null;
+export declare function writeLogFile(opts: {
+    fileName: string;
+    fileExt: string;
+    fileContent: string | ReportDumpWithAttributes;
+    type: 'dump' | 'cache' | 'report' | 'tmp';
+    generateReport?: boolean;
+    appendReport?: boolean;
+}): string | null;
+export declare function getTmpDir(): string | null;
+export declare function getTmpFile(fileExtWithoutDot: string): string | null;
+export declare function overlapped(container: Rect, target: Rect): boolean;
+export declare function sleep(ms: number): Promise<unknown>;
+export declare function replacerForPageObject(_key: string, value: any): any;
+export declare function stringifyDumpData(data: any, indents?: number): string;
+export declare function getVersion(): string;
+export declare function uploadTestInfoToServer({ testUrl, serverUrl, }: {
+    testUrl: string;
+    serverUrl?: string;
+}): void;

package/dist/types/yaml/builder.d.ts

@@ -0,0 +1,2 @@
+import type { RpasceneYamlScriptWebEnv, RpasceneYamlTask } from '../types';
+export declare function buildYaml(env: RpasceneYamlScriptWebEnv, tasks: RpasceneYamlTask[]): string;

package/dist/types/yaml/index.d.ts

@@ -0,0 +1,3 @@
+export * from './player';
+export * from './builder';
+export * from './utils';

package/dist/types/yaml/player.d.ts

@@ -0,0 +1,34 @@
+import type { Agent } from '../agent/agent';
+import type { FreeFn, RpasceneYamlScript, RpasceneYamlScriptEnv, ScriptPlayerStatusValue, ScriptPlayerTaskStatus } from '../types';
+export declare class ScriptPlayer<T extends RpasceneYamlScriptEnv> {
+    private script;
+    private setupAgent;
+    onTaskStatusChange?: ((taskStatus: ScriptPlayerTaskStatus) => void) | undefined;
+    currentTaskIndex?: number;
+    taskStatusList: ScriptPlayerTaskStatus[];
+    status: ScriptPlayerStatusValue;
+    reportFile?: string | null;
+    result: Record<string, any>;
+    private unnamedResultIndex;
+    output?: string | null;
+    unstableLogContent?: string | null;
+    errorInSetup?: Error;
+    private interfaceAgent;
+    agentStatusTip?: string;
+    target?: RpasceneYamlScriptEnv;
+    private actionSpace;
+    private scriptPath?;
+    constructor(script: RpasceneYamlScript, setupAgent: (platform: T) => Promise<{
+        agent: Agent;
+        freeFn: FreeFn[];
+    }>, onTaskStatusChange?: ((taskStatus: ScriptPlayerTaskStatus) => void) | undefined, scriptPath?: string);
+    private setResult;
+    private setPlayerStatus;
+    private notifyCurrentTaskStatusChange;
+    private setTaskStatus;
+    private setTaskIndex;
+    private flushResult;
+    private flushUnstableLogContent;
+    playTask(taskStatus: ScriptPlayerTaskStatus, agent: Agent): Promise<void>;
+    run(): Promise<void>;
+}

package/dist/types/yaml/utils.d.ts

@@ -0,0 +1,9 @@
+import type { TUserPrompt } from '../ai-model/common';
+import type { DetailedLocateParam, LocateOption, RpasceneYamlScript } from '../types';
+export declare function interpolateEnvVars(content: string): string;
+export declare function parseYamlScript(content: string, filePath?: string): RpasceneYamlScript;
+export declare function buildDetailedLocateParam(locatePrompt: TUserPrompt, opt?: LocateOption): DetailedLocateParam | undefined;
+export declare function buildDetailedLocateParamAndRestParams(locatePrompt: TUserPrompt, opt: LocateOption | undefined, excludeKeys?: string[]): {
+    locateParam: DetailedLocateParam | undefined;
+    restParams: Record<string, any>;
+};