@aiscene/core 1.7.0

package/dist/es/skill/index.mjs

@@ -0,0 +1,38 @@
+import { CLIError, runToolsCLI } from "@midscene/shared/cli";
+import { BaseMidsceneTools } from "@midscene/shared/mcp";
+import { Agent } from "../agent/agent.mjs";
+function _define_property(obj, key, value) {
+    if (key in obj) Object.defineProperty(obj, key, {
+        value: value,
+        enumerable: true,
+        configurable: true,
+        writable: true
+    });
+    else obj[key] = value;
+    return obj;
+}
+class SkillMidsceneTools extends BaseMidsceneTools {
+    createTemporaryDevice() {
+        return new this.DeviceClass();
+    }
+    async ensureAgent() {
+        if (!this.agent) {
+            const device = new this.DeviceClass();
+            this.agent = new Agent(device);
+        }
+        return this.agent;
+    }
+    constructor(DeviceClass){
+        super(), _define_property(this, "DeviceClass", void 0), this.DeviceClass = DeviceClass;
+    }
+}
+function runSkillCLI(options) {
+    const tools = new SkillMidsceneTools(options.DeviceClass);
+    return runToolsCLI(tools, options.scriptName).catch((e)=>{
+        if (!(e instanceof CLIError)) console.error(e);
+        process.exit(e instanceof CLIError ? e.exitCode : 1);
+    });
+}
+export { runSkillCLI };
+
+//# sourceMappingURL=index.mjs.map

package/dist/es/skill/index.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"skill/index.mjs","sources":["../../../src/skill/index.ts"],"sourcesContent":["import { CLIError, runToolsCLI } from '@midscene/shared/cli';\nimport { BaseMidsceneTools } from '@midscene/shared/mcp';\nimport type { BaseAgent, BaseDevice } from '@midscene/shared/mcp';\nimport { Agent } from '../agent/agent';\nimport type { AbstractInterface } from '../device';\n\ntype DeviceClass = new (...args: any[]) => AbstractInterface;\n\n/**\n * Skill tools manager that lazily creates Agent from a Device class.\n * Used by runSkillCLI for CLI / Agent Skills scenarios where no agent exists at startup.\n */\nclass SkillMidsceneTools extends BaseMidsceneTools<BaseAgent> {\n  constructor(private DeviceClass: DeviceClass) {\n    super();\n  }\n\n  protected createTemporaryDevice(): BaseDevice {\n    return new this.DeviceClass() as unknown as BaseDevice;\n  }\n\n  protected async ensureAgent(): Promise<BaseAgent> {\n    if (!this.agent) {\n      const device = new this.DeviceClass();\n      this.agent = new Agent(device) as unknown as BaseAgent;\n    }\n    return this.agent;\n  }\n}\n\nexport interface SkillCLIOptions {\n  scriptName: string;\n  DeviceClass: DeviceClass;\n}\n\n/**\n * Launch a Skill CLI for a custom interface Device class.\n * This enables AI coding assistants (Claude Code, Cline, etc.) to control\n * your custom interface through CLI commands.\n *\n * @example\n * ```typescript\n * #!/usr/bin/env node\n * import { runSkillCLI } from '@midscene/core/skill';\n * import { SampleDevice } from './sample-device';\n *\n * runSkillCLI({\n *   DeviceClass: SampleDevice,\n *   scriptName: 'my-device',\n * });\n * ```\n */\nexport function runSkillCLI(options: SkillCLIOptions): Promise<void> {\n  const tools = new SkillMidsceneTools(options.DeviceClass);\n  return runToolsCLI(tools, options.scriptName).catch((e) => {\n    if (!(e instanceof CLIError)) console.error(e);\n    process.exit(e instanceof CLIError ? e.exitCode : 1);\n  });\n}\n"],"names":["SkillMidsceneTools","BaseMidsceneTools","device","Agent","DeviceClass","runSkillCLI","options","tools","runToolsCLI","e","CLIError","console","process"],"mappings":";;;;;;;;;;;;;AAYA,MAAMA,2BAA2BC;IAKrB,wBAAoC;QAC5C,OAAO,IAAI,IAAI,CAAC,WAAW;IAC7B;IAEA,MAAgB,cAAkC;QAChD,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE;YACf,MAAMC,SAAS,IAAI,IAAI,CAAC,WAAW;YACnC,IAAI,CAAC,KAAK,GAAG,IAAIC,MAAMD;QACzB;QACA,OAAO,IAAI,CAAC,KAAK;IACnB;IAdA,YAAoBE,WAAwB,CAAE;QAC5C,KAAK,wDADaA,WAAW,GAAXA;IAEpB;AAaF;AAwBO,SAASC,YAAYC,OAAwB;IAClD,MAAMC,QAAQ,IAAIP,mBAAmBM,QAAQ,WAAW;IACxD,OAAOE,YAAYD,OAAOD,QAAQ,UAAU,EAAE,KAAK,CAAC,CAACG;QACnD,IAAI,CAAEA,CAAAA,aAAaC,QAAO,GAAIC,QAAQ,KAAK,CAACF;QAC5CG,QAAQ,IAAI,CAACH,aAAaC,WAAWD,EAAE,QAAQ,GAAG;IACpD;AACF"}

package/dist/es/task-runner.mjs

@@ -0,0 +1,263 @@
+import { setTimingFieldOnce } from "./task-timing.mjs";
+import { ExecutionDump } from "./types.mjs";
+import { getDebug } from "@midscene/shared/logger";
+import { assert, uuid } from "@midscene/shared/utils";
+function _define_property(obj, key, value) {
+    if (key in obj) Object.defineProperty(obj, key, {
+        value: value,
+        enumerable: true,
+        configurable: true,
+        writable: true
+    });
+    else obj[key] = value;
+    return obj;
+}
+const debug = getDebug('task-runner');
+const UI_CONTEXT_CACHE_TTL_MS = 300;
+class TaskRunner {
+    async emitOnTaskUpdate(error) {
+        if (!this.onTaskUpdate) return;
+        await this.onTaskUpdate(this, error);
+    }
+    async getUiContext(options) {
+        const now = Date.now();
+        const shouldReuse = !options?.forceRefresh && this.lastUiContext && now - this.lastUiContext.capturedAt <= UI_CONTEXT_CACHE_TTL_MS;
+        if (shouldReuse && this.lastUiContext?.context) {
+            debug(`reuse cached uiContext captured ${now - this.lastUiContext.capturedAt}ms ago`);
+            return this.lastUiContext?.context;
+        }
+        try {
+            const uiContext = await this.uiContextBuilder();
+            if (uiContext) this.lastUiContext = {
+                context: uiContext,
+                capturedAt: Date.now()
+            };
+            else this.lastUiContext = void 0;
+            return uiContext;
+        } catch (error) {
+            this.lastUiContext = void 0;
+            throw error;
+        }
+    }
+    async captureScreenshot() {
+        try {
+            const uiContext = await this.getUiContext({
+                forceRefresh: true
+            });
+            return uiContext?.screenshot;
+        } catch (error) {
+            console.error('error while capturing screenshot', error);
+        }
+    }
+    attachRecorderItem(task, screenshot, phase) {
+        if (!phase || !screenshot) return;
+        const recorderItem = {
+            type: 'screenshot',
+            ts: Date.now(),
+            screenshot,
+            timing: phase
+        };
+        if (!task.recorder) {
+            task.recorder = [
+                recorderItem
+            ];
+            return;
+        }
+        task.recorder.push(recorderItem);
+    }
+    markTaskAsPending(task) {
+        return {
+            taskId: uuid(),
+            status: 'pending',
+            ...task
+        };
+    }
+    normalizeStatusFromError(options, errorMessage) {
+        if ('error' !== this.status) return;
+        assert(options?.allowWhenError, errorMessage || `task runner is in error state, cannot proceed\nerror=${this.latestErrorTask()?.error}\n${this.latestErrorTask()?.errorStack}`);
+        this.status = this.tasks.length > 0 ? 'pending' : 'init';
+    }
+    async append(task, options) {
+        this.normalizeStatusFromError(options, `task runner is in error state, cannot append task\nerror=${this.latestErrorTask()?.error}\n${this.latestErrorTask()?.errorStack}`);
+        if (Array.isArray(task)) this.tasks.push(...task.map((item)=>this.markTaskAsPending(item)));
+        else this.tasks.push(this.markTaskAsPending(task));
+        if ('running' !== this.status) this.status = 'pending';
+        await this.emitOnTaskUpdate();
+    }
+    async appendAndFlush(task, options) {
+        await this.append(task, options);
+        return this.flush(options);
+    }
+    async flush(options) {
+        if ('init' === this.status && this.tasks.length > 0) console.warn('illegal state for task runner, status is init but tasks are not empty');
+        this.normalizeStatusFromError(options, 'task runner is in error state');
+        assert('running' !== this.status, 'task runner is already running');
+        assert('completed' !== this.status, 'task runner is already completed');
+        const nextPendingIndex = this.tasks.findIndex((task)=>'pending' === task.status);
+        if (nextPendingIndex < 0) return;
+        this.status = 'running';
+        await this.emitOnTaskUpdate();
+        let taskIndex = nextPendingIndex;
+        let successfullyCompleted = true;
+        let previousFindOutput;
+        while(taskIndex < this.tasks.length){
+            const task = this.tasks[taskIndex];
+            assert('pending' === task.status, `task status should be pending, but got: ${task.status}`);
+            task.timing = {
+                start: Date.now()
+            };
+            try {
+                task.status = 'running';
+                await this.emitOnTaskUpdate();
+                try {
+                    if (this.onTaskStart) await this.onTaskStart(task);
+                } catch (e) {
+                    console.error('error in onTaskStart', e);
+                }
+                assert([
+                    'Insight',
+                    'Action Space',
+                    'Planning'
+                ].indexOf(task.type) >= 0, `unsupported task type: ${task.type}`);
+                const { executor, param } = task;
+                assert(executor, `executor is required for task type: ${task.type}`);
+                let returnValue;
+                const forceRefresh = 'Insight' === task.type;
+                setTimingFieldOnce(task.timing, 'getUiContextStart');
+                const uiContext = await this.getUiContext({
+                    forceRefresh
+                });
+                setTimingFieldOnce(task.timing, 'getUiContextEnd');
+                task.uiContext = uiContext;
+                const executorContext = {
+                    task,
+                    element: previousFindOutput?.element,
+                    uiContext
+                };
+                if ('Insight' === task.type) {
+                    assert('Query' === task.subType || 'Assert' === task.subType || 'WaitFor' === task.subType || 'Boolean' === task.subType || 'Number' === task.subType || 'String' === task.subType, `unsupported service subType: ${task.subType}`);
+                    returnValue = await task.executor(param, executorContext);
+                } else if ('Planning' === task.type) {
+                    returnValue = await task.executor(param, executorContext);
+                    if ('Locate' === task.subType) previousFindOutput = returnValue?.output;
+                } else if ('Action Space' === task.type) returnValue = await task.executor(param, executorContext);
+                else {
+                    console.warn(`unsupported task type: ${task.type}, will try to execute it directly`);
+                    returnValue = await task.executor(param, executorContext);
+                }
+                const isLastTask = taskIndex === this.tasks.length - 1;
+                if (isLastTask) {
+                    setTimingFieldOnce(task.timing, 'captureAfterCallingSnapshotStart');
+                    const screenshot = await this.captureScreenshot();
+                    this.attachRecorderItem(task, screenshot, 'after-calling');
+                    setTimingFieldOnce(task.timing, 'captureAfterCallingSnapshotEnd');
+                }
+                Object.assign(task, returnValue);
+                task.status = 'finished';
+                task.timing.end = Date.now();
+                task.timing.cost = task.timing.end - task.timing.start;
+                await this.emitOnTaskUpdate();
+                taskIndex++;
+            } catch (e) {
+                successfullyCompleted = false;
+                task.error = e;
+                task.errorMessage = e?.message || ('string' == typeof e ? e : 'error-without-message');
+                task.errorStack = e.stack;
+                task.status = 'failed';
+                task.timing.end = Date.now();
+                task.timing.cost = task.timing.end - task.timing.start;
+                await this.emitOnTaskUpdate();
+                break;
+            }
+        }
+        for(let i = taskIndex + 1; i < this.tasks.length; i++)this.tasks[i].status = 'cancelled';
+        if (taskIndex + 1 < this.tasks.length) await this.emitOnTaskUpdate();
+        let finalizeError;
+        if (successfullyCompleted) {
+            this.status = 'completed';
+            await this.emitOnTaskUpdate();
+        } else {
+            this.status = 'error';
+            const errorTask = this.latestErrorTask();
+            const messageBase = errorTask?.errorMessage || (errorTask?.error ? String(errorTask.error) : 'Task execution failed');
+            const stack = errorTask?.errorStack;
+            const message = stack ? `${messageBase}\n${stack}` : messageBase;
+            finalizeError = new TaskExecutionError(message, this, errorTask, {
+                cause: errorTask?.error
+            });
+            await this.emitOnTaskUpdate(finalizeError);
+        }
+        if (finalizeError) throw finalizeError;
+        if (this.tasks.length) {
+            const outputIndex = Math.min(taskIndex, this.tasks.length - 1);
+            const { thought, output } = this.tasks[outputIndex];
+            return {
+                thought,
+                output
+            };
+        }
+    }
+    isInErrorState() {
+        return 'error' === this.status;
+    }
+    latestErrorTask() {
+        if ('error' !== this.status) return null;
+        for(let i = this.tasks.length - 1; i >= 0; i--)if ('failed' === this.tasks[i].status) return this.tasks[i];
+        return null;
+    }
+    dump() {
+        return new ExecutionDump({
+            id: this.id,
+            logTime: this.executionLogTime,
+            name: this.name,
+            tasks: this.tasks
+        });
+    }
+    async appendErrorPlan(errorMsg) {
+        const errorTask = {
+            type: 'Action Space',
+            subType: 'Error',
+            param: {
+                thought: errorMsg
+            },
+            thought: errorMsg,
+            executor: async ()=>{
+                throw new Error(errorMsg || 'error without thought');
+            }
+        };
+        await this.appendAndFlush(errorTask);
+        return {
+            output: void 0,
+            runner: this
+        };
+    }
+    constructor(name, uiContextBuilder, options){
+        _define_property(this, "id", void 0);
+        _define_property(this, "name", void 0);
+        _define_property(this, "tasks", void 0);
+        _define_property(this, "status", void 0);
+        _define_property(this, "onTaskStart", void 0);
+        _define_property(this, "uiContextBuilder", void 0);
+        _define_property(this, "onTaskUpdate", void 0);
+        _define_property(this, "executionLogTime", void 0);
+        _define_property(this, "lastUiContext", void 0);
+        this.id = uuid();
+        this.status = options?.tasks && options.tasks.length > 0 ? 'pending' : 'init';
+        this.name = name;
+        this.tasks = (options?.tasks || []).map((item)=>this.markTaskAsPending(item));
+        this.onTaskStart = options?.onTaskStart;
+        this.uiContextBuilder = uiContextBuilder;
+        this.onTaskUpdate = options?.onTaskUpdate;
+        this.executionLogTime = Date.now();
+    }
+}
+class TaskExecutionError extends Error {
+    constructor(message, runner, errorTask, options){
+        super(message, options), _define_property(this, "runner", void 0), _define_property(this, "errorTask", void 0);
+        this.runner = runner;
+        this.errorTask = errorTask;
+    }
+}
+export { TaskExecutionError, TaskRunner };
+
+//# sourceMappingURL=task-runner.mjs.map

package/dist/es/task-runner.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"task-runner.mjs","sources":["../../src/task-runner.ts"],"sourcesContent":["import type { ScreenshotItem } from '@/screenshot-item';\nimport { setTimingFieldOnce } from '@/task-timing';\nimport {\n  ExecutionDump,\n  type ExecutionRecorderItem,\n  type ExecutionTask,\n  type ExecutionTaskActionApply,\n  type ExecutionTaskApply,\n  type ExecutionTaskPlanningLocateOutput,\n  type ExecutionTaskProgressOptions,\n  type ExecutionTaskReturn,\n  type ExecutorContext,\n  type PlanningActionParamError,\n  type UIContext,\n} from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert, uuid } from '@midscene/shared/utils';\n\nconst debug = getDebug('task-runner');\nconst UI_CONTEXT_CACHE_TTL_MS = 300;\n\ntype TaskRunnerInitOptions = ExecutionTaskProgressOptions & {\n  tasks?: ExecutionTaskApply[];\n  onTaskUpdate?: (\n    runner: TaskRunner,\n    error?: TaskExecutionError,\n  ) => Promise<void> | void;\n};\n\ntype TaskRunnerOperationOptions = {\n  allowWhenError?: boolean;\n};\n\nexport class TaskRunner {\n  readonly id: string;\n  name: string;\n\n  tasks: ExecutionTask[];\n\n  // status of runner\n  status: 'init' | 'pending' | 'running' | 'completed' | 'error';\n\n  onTaskStart?: ExecutionTaskProgressOptions['onTaskStart'];\n\n  private readonly uiContextBuilder: () => Promise<UIContext>;\n\n  private readonly onTaskUpdate?:\n    | ((runner: TaskRunner, error?: TaskExecutionError) => Promise<void> | void)\n    | undefined;\n\n  private readonly executionLogTime: number;\n\n  constructor(\n    name: string,\n    uiContextBuilder: () => Promise<UIContext>,\n    options?: TaskRunnerInitOptions,\n  ) {\n    this.id = uuid();\n    this.status =\n      options?.tasks && options.tasks.length > 0 ? 'pending' : 'init';\n    this.name = name;\n    this.tasks = (options?.tasks || []).map((item) =>\n      this.markTaskAsPending(item),\n    );\n    this.onTaskStart = options?.onTaskStart;\n    this.uiContextBuilder = uiContextBuilder;\n    this.onTaskUpdate = options?.onTaskUpdate;\n    this.executionLogTime = Date.now();\n  }\n\n  private async emitOnTaskUpdate(error?: TaskExecutionError): Promise<void> {\n    if (!this.onTaskUpdate) {\n      return;\n    }\n    await this.onTaskUpdate(this, error);\n  }\n\n  private lastUiContext?: {\n    context: UIContext;\n    capturedAt: number;\n  };\n\n  private async getUiContext(options?: { forceRefresh?: boolean }): Promise<\n    UIContext | undefined\n  > {\n    const now = Date.now();\n    const shouldReuse =\n      !options?.forceRefresh &&\n      this.lastUiContext &&\n      now - this.lastUiContext.capturedAt <= UI_CONTEXT_CACHE_TTL_MS;\n\n    if (shouldReuse && this.lastUiContext?.context) {\n      debug(\n        `reuse cached uiContext captured ${now - this.lastUiContext.capturedAt}ms ago`,\n      );\n      return this.lastUiContext?.context;\n    }\n\n    try {\n      const uiContext = await this.uiContextBuilder();\n      if (uiContext) {\n        this.lastUiContext = {\n          context: uiContext,\n          capturedAt: Date.now(),\n        };\n      } else {\n        this.lastUiContext = undefined;\n      }\n      return uiContext;\n    } catch (error) {\n      this.lastUiContext = undefined;\n      throw error;\n    }\n  }\n\n  private async captureScreenshot(): Promise<ScreenshotItem | undefined> {\n    try {\n      const uiContext = await this.getUiContext({ forceRefresh: true });\n      return uiContext?.screenshot;\n    } catch (error) {\n      console.error('error while capturing screenshot', error);\n    }\n    return undefined;\n  }\n\n  private attachRecorderItem(\n    task: ExecutionTask,\n    screenshot: ScreenshotItem | undefined,\n    phase: 'after-calling',\n  ): void {\n    if (!phase || !screenshot) {\n      return;\n    }\n\n    const recorderItem: ExecutionRecorderItem = {\n      type: 'screenshot',\n      ts: Date.now(),\n      screenshot,\n      timing: phase,\n    };\n\n    if (!task.recorder) {\n      task.recorder = [recorderItem];\n      return;\n    }\n    task.recorder.push(recorderItem);\n  }\n\n  private markTaskAsPending(task: ExecutionTaskApply): ExecutionTask {\n    return {\n      taskId: uuid(),\n      status: 'pending',\n      ...task,\n    };\n  }\n\n  private normalizeStatusFromError(\n    options?: TaskRunnerOperationOptions,\n    errorMessage?: string,\n  ): void {\n    if (this.status !== 'error') {\n      return;\n    }\n    assert(\n      options?.allowWhenError,\n      errorMessage ||\n        `task runner is in error state, cannot proceed\\nerror=${this.latestErrorTask()?.error}\\n${this.latestErrorTask()?.errorStack}`,\n    );\n    // reset runner state so new tasks can run\n    this.status = this.tasks.length > 0 ? 'pending' : 'init';\n  }\n\n  async append(\n    task: ExecutionTaskApply[] | ExecutionTaskApply,\n    options?: TaskRunnerOperationOptions,\n  ): Promise<void> {\n    this.normalizeStatusFromError(\n      options,\n      `task runner is in error state, cannot append task\\nerror=${this.latestErrorTask()?.error}\\n${this.latestErrorTask()?.errorStack}`,\n    );\n    if (Array.isArray(task)) {\n      this.tasks.push(...task.map((item) => this.markTaskAsPending(item)));\n    } else {\n      this.tasks.push(this.markTaskAsPending(task));\n    }\n    if (this.status !== 'running') {\n      this.status = 'pending';\n    }\n    await this.emitOnTaskUpdate();\n  }\n\n  async appendAndFlush(\n    task: ExecutionTaskApply[] | ExecutionTaskApply,\n    options?: TaskRunnerOperationOptions,\n  ): Promise<{ output: any; thought?: string } | undefined> {\n    await this.append(task, options);\n    return this.flush(options);\n  }\n\n  async flush(\n    options?: TaskRunnerOperationOptions,\n  ): Promise<{ output: any; thought?: string } | undefined> {\n    if (this.status === 'init' && this.tasks.length > 0) {\n      console.warn(\n        'illegal state for task runner, status is init but tasks are not empty',\n      );\n    }\n\n    this.normalizeStatusFromError(options, 'task runner is in error state');\n    assert(this.status !== 'running', 'task runner is already running');\n    assert(this.status !== 'completed', 'task runner is already completed');\n\n    const nextPendingIndex = this.tasks.findIndex(\n      (task) => task.status === 'pending',\n    );\n    if (nextPendingIndex < 0) {\n      // all tasks are completed\n      return;\n    }\n\n    this.status = 'running';\n    await this.emitOnTaskUpdate();\n    let taskIndex = nextPendingIndex;\n    let successfullyCompleted = true;\n\n    let previousFindOutput: ExecutionTaskPlanningLocateOutput | undefined;\n\n    while (taskIndex < this.tasks.length) {\n      const task = this.tasks[taskIndex];\n      assert(\n        task.status === 'pending',\n        `task status should be pending, but got: ${task.status}`,\n      );\n      task.timing = {\n        start: Date.now(),\n      };\n      try {\n        task.status = 'running';\n        await this.emitOnTaskUpdate();\n        try {\n          if (this.onTaskStart) {\n            await this.onTaskStart(task);\n          }\n        } catch (e) {\n          console.error('error in onTaskStart', e);\n        }\n        assert(\n          ['Insight', 'Action Space', 'Planning'].indexOf(task.type) >= 0,\n          `unsupported task type: ${task.type}`,\n        );\n\n        const { executor, param } = task;\n        assert(executor, `executor is required for task type: ${task.type}`);\n\n        let returnValue;\n        // For Insight tasks (Query/Assert/WaitFor), always get fresh context\n        // to ensure we have the latest UI state after any preceding actions\n        const forceRefresh = task.type === 'Insight';\n        setTimingFieldOnce(task.timing, 'getUiContextStart');\n        const uiContext = await this.getUiContext({ forceRefresh });\n        setTimingFieldOnce(task.timing, 'getUiContextEnd');\n\n        task.uiContext = uiContext;\n        const executorContext: ExecutorContext = {\n          task,\n          element: previousFindOutput?.element,\n          uiContext,\n        };\n\n        if (task.type === 'Insight') {\n          assert(\n            task.subType === 'Query' ||\n              task.subType === 'Assert' ||\n              task.subType === 'WaitFor' ||\n              task.subType === 'Boolean' ||\n              task.subType === 'Number' ||\n              task.subType === 'String',\n            `unsupported service subType: ${task.subType}`,\n          );\n          returnValue = await task.executor(param, executorContext);\n        } else if (task.type === 'Planning') {\n          returnValue = await task.executor(param, executorContext);\n          if (task.subType === 'Locate') {\n            previousFindOutput = (\n              returnValue as ExecutionTaskReturn<ExecutionTaskPlanningLocateOutput>\n            )?.output;\n          }\n        } else if (task.type === 'Action Space') {\n          returnValue = await task.executor(param, executorContext);\n        } else {\n          console.warn(\n            `unsupported task type: ${task.type}, will try to execute it directly`,\n          );\n          returnValue = await task.executor(param, executorContext);\n        }\n\n        const isLastTask = taskIndex === this.tasks.length - 1;\n\n        if (isLastTask) {\n          setTimingFieldOnce(task.timing, 'captureAfterCallingSnapshotStart');\n          const screenshot = await this.captureScreenshot();\n          this.attachRecorderItem(task, screenshot, 'after-calling');\n          setTimingFieldOnce(task.timing, 'captureAfterCallingSnapshotEnd');\n        }\n\n        Object.assign(task, returnValue);\n        task.status = 'finished';\n        task.timing.end = Date.now();\n        task.timing.cost = task.timing.end - task.timing.start;\n        await this.emitOnTaskUpdate();\n        taskIndex++;\n      } catch (e: any) {\n        successfullyCompleted = false;\n        task.error = e;\n        task.errorMessage =\n          e?.message || (typeof e === 'string' ? e : 'error-without-message');\n        task.errorStack = e.stack;\n\n        task.status = 'failed';\n        task.timing.end = Date.now();\n        task.timing.cost = task.timing.end - task.timing.start;\n        await this.emitOnTaskUpdate();\n        break;\n      }\n    }\n\n    // set all remaining tasks as cancelled\n    for (let i = taskIndex + 1; i < this.tasks.length; i++) {\n      this.tasks[i].status = 'cancelled';\n    }\n    if (taskIndex + 1 < this.tasks.length) {\n      await this.emitOnTaskUpdate();\n    }\n\n    let finalizeError: TaskExecutionError | undefined;\n    if (!successfullyCompleted) {\n      this.status = 'error';\n      const errorTask = this.latestErrorTask();\n      const messageBase =\n        errorTask?.errorMessage ||\n        (errorTask?.error ? String(errorTask.error) : 'Task execution failed');\n      const stack = errorTask?.errorStack;\n      const message = stack ? `${messageBase}\\n${stack}` : messageBase;\n      finalizeError = new TaskExecutionError(message, this, errorTask, {\n        cause: errorTask?.error,\n      });\n      await this.emitOnTaskUpdate(finalizeError);\n    } else {\n      this.status = 'completed';\n      await this.emitOnTaskUpdate();\n    }\n\n    if (finalizeError) {\n      throw finalizeError;\n    }\n\n    if (this.tasks.length) {\n      // return the last output\n      const outputIndex = Math.min(taskIndex, this.tasks.length - 1);\n      const { thought, output } = this.tasks[outputIndex];\n      return {\n        thought,\n        output,\n      };\n    }\n  }\n\n  isInErrorState(): boolean {\n    return this.status === 'error';\n  }\n\n  latestErrorTask(): ExecutionTask | null {\n    if (this.status !== 'error') {\n      return null;\n    }\n    // Find the LAST failed task (not the first one)\n    // This is important when using allowWhenError to continue after errors\n    for (let i = this.tasks.length - 1; i >= 0; i--) {\n      if (this.tasks[i].status === 'failed') {\n        return this.tasks[i];\n      }\n    }\n    return null;\n  }\n\n  dump(): ExecutionDump {\n    return new ExecutionDump({\n      id: this.id,\n      logTime: this.executionLogTime,\n      name: this.name,\n      tasks: this.tasks,\n    });\n  }\n\n  async appendErrorPlan(errorMsg: string): Promise<{\n    output: undefined;\n    runner: TaskRunner;\n  }> {\n    const errorTask: ExecutionTaskActionApply<PlanningActionParamError> = {\n      type: 'Action Space',\n      subType: 'Error',\n      param: {\n        thought: errorMsg,\n      },\n      thought: errorMsg,\n      executor: async () => {\n        throw new Error(errorMsg || 'error without thought');\n      },\n    };\n    await this.appendAndFlush(errorTask);\n\n    return {\n      output: undefined,\n      runner: this,\n    };\n  }\n}\n\nexport class TaskExecutionError extends Error {\n  runner: TaskRunner;\n\n  errorTask: ExecutionTask | null;\n\n  constructor(\n    message: string,\n    runner: TaskRunner,\n    errorTask: ExecutionTask | null,\n    options?: { cause?: unknown },\n  ) {\n    super(message, options);\n    this.runner = runner;\n    this.errorTask = errorTask;\n  }\n}\n"],"names":["debug","getDebug","UI_CONTEXT_CACHE_TTL_MS","TaskRunner","error","options","now","Date","shouldReuse","uiContext","undefined","console","task","screenshot","phase","recorderItem","uuid","errorMessage","assert","Array","item","nextPendingIndex","taskIndex","successfullyCompleted","previousFindOutput","e","executor","param","returnValue","forceRefresh","setTimingFieldOnce","executorContext","isLastTask","Object","i","finalizeError","errorTask","messageBase","String","stack","message","TaskExecutionError","outputIndex","Math","thought","output","ExecutionDump","errorMsg","Error","name","uiContextBuilder","runner"],"mappings":";;;;;;;;;;;;;;AAkBA,MAAMA,QAAQC,SAAS;AACvB,MAAMC,0BAA0B;AAczB,MAAMC;IAqCX,MAAc,iBAAiBC,KAA0B,EAAiB;QACxE,IAAI,CAAC,IAAI,CAAC,YAAY,EACpB;QAEF,MAAM,IAAI,CAAC,YAAY,CAAC,IAAI,EAAEA;IAChC;IAOA,MAAc,aAAaC,OAAoC,EAE7D;QACA,MAAMC,MAAMC,KAAK,GAAG;QACpB,MAAMC,cACJ,CAACH,SAAS,gBACV,IAAI,CAAC,aAAa,IAClBC,MAAM,IAAI,CAAC,aAAa,CAAC,UAAU,IAAIJ;QAEzC,IAAIM,eAAe,IAAI,CAAC,aAAa,EAAE,SAAS;YAC9CR,MACE,CAAC,gCAAgC,EAAEM,MAAM,IAAI,CAAC,aAAa,CAAC,UAAU,CAAC,MAAM,CAAC;YAEhF,OAAO,IAAI,CAAC,aAAa,EAAE;QAC7B;QAEA,IAAI;YACF,MAAMG,YAAY,MAAM,IAAI,CAAC,gBAAgB;YAC7C,IAAIA,WACF,IAAI,CAAC,aAAa,GAAG;gBACnB,SAASA;gBACT,YAAYF,KAAK,GAAG;YACtB;iBAEA,IAAI,CAAC,aAAa,GAAGG;YAEvB,OAAOD;QACT,EAAE,OAAOL,OAAO;YACd,IAAI,CAAC,aAAa,GAAGM;YACrB,MAAMN;QACR;IACF;IAEA,MAAc,oBAAyD;QACrE,IAAI;YACF,MAAMK,YAAY,MAAM,IAAI,CAAC,YAAY,CAAC;gBAAE,cAAc;YAAK;YAC/D,OAAOA,WAAW;QACpB,EAAE,OAAOL,OAAO;YACdO,QAAQ,KAAK,CAAC,oCAAoCP;QACpD;IAEF;IAEQ,mBACNQ,IAAmB,EACnBC,UAAsC,EACtCC,KAAsB,EAChB;QACN,IAAI,CAACA,SAAS,CAACD,YACb;QAGF,MAAME,eAAsC;YAC1C,MAAM;YACN,IAAIR,KAAK,GAAG;YACZM;YACA,QAAQC;QACV;QAEA,IAAI,CAACF,KAAK,QAAQ,EAAE;YAClBA,KAAK,QAAQ,GAAG;gBAACG;aAAa;YAC9B;QACF;QACAH,KAAK,QAAQ,CAAC,IAAI,CAACG;IACrB;IAEQ,kBAAkBH,IAAwB,EAAiB;QACjE,OAAO;YACL,QAAQI;YACR,QAAQ;YACR,GAAGJ,IAAI;QACT;IACF;IAEQ,yBACNP,OAAoC,EACpCY,YAAqB,EACf;QACN,IAAI,AAAgB,YAAhB,IAAI,CAAC,MAAM,EACb;QAEFC,OACEb,SAAS,gBACTY,gBACE,CAAC,qDAAqD,EAAE,IAAI,CAAC,eAAe,IAAI,MAAM,EAAE,EAAE,IAAI,CAAC,eAAe,IAAI,YAAY;QAGlI,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,YAAY;IACpD;IAEA,MAAM,OACJL,IAA+C,EAC/CP,OAAoC,EACrB;QACf,IAAI,CAAC,wBAAwB,CAC3BA,SACA,CAAC,yDAAyD,EAAE,IAAI,CAAC,eAAe,IAAI,MAAM,EAAE,EAAE,IAAI,CAAC,eAAe,IAAI,YAAY;QAEpI,IAAIc,MAAM,OAAO,CAACP,OAChB,IAAI,CAAC,KAAK,CAAC,IAAI,IAAIA,KAAK,GAAG,CAAC,CAACQ,OAAS,IAAI,CAAC,iBAAiB,CAACA;aAE7D,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAACR;QAEzC,IAAI,AAAgB,cAAhB,IAAI,CAAC,MAAM,EACb,IAAI,CAAC,MAAM,GAAG;QAEhB,MAAM,IAAI,CAAC,gBAAgB;IAC7B;IAEA,MAAM,eACJA,IAA+C,EAC/CP,OAAoC,EACoB;QACxD,MAAM,IAAI,CAAC,MAAM,CAACO,MAAMP;QACxB,OAAO,IAAI,CAAC,KAAK,CAACA;IACpB;IAEA,MAAM,MACJA,OAAoC,EACoB;QACxD,IAAI,AAAgB,WAAhB,IAAI,CAAC,MAAM,IAAe,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,GAChDM,QAAQ,IAAI,CACV;QAIJ,IAAI,CAAC,wBAAwB,CAACN,SAAS;QACvCa,OAAO,AAAgB,cAAhB,IAAI,CAAC,MAAM,EAAgB;QAClCA,OAAO,AAAgB,gBAAhB,IAAI,CAAC,MAAM,EAAkB;QAEpC,MAAMG,mBAAmB,IAAI,CAAC,KAAK,CAAC,SAAS,CAC3C,CAACT,OAASA,AAAgB,cAAhBA,KAAK,MAAM;QAEvB,IAAIS,mBAAmB,GAErB;QAGF,IAAI,CAAC,MAAM,GAAG;QACd,MAAM,IAAI,CAAC,gBAAgB;QAC3B,IAAIC,YAAYD;QAChB,IAAIE,wBAAwB;QAE5B,IAAIC;QAEJ,MAAOF,YAAY,IAAI,CAAC,KAAK,CAAC,MAAM,CAAE;YACpC,MAAMV,OAAO,IAAI,CAAC,KAAK,CAACU,UAAU;YAClCJ,OACEN,AAAgB,cAAhBA,KAAK,MAAM,EACX,CAAC,wCAAwC,EAAEA,KAAK,MAAM,EAAE;YAE1DA,KAAK,MAAM,GAAG;gBACZ,OAAOL,KAAK,GAAG;YACjB;YACA,IAAI;gBACFK,KAAK,MAAM,GAAG;gBACd,MAAM,IAAI,CAAC,gBAAgB;gBAC3B,IAAI;oBACF,IAAI,IAAI,CAAC,WAAW,EAClB,MAAM,IAAI,CAAC,WAAW,CAACA;gBAE3B,EAAE,OAAOa,GAAG;oBACVd,QAAQ,KAAK,CAAC,wBAAwBc;gBACxC;gBACAP,OACE;oBAAC;oBAAW;oBAAgB;iBAAW,CAAC,OAAO,CAACN,KAAK,IAAI,KAAK,GAC9D,CAAC,uBAAuB,EAAEA,KAAK,IAAI,EAAE;gBAGvC,MAAM,EAAEc,QAAQ,EAAEC,KAAK,EAAE,GAAGf;gBAC5BM,OAAOQ,UAAU,CAAC,oCAAoC,EAAEd,KAAK,IAAI,EAAE;gBAEnE,IAAIgB;gBAGJ,MAAMC,eAAejB,AAAc,cAAdA,KAAK,IAAI;gBAC9BkB,mBAAmBlB,KAAK,MAAM,EAAE;gBAChC,MAAMH,YAAY,MAAM,IAAI,CAAC,YAAY,CAAC;oBAAEoB;gBAAa;gBACzDC,mBAAmBlB,KAAK,MAAM,EAAE;gBAEhCA,KAAK,SAAS,GAAGH;gBACjB,MAAMsB,kBAAmC;oBACvCnB;oBACA,SAASY,oBAAoB;oBAC7Bf;gBACF;gBAEA,IAAIG,AAAc,cAAdA,KAAK,IAAI,EAAgB;oBAC3BM,OACEN,AAAiB,YAAjBA,KAAK,OAAO,IACVA,AAAiB,aAAjBA,KAAK,OAAO,IACZA,AAAiB,cAAjBA,KAAK,OAAO,IACZA,AAAiB,cAAjBA,KAAK,OAAO,IACZA,AAAiB,aAAjBA,KAAK,OAAO,IACZA,AAAiB,aAAjBA,KAAK,OAAO,EACd,CAAC,6BAA6B,EAAEA,KAAK,OAAO,EAAE;oBAEhDgB,cAAc,MAAMhB,KAAK,QAAQ,CAACe,OAAOI;gBAC3C,OAAO,IAAInB,AAAc,eAAdA,KAAK,IAAI,EAAiB;oBACnCgB,cAAc,MAAMhB,KAAK,QAAQ,CAACe,OAAOI;oBACzC,IAAInB,AAAiB,aAAjBA,KAAK,OAAO,EACdY,qBACEI,aACC;gBAEP,OAAO,IAAIhB,AAAc,mBAAdA,KAAK,IAAI,EAClBgB,cAAc,MAAMhB,KAAK,QAAQ,CAACe,OAAOI;qBACpC;oBACLpB,QAAQ,IAAI,CACV,CAAC,uBAAuB,EAAEC,KAAK,IAAI,CAAC,iCAAiC,CAAC;oBAExEgB,cAAc,MAAMhB,KAAK,QAAQ,CAACe,OAAOI;gBAC3C;gBAEA,MAAMC,aAAaV,cAAc,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG;gBAErD,IAAIU,YAAY;oBACdF,mBAAmBlB,KAAK,MAAM,EAAE;oBAChC,MAAMC,aAAa,MAAM,IAAI,CAAC,iBAAiB;oBAC/C,IAAI,CAAC,kBAAkB,CAACD,MAAMC,YAAY;oBAC1CiB,mBAAmBlB,KAAK,MAAM,EAAE;gBAClC;gBAEAqB,OAAO,MAAM,CAACrB,MAAMgB;gBACpBhB,KAAK,MAAM,GAAG;gBACdA,KAAK,MAAM,CAAC,GAAG,GAAGL,KAAK,GAAG;gBAC1BK,KAAK,MAAM,CAAC,IAAI,GAAGA,KAAK,MAAM,CAAC,GAAG,GAAGA,KAAK,MAAM,CAAC,KAAK;gBACtD,MAAM,IAAI,CAAC,gBAAgB;gBAC3BU;YACF,EAAE,OAAOG,GAAQ;gBACfF,wBAAwB;gBACxBX,KAAK,KAAK,GAAGa;gBACbb,KAAK,YAAY,GACfa,GAAG,WAAY,CAAa,YAAb,OAAOA,IAAiBA,IAAI,uBAAsB;gBACnEb,KAAK,UAAU,GAAGa,EAAE,KAAK;gBAEzBb,KAAK,MAAM,GAAG;gBACdA,KAAK,MAAM,CAAC,GAAG,GAAGL,KAAK,GAAG;gBAC1BK,KAAK,MAAM,CAAC,IAAI,GAAGA,KAAK,MAAM,CAAC,GAAG,GAAGA,KAAK,MAAM,CAAC,KAAK;gBACtD,MAAM,IAAI,CAAC,gBAAgB;gBAC3B;YACF;QACF;QAGA,IAAK,IAAIsB,IAAIZ,YAAY,GAAGY,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,EAAEA,IACjD,IAAI,CAAC,KAAK,CAACA,EAAE,CAAC,MAAM,GAAG;QAEzB,IAAIZ,YAAY,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,EACnC,MAAM,IAAI,CAAC,gBAAgB;QAG7B,IAAIa;QACJ,IAAKZ,uBAYE;YACL,IAAI,CAAC,MAAM,GAAG;YACd,MAAM,IAAI,CAAC,gBAAgB;QAC7B,OAf4B;YAC1B,IAAI,CAAC,MAAM,GAAG;YACd,MAAMa,YAAY,IAAI,CAAC,eAAe;YACtC,MAAMC,cACJD,WAAW,gBACVA,CAAAA,WAAW,QAAQE,OAAOF,UAAU,KAAK,IAAI,uBAAsB;YACtE,MAAMG,QAAQH,WAAW;YACzB,MAAMI,UAAUD,QAAQ,GAAGF,YAAY,EAAE,EAAEE,OAAO,GAAGF;YACrDF,gBAAgB,IAAIM,mBAAmBD,SAAS,IAAI,EAAEJ,WAAW;gBAC/D,OAAOA,WAAW;YACpB;YACA,MAAM,IAAI,CAAC,gBAAgB,CAACD;QAC9B;QAKA,IAAIA,eACF,MAAMA;QAGR,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE;YAErB,MAAMO,cAAcC,KAAK,GAAG,CAACrB,WAAW,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG;YAC5D,MAAM,EAAEsB,OAAO,EAAEC,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAACH,YAAY;YACnD,OAAO;gBACLE;gBACAC;YACF;QACF;IACF;IAEA,iBAA0B;QACxB,OAAO,AAAgB,YAAhB,IAAI,CAAC,MAAM;IACpB;IAEA,kBAAwC;QACtC,IAAI,AAAgB,YAAhB,IAAI,CAAC,MAAM,EACb,OAAO;QAIT,IAAK,IAAIX,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,GAAGA,KAAK,GAAGA,IAC1C,IAAI,AAAyB,aAAzB,IAAI,CAAC,KAAK,CAACA,EAAE,CAAC,MAAM,EACtB,OAAO,IAAI,CAAC,KAAK,CAACA,EAAE;QAGxB,OAAO;IACT;IAEA,OAAsB;QACpB,OAAO,IAAIY,cAAc;YACvB,IAAI,IAAI,CAAC,EAAE;YACX,SAAS,IAAI,CAAC,gBAAgB;YAC9B,MAAM,IAAI,CAAC,IAAI;YACf,OAAO,IAAI,CAAC,KAAK;QACnB;IACF;IAEA,MAAM,gBAAgBC,QAAgB,EAGnC;QACD,MAAMX,YAAgE;YACpE,MAAM;YACN,SAAS;YACT,OAAO;gBACL,SAASW;YACX;YACA,SAASA;YACT,UAAU;gBACR,MAAM,IAAIC,MAAMD,YAAY;YAC9B;QACF;QACA,MAAM,IAAI,CAAC,cAAc,CAACX;QAE1B,OAAO;YACL,QAAQ1B;YACR,QAAQ,IAAI;QACd;IACF;IA3WA,YACEuC,IAAY,EACZC,gBAA0C,EAC1C7C,OAA+B,CAC/B;QAtBF,uBAAS,MAAT;QACA;QAEA;QAGA;QAEA;QAEA,uBAAiB,oBAAjB;QAEA,uBAAiB,gBAAjB;QAIA,uBAAiB,oBAAjB;QA2BA,uBAAQ,iBAAR;QApBE,IAAI,CAAC,EAAE,GAAGW;QACV,IAAI,CAAC,MAAM,GACTX,SAAS,SAASA,QAAQ,KAAK,CAAC,MAAM,GAAG,IAAI,YAAY;QAC3D,IAAI,CAAC,IAAI,GAAG4C;QACZ,IAAI,CAAC,KAAK,GAAI5C,AAAAA,CAAAA,SAAS,SAAS,EAAC,EAAG,GAAG,CAAC,CAACe,OACvC,IAAI,CAAC,iBAAiB,CAACA;QAEzB,IAAI,CAAC,WAAW,GAAGf,SAAS;QAC5B,IAAI,CAAC,gBAAgB,GAAG6C;QACxB,IAAI,CAAC,YAAY,GAAG7C,SAAS;QAC7B,IAAI,CAAC,gBAAgB,GAAGE,KAAK,GAAG;IAClC;AA4VF;AAEO,MAAMkC,2BAA2BO;IAKtC,YACER,OAAe,EACfW,MAAkB,EAClBf,SAA+B,EAC/B/B,OAA6B,CAC7B;QACA,KAAK,CAACmC,SAASnC,UAVjB,0CAEA;QASE,IAAI,CAAC,MAAM,GAAG8C;QACd,IAAI,CAAC,SAAS,GAAGf;IACnB;AACF"}

package/dist/es/task-timing.mjs

@@ -0,0 +1,12 @@
+import { getDebug } from "@midscene/shared/logger";
+const debugTiming = getDebug('task-timing');
+function setTimingFieldOnce(timing, field) {
+    if (!timing) return void debugTiming(`[warning] timing object missing, skip set. field=${field}`);
+    const value = Date.now();
+    const existingValue = timing[field];
+    if (void 0 !== existingValue) return void debugTiming(`[warning] duplicate timing field set ignored. field=${field}, existing=${existingValue}, incoming=${value}`);
+    timing[field] = value;
+}
+export { setTimingFieldOnce };
+
+//# sourceMappingURL=task-timing.mjs.map

package/dist/es/task-timing.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"task-timing.mjs","sources":["../../src/task-timing.ts"],"sourcesContent":["import type { ExecutionTask } from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\n\nconst debugTiming = getDebug('task-timing');\n\ntype ExecutionTaskTiming = NonNullable<ExecutionTask['timing']>;\n\ntype NumericTimingField = {\n  [K in keyof ExecutionTaskTiming]-?: ExecutionTaskTiming[K] extends\n    | number\n    | undefined\n    ? K\n    : never;\n}[keyof ExecutionTaskTiming];\n\nexport type TimingSettableField = Exclude<\n  NumericTimingField,\n  'start' | 'end' | 'cost'\n>;\n\nexport function setTimingFieldOnce(\n  timing: ExecutionTaskTiming | undefined,\n  field: TimingSettableField,\n): void {\n  if (!timing) {\n    debugTiming(`[warning] timing object missing, skip set. field=${field}`);\n    return;\n  }\n\n  const value = Date.now();\n  const existingValue = timing[field];\n  if (existingValue !== undefined) {\n    debugTiming(\n      `[warning] duplicate timing field set ignored. field=${field}, existing=${existingValue}, incoming=${value}`,\n    );\n    return;\n  }\n\n  timing[field] = value;\n}\n"],"names":["debugTiming","getDebug","setTimingFieldOnce","timing","field","value","Date","existingValue","undefined"],"mappings":";AAGA,MAAMA,cAAcC,SAAS;AAiBtB,SAASC,mBACdC,MAAuC,EACvCC,KAA0B;IAE1B,IAAI,CAACD,QAAQ,YACXH,YAAY,CAAC,iDAAiD,EAAEI,OAAO;IAIzE,MAAMC,QAAQC,KAAK,GAAG;IACtB,MAAMC,gBAAgBJ,MAAM,CAACC,MAAM;IACnC,IAAIG,AAAkBC,WAAlBD,eAA6B,YAC/BP,YACE,CAAC,oDAAoD,EAAEI,MAAM,WAAW,EAAEG,cAAc,WAAW,EAAEF,OAAO;IAKhHF,MAAM,CAACC,MAAM,GAAGC;AAClB"}

package/dist/es/tree.mjs

@@ -0,0 +1,13 @@
+import { descriptionOfTree, treeToList, trimAttributes, truncateText } from "@midscene/shared/extractor";
+const ELEMENT_COUNT_WARNING_THRESHOLD = 5000;
+const TREE_SIZE_WARNING_MESSAGE = 'The number of elements is too large, it may cause the prompt to be too long, please use domIncluded: "visible-only" to reduce the number of elements';
+function tree_descriptionOfTree(tree, truncateTextLength, filterNonTextContent = false, visibleOnly = true) {
+    if (!visibleOnly) {
+        const flatElements = treeToList(tree);
+        if (flatElements.length >= ELEMENT_COUNT_WARNING_THRESHOLD) console.warn(TREE_SIZE_WARNING_MESSAGE);
+    }
+    return descriptionOfTree(tree, truncateTextLength, filterNonTextContent, visibleOnly);
+}
+export { tree_descriptionOfTree as descriptionOfTree, trimAttributes, truncateText };
+
+//# sourceMappingURL=tree.mjs.map

package/dist/es/tree.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"tree.mjs","sources":["../../src/tree.ts"],"sourcesContent":["import type { BaseElement, ElementTreeNode } from '@midscene/shared/types';\n\nimport {\n  descriptionOfTree as sharedDescriptionOfTree,\n  treeToList,\n  trimAttributes,\n  truncateText,\n} from '@midscene/shared/extractor';\n\nconst ELEMENT_COUNT_WARNING_THRESHOLD = 5000;\nconst TREE_SIZE_WARNING_MESSAGE =\n  'The number of elements is too large, it may cause the prompt to be too long, please use domIncluded: \"visible-only\" to reduce the number of elements';\n\nexport { trimAttributes, truncateText };\n\nexport function descriptionOfTree<\n  ElementType extends BaseElement = BaseElement,\n>(\n  tree: ElementTreeNode<ElementType>,\n  truncateTextLength?: number,\n  filterNonTextContent = false,\n  visibleOnly = true,\n) {\n  if (!visibleOnly) {\n    const flatElements = treeToList(tree);\n    if (flatElements.length >= ELEMENT_COUNT_WARNING_THRESHOLD) {\n      console.warn(TREE_SIZE_WARNING_MESSAGE);\n    }\n  }\n\n  return sharedDescriptionOfTree(\n    tree,\n    truncateTextLength,\n    filterNonTextContent,\n    visibleOnly,\n  );\n}\n"],"names":["ELEMENT_COUNT_WARNING_THRESHOLD","TREE_SIZE_WARNING_MESSAGE","descriptionOfTree","tree","truncateTextLength","filterNonTextContent","visibleOnly","flatElements","treeToList","console","sharedDescriptionOfTree"],"mappings":";AASA,MAAMA,kCAAkC;AACxC,MAAMC,4BACJ;AAIK,SAASC,uBAGdC,IAAkC,EAClCC,kBAA2B,EAC3BC,uBAAuB,KAAK,EAC5BC,cAAc,IAAI;IAElB,IAAI,CAACA,aAAa;QAChB,MAAMC,eAAeC,WAAWL;QAChC,IAAII,aAAa,MAAM,IAAIP,iCACzBS,QAAQ,IAAI,CAACR;IAEjB;IAEA,OAAOS,kBACLP,MACAC,oBACAC,sBACAC;AAEJ"}

package/dist/es/types.mjs

@@ -0,0 +1,199 @@
+import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { restoreImageReferences } from "./dump/image-restoration.mjs";
+import { ScreenshotItem } from "./screenshot-item.mjs";
+export * from "./yaml.mjs";
+function _define_property(obj, key, value) {
+    if (key in obj) Object.defineProperty(obj, key, {
+        value: value,
+        enumerable: true,
+        configurable: true,
+        writable: true
+    });
+    else obj[key] = value;
+    return obj;
+}
+class UIContext {
+}
+class ServiceError extends Error {
+    constructor(message, dump){
+        super(message), _define_property(this, "dump", void 0);
+        this.name = 'ServiceError';
+        this.dump = dump;
+    }
+}
+function replacerForDumpSerialization(_key, value) {
+    if (value && value.constructor?.name === 'Page') return '[Page object]';
+    if (value && value.constructor?.name === 'Browser') return '[Browser object]';
+    if (value && 'function' == typeof value.toSerializable) return value.toSerializable();
+    return value;
+}
+function reviverForDumpDeserialization(key, value) {
+    if ('screenshot' !== key || 'object' != typeof value || null === value) return value;
+    if (ScreenshotItem.isSerialized(value)) return value;
+    'base64' in value && value.base64;
+    return value;
+}
+class ExecutionDump {
+    serialize(indents) {
+        return JSON.stringify(this.toJSON(), replacerForDumpSerialization, indents);
+    }
+    toJSON() {
+        return {
+            id: this.id,
+            logTime: this.logTime,
+            name: this.name,
+            description: this.description,
+            tasks: this.tasks.map((task)=>({
+                    ...task,
+                    recorder: task.recorder || []
+                })),
+            aiActContext: this.aiActContext
+        };
+    }
+    static fromSerializedString(serialized) {
+        const parsed = JSON.parse(serialized, reviverForDumpDeserialization);
+        return new ExecutionDump(parsed);
+    }
+    static fromJSON(data) {
+        return new ExecutionDump(data);
+    }
+    collectScreenshots() {
+        const screenshots = [];
+        for (const task of this.tasks){
+            if (task.uiContext?.screenshot instanceof ScreenshotItem) screenshots.push(task.uiContext.screenshot);
+            if (task.recorder) {
+                for (const record of task.recorder)if (record.screenshot instanceof ScreenshotItem) screenshots.push(record.screenshot);
+            }
+        }
+        return screenshots;
+    }
+    constructor(data){
+        _define_property(this, "id", void 0);
+        _define_property(this, "logTime", void 0);
+        _define_property(this, "name", void 0);
+        _define_property(this, "description", void 0);
+        _define_property(this, "tasks", void 0);
+        _define_property(this, "aiActContext", void 0);
+        this.id = data.id;
+        this.logTime = data.logTime;
+        this.name = data.name;
+        this.description = data.description;
+        this.tasks = data.tasks;
+        this.aiActContext = data.aiActContext;
+    }
+}
+class GroupedActionDump {
+    serialize(indents) {
+        return JSON.stringify(this.toJSON(), replacerForDumpSerialization, indents);
+    }
+    serializeWithInlineScreenshots(indents) {
+        const processValue = (obj)=>{
+            if (obj instanceof ScreenshotItem) return {
+                base64: obj.base64,
+                capturedAt: obj.capturedAt
+            };
+            if (Array.isArray(obj)) return obj.map(processValue);
+            if (obj && 'object' == typeof obj) {
+                const entries = Object.entries(obj).map(([key, value])=>[
+                        key,
+                        processValue(value)
+                    ]);
+                return Object.fromEntries(entries);
+            }
+            return obj;
+        };
+        const data = processValue(this.toJSON());
+        return JSON.stringify(data, null, indents);
+    }
+    toJSON() {
+        return {
+            sdkVersion: this.sdkVersion,
+            groupName: this.groupName,
+            groupDescription: this.groupDescription,
+            modelBriefs: this.modelBriefs,
+            executions: this.executions.map((exec)=>exec.toJSON()),
+            deviceType: this.deviceType
+        };
+    }
+    static fromSerializedString(serialized) {
+        const parsed = JSON.parse(serialized, reviverForDumpDeserialization);
+        return new GroupedActionDump(parsed);
+    }
+    static fromJSON(data) {
+        return new GroupedActionDump(data);
+    }
+    collectAllScreenshots() {
+        const screenshots = [];
+        for (const execution of this.executions)screenshots.push(...execution.collectScreenshots());
+        return screenshots;
+    }
+    serializeToFiles(basePath) {
+        const screenshotsDir = `${basePath}.screenshots`;
+        if (!existsSync(screenshotsDir)) mkdirSync(screenshotsDir, {
+            recursive: true
+        });
+        const screenshotMap = {};
+        const screenshots = this.collectAllScreenshots();
+        for (const screenshot of screenshots)if (screenshot.hasBase64()) {
+            const imagePath = join(screenshotsDir, `${screenshot.id}.${screenshot.extension}`);
+            const rawBase64 = screenshot.rawBase64;
+            writeFileSync(imagePath, Buffer.from(rawBase64, 'base64'));
+            screenshotMap[screenshot.id] = imagePath;
+        }
+        writeFileSync(`${basePath}.screenshots.json`, JSON.stringify(screenshotMap), 'utf-8');
+        writeFileSync(basePath, this.serialize(), 'utf-8');
+    }
+    static fromFilesAsInlineJson(basePath) {
+        const dumpString = readFileSync(basePath, 'utf-8');
+        const screenshotsMapPath = `${basePath}.screenshots.json`;
+        if (!existsSync(screenshotsMapPath)) return dumpString;
+        const screenshotMap = JSON.parse(readFileSync(screenshotsMapPath, 'utf-8'));
+        const imageMap = {};
+        for (const [id, filePath] of Object.entries(screenshotMap))if (existsSync(filePath)) {
+            const data = readFileSync(filePath);
+            const mime = filePath.endsWith('.jpeg') || filePath.endsWith('.jpg') ? 'jpeg' : 'png';
+            imageMap[id] = `data:image/${mime};base64,${data.toString('base64')}`;
+        }
+        const dumpData = JSON.parse(dumpString);
+        const processedData = restoreImageReferences(dumpData, (id)=>imageMap[id] ?? '');
+        return JSON.stringify(processedData);
+    }
+    static cleanupFiles(basePath) {
+        const filesToClean = [
+            basePath,
+            `${basePath}.screenshots.json`,
+            `${basePath}.screenshots`
+        ];
+        for (const filePath of filesToClean)try {
+            rmSync(filePath, {
+                force: true,
+                recursive: true
+            });
+        } catch  {}
+    }
+    static getFilePaths(basePath) {
+        return [
+            basePath,
+            `${basePath}.screenshots.json`,
+            `${basePath}.screenshots`
+        ];
+    }
+    constructor(data){
+        _define_property(this, "sdkVersion", void 0);
+        _define_property(this, "groupName", void 0);
+        _define_property(this, "groupDescription", void 0);
+        _define_property(this, "modelBriefs", void 0);
+        _define_property(this, "executions", void 0);
+        _define_property(this, "deviceType", void 0);
+        this.sdkVersion = data.sdkVersion;
+        this.groupName = data.groupName;
+        this.groupDescription = data.groupDescription;
+        this.modelBriefs = data.modelBriefs;
+        this.executions = data.executions.map((exec)=>exec instanceof ExecutionDump ? exec : ExecutionDump.fromJSON(exec));
+        this.deviceType = data.deviceType;
+    }
+}
+export { ExecutionDump, GroupedActionDump, ServiceError, UIContext };
+
+//# sourceMappingURL=types.mjs.map