@aiscene/core 8.0.1 → 8.0.3

package/dist/es/screenshot-item.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"screenshot-item.mjs","sources":["../../src/screenshot-item.ts"],"sourcesContent":["import { readFileSync } from 'node:fs';\nimport { uuid } from '@midscene/shared/utils';\nimport { extractImageByIdSync } from './dump/html-utils';\nimport {\n  type ScreenshotRef,\n  normalizeScreenshotRef,\n} from './dump/screenshot-store';\n\n/**\n * Serialization format for ScreenshotItem\n * - { $screenshot: \"id\" } - inline mode, references imageMap in HTML\n * - { base64: \"path\" } - directory mode, references external file path\n */\nexport type ScreenshotSerializeFormat = ScreenshotRef;\n\n/**\n * Detect image format from base64 data URI prefix.\n */\nfunction detectFormat(base64: string): 'png' | 'jpeg' {\n  if (base64.startsWith('data:image/jpeg')) return 'jpeg';\n  if (base64.startsWith('data:image/jpg')) return 'jpeg';\n  return 'png';\n}\n\n/**\n * ScreenshotItem encapsulates screenshot data.\n *\n * Supports lazy loading after memory release:\n * - inline mode: reads from HTML file using streaming (extractImageByIdSync)\n * - directory mode: reads from file on disk\n *\n * After persistence, memory is released but the screenshot can be recovered\n * on-demand from disk, making it safe to release memory at any time.\n */\nexport class ScreenshotItem {\n  private _id: string;\n  private _base64: string | null;\n  private _format: 'png' | 'jpeg';\n  private _capturedAt: number;\n  private _serializedRef: ScreenshotRef | null = null;\n  private _persistedPath: string | null = null;\n  private _persistedHtmlPath: string | null = null;\n\n  private constructor(id: string, base64: string, capturedAt: number) {\n    this._id = id;\n    this._base64 = base64;\n    this._format = detectFormat(base64);\n    this._capturedAt = capturedAt;\n  }\n\n  /** Create a new ScreenshotItem from base64 data */\n  static create(base64: string, capturedAt: number): ScreenshotItem {\n    return new ScreenshotItem(uuid(), base64, capturedAt);\n  }\n\n  get id(): string {\n    return this._id;\n  }\n\n  /** Get the image format (png or jpeg) */\n  get format(): 'png' | 'jpeg' {\n    return this._format;\n  }\n\n  /** Get the file extension for this screenshot */\n  get extension(): string {\n    return this._format === 'jpeg' ? 'jpeg' : 'png';\n  }\n\n  /** Get screenshot capture timestamp in milliseconds */\n  get capturedAt(): number {\n    return this._capturedAt;\n  }\n\n  get base64(): string {\n    // If data is in memory, return it directly\n    if (this._base64 !== null) {\n      return this._base64;\n    }\n\n    const loadFromFile = (): string => {\n      if (this._persistedPath === null) {\n        throw new Error(`Screenshot ${this._id}: file recovery path missing`);\n      }\n      const buffer = readFileSync(this._persistedPath);\n      return `data:image/${this._format};base64,${buffer.toString('base64')}`;\n    };\n\n    const loadFromInline = (): string => {\n      if (this._persistedHtmlPath === null) {\n        throw new Error(`Screenshot ${this._id}: HTML recovery path missing`);\n      }\n      const data = extractImageByIdSync(this._persistedHtmlPath, this._id);\n      if (data) {\n        return data;\n      }\n      throw new Error(\n        `Screenshot ${this._id}: cannot recover from HTML (id not found in ${this._persistedHtmlPath})`,\n      );\n    };\n\n    // Recover from the primary serialized mode first.\n    if (this._serializedRef?.storage === 'file') {\n      return loadFromFile();\n    }\n\n    if (this._serializedRef?.storage === 'inline') {\n      return loadFromInline();\n    }\n\n    // Fall back to whichever recovery path is available.\n    if (this._persistedPath !== null) {\n      return loadFromFile();\n    }\n\n    if (this._persistedHtmlPath !== null) {\n      return loadFromInline();\n    }\n\n    throw new Error(\n      `Screenshot ${this._id}: base64 data released without recovery path`,\n    );\n  }\n\n  /** Check if base64 data is still available in memory (not yet released) */\n  hasBase64(): boolean {\n    return this._base64 !== null;\n  }\n\n  /**\n   * Mark as persisted to HTML (inline mode).\n   * Releases base64 memory, but keeps HTML path for lazy loading recovery.\n   * @param htmlPath - absolute path to the HTML file containing the image\n   */\n  markPersistedInline(htmlPath: string): ScreenshotRef {\n    const ref = this.createRef('inline');\n    this._serializedRef = ref;\n    this._persistedHtmlPath = htmlPath;\n    this._base64 = null;\n    return ref;\n  }\n\n  /**\n   * Register a file-backed recovery path without changing the serialized mode.\n   * Used when inline persistence also needs a shared file copy next to dumps.\n   */\n  registerPersistedFileCopy(\n    relativePath: string,\n    absolutePath: string,\n  ): ScreenshotRef {\n    const ref = this.createRef('file', relativePath);\n    this._persistedPath = absolutePath;\n    this._base64 = null;\n    return ref;\n  }\n\n  /**\n   * Mark as persisted to file (directory mode).\n   * Releases base64 memory, but keeps file path for lazy loading recovery.\n   * @param relativePath - relative path for serialization (e.g., \"./screenshots/id.jpeg\")\n   * @param absolutePath - absolute path for lazy loading recovery\n   */\n  markPersistedToPath(\n    relativePath: string,\n    absolutePath: string,\n  ): ScreenshotRef {\n    const ref = this.registerPersistedFileCopy(relativePath, absolutePath);\n    this._serializedRef = ref;\n    return ref;\n  }\n\n  /** Serialize for JSON - format depends on persistence state */\n  toSerializable(): ScreenshotSerializeFormat {\n    return (\n      this._serializedRef ?? {\n        type: 'midscene_screenshot_ref',\n        id: this._id,\n        capturedAt: this._capturedAt,\n        mimeType: this._format === 'jpeg' ? 'image/jpeg' : 'image/png',\n        storage: 'inline',\n      }\n    );\n  }\n\n  /** Check if a value is a serialized ScreenshotItem reference (inline or directory mode) */\n  static isSerialized(value: unknown): value is ScreenshotSerializeFormat {\n    return normalizeScreenshotRef(value) !== null;\n  }\n\n  private createRef(\n    storage: 'inline' | 'file',\n    relativePath?: string,\n  ): ScreenshotRef {\n    const baseRef: Omit<ScreenshotRef, 'path'> = {\n      type: 'midscene_screenshot_ref',\n      id: this._id,\n      capturedAt: this._capturedAt,\n      mimeType: this._format === 'jpeg' ? 'image/jpeg' : 'image/png',\n      storage,\n    };\n    if (storage === 'file') {\n      return {\n        ...baseRef,\n        storage,\n        path: relativePath!,\n      };\n    }\n    return baseRef;\n  }\n\n  /**\n   * Get base64 data without the data URI prefix.\n   * Useful for writing raw binary data to files.\n   */\n  get rawBase64(): string {\n    return this.base64.replace(/^data:image\\/(png|jpeg|jpg);base64,/, '');\n  }\n}\n"],"names":["detectFormat","base64","ScreenshotItem","capturedAt","uuid","loadFromFile","Error","buffer","readFileSync","loadFromInline","data","extractImageByIdSync","htmlPath","ref","relativePath","absolutePath","value","normalizeScreenshotRef","storage","baseRef","id"],"mappings":";;;;;;;;;;;;;;AAkBA,SAASA,aAAaC,MAAc;IAClC,IAAIA,OAAO,UAAU,CAAC,oBAAoB,OAAO;IACjD,IAAIA,OAAO,UAAU,CAAC,mBAAmB,OAAO;IAChD,OAAO;AACT;AAYO,MAAMC;IAiBX,OAAO,OAAOD,MAAc,EAAEE,UAAkB,EAAkB;QAChE,OAAO,IAAID,eAAeE,QAAQH,QAAQE;IAC5C;IAEA,IAAI,KAAa;QACf,OAAO,IAAI,CAAC,GAAG;IACjB;IAGA,IAAI,SAAyB;QAC3B,OAAO,IAAI,CAAC,OAAO;IACrB;IAGA,IAAI,YAAoB;QACtB,OAAO,AAAiB,WAAjB,IAAI,CAAC,OAAO,GAAc,SAAS;IAC5C;IAGA,IAAI,aAAqB;QACvB,OAAO,IAAI,CAAC,WAAW;IACzB;IAEA,IAAI,SAAiB;QAEnB,IAAI,AAAiB,SAAjB,IAAI,CAAC,OAAO,EACd,OAAO,IAAI,CAAC,OAAO;QAGrB,MAAME,eAAe;YACnB,IAAI,AAAwB,SAAxB,IAAI,CAAC,cAAc,EACrB,MAAM,IAAIC,MAAM,CAAC,WAAW,EAAE,IAAI,CAAC,GAAG,CAAC,4BAA4B,CAAC;YAEtE,MAAMC,SAASC,aAAa,IAAI,CAAC,cAAc;YAC/C,OAAO,CAAC,WAAW,EAAE,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAED,OAAO,QAAQ,CAAC,WAAW;QACzE;QAEA,MAAME,iBAAiB;YACrB,IAAI,AAA4B,SAA5B,IAAI,CAAC,kBAAkB,EACzB,MAAM,IAAIH,MAAM,CAAC,WAAW,EAAE,IAAI,CAAC,GAAG,CAAC,4BAA4B,CAAC;YAEtE,MAAMI,OAAOC,qBAAqB,IAAI,CAAC,kBAAkB,EAAE,IAAI,CAAC,GAAG;YACnE,IAAID,MACF,OAAOA;YAET,MAAM,IAAIJ,MACR,CAAC,WAAW,EAAE,IAAI,CAAC,GAAG,CAAC,4CAA4C,EAAE,IAAI,CAAC,kBAAkB,CAAC,CAAC,CAAC;QAEnG;QAGA,IAAI,IAAI,CAAC,cAAc,EAAE,YAAY,QACnC,OAAOD;QAGT,IAAI,IAAI,CAAC,cAAc,EAAE,YAAY,UACnC,OAAOI;QAIT,IAAI,AAAwB,SAAxB,IAAI,CAAC,cAAc,EACrB,OAAOJ;QAGT,IAAI,AAA4B,SAA5B,IAAI,CAAC,kBAAkB,EACzB,OAAOI;QAGT,MAAM,IAAIH,MACR,CAAC,WAAW,EAAE,IAAI,CAAC,GAAG,CAAC,4CAA4C,CAAC;IAExE;IAGA,YAAqB;QACnB,OAAO,AAAiB,SAAjB,IAAI,CAAC,OAAO;IACrB;IAOA,oBAAoBM,QAAgB,EAAiB;QACnD,MAAMC,MAAM,IAAI,CAAC,SAAS,CAAC;QAC3B,IAAI,CAAC,cAAc,GAAGA;QACtB,IAAI,CAAC,kBAAkB,GAAGD;QAC1B,IAAI,CAAC,OAAO,GAAG;QACf,OAAOC;IACT;IAMA,0BACEC,YAAoB,EACpBC,YAAoB,EACL;QACf,MAAMF,MAAM,IAAI,CAAC,SAAS,CAAC,QAAQC;QACnC,IAAI,CAAC,cAAc,GAAGC;QACtB,IAAI,CAAC,OAAO,GAAG;QACf,OAAOF;IACT;IAQA,oBACEC,YAAoB,EACpBC,YAAoB,EACL;QACf,MAAMF,MAAM,IAAI,CAAC,yBAAyB,CAACC,cAAcC;QACzD,IAAI,CAAC,cAAc,GAAGF;QACtB,OAAOA;IACT;IAGA,iBAA4C;QAC1C,OACE,IAAI,CAAC,cAAc,IAAI;YACrB,MAAM;YACN,IAAI,IAAI,CAAC,GAAG;YACZ,YAAY,IAAI,CAAC,WAAW;YAC5B,UAAU,AAAiB,WAAjB,IAAI,CAAC,OAAO,GAAc,eAAe;YACnD,SAAS;QACX;IAEJ;IAGA,OAAO,aAAaG,KAAc,EAAsC;QACtE,OAAOC,AAAkC,SAAlCA,uBAAuBD;IAChC;IAEQ,UACNE,OAA0B,EAC1BJ,YAAqB,EACN;QACf,MAAMK,UAAuC;YAC3C,MAAM;YACN,IAAI,IAAI,CAAC,GAAG;YACZ,YAAY,IAAI,CAAC,WAAW;YAC5B,UAAU,AAAiB,WAAjB,IAAI,CAAC,OAAO,GAAc,eAAe;YACnDD;QACF;QACA,IAAIA,AAAY,WAAZA,SACF,OAAO;YACL,GAAGC,OAAO;YACVD;YACA,MAAMJ;QACR;QAEF,OAAOK;IACT;IAMA,IAAI,YAAoB;QACtB,OAAO,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,uCAAuC;IACpE;IA7KA,YAAoBC,EAAU,EAAEnB,MAAc,EAAEE,UAAkB,CAAE;QARpE,uBAAQ,OAAR;QACA,uBAAQ,WAAR;QACA,uBAAQ,WAAR;QACA,uBAAQ,eAAR;QACA,uBAAQ,kBAAuC;QAC/C,uBAAQ,kBAAgC;QACxC,uBAAQ,sBAAoC;QAG1C,IAAI,CAAC,GAAG,GAAGiB;QACX,IAAI,CAAC,OAAO,GAAGnB;QACf,IAAI,CAAC,OAAO,GAAGD,aAAaC;QAC5B,IAAI,CAAC,WAAW,GAAGE;IACrB;AAyKF"}
+{"version":3,"file":"screenshot-item.mjs","sources":["../../src/screenshot-item.ts"],"sourcesContent":["import { readFileSync } from 'node:fs';\nimport { uuid } from '@aiscene/shared/utils';\nimport { extractImageByIdSync } from './dump/html-utils';\nimport {\n  type ScreenshotRef,\n  normalizeScreenshotRef,\n} from './dump/screenshot-store';\n\n/**\n * Serialization format for ScreenshotItem\n * - { $screenshot: \"id\" } - inline mode, references imageMap in HTML\n * - { base64: \"path\" } - directory mode, references external file path\n */\nexport type ScreenshotSerializeFormat = ScreenshotRef;\n\n/**\n * Detect image format from base64 data URI prefix.\n */\nfunction detectFormat(base64: string): 'png' | 'jpeg' {\n  if (base64.startsWith('data:image/jpeg')) return 'jpeg';\n  if (base64.startsWith('data:image/jpg')) return 'jpeg';\n  return 'png';\n}\n\n/**\n * ScreenshotItem encapsulates screenshot data.\n *\n * Supports lazy loading after memory release:\n * - inline mode: reads from HTML file using streaming (extractImageByIdSync)\n * - directory mode: reads from file on disk\n *\n * After persistence, memory is released but the screenshot can be recovered\n * on-demand from disk, making it safe to release memory at any time.\n */\nexport class ScreenshotItem {\n  private _id: string;\n  private _base64: string | null;\n  private _format: 'png' | 'jpeg';\n  private _capturedAt: number;\n  private _serializedRef: ScreenshotRef | null = null;\n  private _persistedPath: string | null = null;\n  private _persistedHtmlPath: string | null = null;\n\n  private constructor(id: string, base64: string, capturedAt: number) {\n    this._id = id;\n    this._base64 = base64;\n    this._format = detectFormat(base64);\n    this._capturedAt = capturedAt;\n  }\n\n  /** Create a new ScreenshotItem from base64 data */\n  static create(base64: string, capturedAt: number): ScreenshotItem {\n    return new ScreenshotItem(uuid(), base64, capturedAt);\n  }\n\n  get id(): string {\n    return this._id;\n  }\n\n  /** Get the image format (png or jpeg) */\n  get format(): 'png' | 'jpeg' {\n    return this._format;\n  }\n\n  /** Get the file extension for this screenshot */\n  get extension(): string {\n    return this._format === 'jpeg' ? 'jpeg' : 'png';\n  }\n\n  /** Get screenshot capture timestamp in milliseconds */\n  get capturedAt(): number {\n    return this._capturedAt;\n  }\n\n  get base64(): string {\n    // If data is in memory, return it directly\n    if (this._base64 !== null) {\n      return this._base64;\n    }\n\n    const loadFromFile = (): string => {\n      if (this._persistedPath === null) {\n        throw new Error(`Screenshot ${this._id}: file recovery path missing`);\n      }\n      const buffer = readFileSync(this._persistedPath);\n      return `data:image/${this._format};base64,${buffer.toString('base64')}`;\n    };\n\n    const loadFromInline = (): string => {\n      if (this._persistedHtmlPath === null) {\n        throw new Error(`Screenshot ${this._id}: HTML recovery path missing`);\n      }\n      const data = extractImageByIdSync(this._persistedHtmlPath, this._id);\n      if (data) {\n        return data;\n      }\n      throw new Error(\n        `Screenshot ${this._id}: cannot recover from HTML (id not found in ${this._persistedHtmlPath})`,\n      );\n    };\n\n    // Recover from the primary serialized mode first.\n    if (this._serializedRef?.storage === 'file') {\n      return loadFromFile();\n    }\n\n    if (this._serializedRef?.storage === 'inline') {\n      return loadFromInline();\n    }\n\n    // Fall back to whichever recovery path is available.\n    if (this._persistedPath !== null) {\n      return loadFromFile();\n    }\n\n    if (this._persistedHtmlPath !== null) {\n      return loadFromInline();\n    }\n\n    throw new Error(\n      `Screenshot ${this._id}: base64 data released without recovery path`,\n    );\n  }\n\n  /** Check if base64 data is still available in memory (not yet released) */\n  hasBase64(): boolean {\n    return this._base64 !== null;\n  }\n\n  /**\n   * Mark as persisted to HTML (inline mode).\n   * Releases base64 memory, but keeps HTML path for lazy loading recovery.\n   * @param htmlPath - absolute path to the HTML file containing the image\n   */\n  markPersistedInline(htmlPath: string): ScreenshotRef {\n    const ref = this.createRef('inline');\n    this._serializedRef = ref;\n    this._persistedHtmlPath = htmlPath;\n    this._base64 = null;\n    return ref;\n  }\n\n  /**\n   * Register a file-backed recovery path without changing the serialized mode.\n   * Used when inline persistence also needs a shared file copy next to dumps.\n   */\n  registerPersistedFileCopy(\n    relativePath: string,\n    absolutePath: string,\n  ): ScreenshotRef {\n    const ref = this.createRef('file', relativePath);\n    this._persistedPath = absolutePath;\n    this._base64 = null;\n    return ref;\n  }\n\n  /**\n   * Mark as persisted to file (directory mode).\n   * Releases base64 memory, but keeps file path for lazy loading recovery.\n   * @param relativePath - relative path for serialization (e.g., \"./screenshots/id.jpeg\")\n   * @param absolutePath - absolute path for lazy loading recovery\n   */\n  markPersistedToPath(\n    relativePath: string,\n    absolutePath: string,\n  ): ScreenshotRef {\n    const ref = this.registerPersistedFileCopy(relativePath, absolutePath);\n    this._serializedRef = ref;\n    return ref;\n  }\n\n  /** Serialize for JSON - format depends on persistence state */\n  toSerializable(): ScreenshotSerializeFormat {\n    return (\n      this._serializedRef ?? {\n        type: 'midscene_screenshot_ref',\n        id: this._id,\n        capturedAt: this._capturedAt,\n        mimeType: this._format === 'jpeg' ? 'image/jpeg' : 'image/png',\n        storage: 'inline',\n      }\n    );\n  }\n\n  /** Check if a value is a serialized ScreenshotItem reference (inline or directory mode) */\n  static isSerialized(value: unknown): value is ScreenshotSerializeFormat {\n    return normalizeScreenshotRef(value) !== null;\n  }\n\n  private createRef(\n    storage: 'inline' | 'file',\n    relativePath?: string,\n  ): ScreenshotRef {\n    const baseRef: Omit<ScreenshotRef, 'path'> = {\n      type: 'midscene_screenshot_ref',\n      id: this._id,\n      capturedAt: this._capturedAt,\n      mimeType: this._format === 'jpeg' ? 'image/jpeg' : 'image/png',\n      storage,\n    };\n    if (storage === 'file') {\n      return {\n        ...baseRef,\n        storage,\n        path: relativePath!,\n      };\n    }\n    return baseRef;\n  }\n\n  /**\n   * Get base64 data without the data URI prefix.\n   * Useful for writing raw binary data to files.\n   */\n  get rawBase64(): string {\n    return this.base64.replace(/^data:image\\/(png|jpeg|jpg);base64,/, '');\n  }\n}\n"],"names":["detectFormat","base64","ScreenshotItem","capturedAt","uuid","loadFromFile","Error","buffer","readFileSync","loadFromInline","data","extractImageByIdSync","htmlPath","ref","relativePath","absolutePath","value","normalizeScreenshotRef","storage","baseRef","id"],"mappings":";;;;;;;;;;;;;;AAkBA,SAASA,aAAaC,MAAc;IAClC,IAAIA,OAAO,UAAU,CAAC,oBAAoB,OAAO;IACjD,IAAIA,OAAO,UAAU,CAAC,mBAAmB,OAAO;IAChD,OAAO;AACT;AAYO,MAAMC;IAiBX,OAAO,OAAOD,MAAc,EAAEE,UAAkB,EAAkB;QAChE,OAAO,IAAID,eAAeE,QAAQH,QAAQE;IAC5C;IAEA,IAAI,KAAa;QACf,OAAO,IAAI,CAAC,GAAG;IACjB;IAGA,IAAI,SAAyB;QAC3B,OAAO,IAAI,CAAC,OAAO;IACrB;IAGA,IAAI,YAAoB;QACtB,OAAO,AAAiB,WAAjB,IAAI,CAAC,OAAO,GAAc,SAAS;IAC5C;IAGA,IAAI,aAAqB;QACvB,OAAO,IAAI,CAAC,WAAW;IACzB;IAEA,IAAI,SAAiB;QAEnB,IAAI,AAAiB,SAAjB,IAAI,CAAC,OAAO,EACd,OAAO,IAAI,CAAC,OAAO;QAGrB,MAAME,eAAe;YACnB,IAAI,AAAwB,SAAxB,IAAI,CAAC,cAAc,EACrB,MAAM,IAAIC,MAAM,CAAC,WAAW,EAAE,IAAI,CAAC,GAAG,CAAC,4BAA4B,CAAC;YAEtE,MAAMC,SAASC,aAAa,IAAI,CAAC,cAAc;YAC/C,OAAO,CAAC,WAAW,EAAE,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAED,OAAO,QAAQ,CAAC,WAAW;QACzE;QAEA,MAAME,iBAAiB;YACrB,IAAI,AAA4B,SAA5B,IAAI,CAAC,kBAAkB,EACzB,MAAM,IAAIH,MAAM,CAAC,WAAW,EAAE,IAAI,CAAC,GAAG,CAAC,4BAA4B,CAAC;YAEtE,MAAMI,OAAOC,qBAAqB,IAAI,CAAC,kBAAkB,EAAE,IAAI,CAAC,GAAG;YACnE,IAAID,MACF,OAAOA;YAET,MAAM,IAAIJ,MACR,CAAC,WAAW,EAAE,IAAI,CAAC,GAAG,CAAC,4CAA4C,EAAE,IAAI,CAAC,kBAAkB,CAAC,CAAC,CAAC;QAEnG;QAGA,IAAI,IAAI,CAAC,cAAc,EAAE,YAAY,QACnC,OAAOD;QAGT,IAAI,IAAI,CAAC,cAAc,EAAE,YAAY,UACnC,OAAOI;QAIT,IAAI,AAAwB,SAAxB,IAAI,CAAC,cAAc,EACrB,OAAOJ;QAGT,IAAI,AAA4B,SAA5B,IAAI,CAAC,kBAAkB,EACzB,OAAOI;QAGT,MAAM,IAAIH,MACR,CAAC,WAAW,EAAE,IAAI,CAAC,GAAG,CAAC,4CAA4C,CAAC;IAExE;IAGA,YAAqB;QACnB,OAAO,AAAiB,SAAjB,IAAI,CAAC,OAAO;IACrB;IAOA,oBAAoBM,QAAgB,EAAiB;QACnD,MAAMC,MAAM,IAAI,CAAC,SAAS,CAAC;QAC3B,IAAI,CAAC,cAAc,GAAGA;QACtB,IAAI,CAAC,kBAAkB,GAAGD;QAC1B,IAAI,CAAC,OAAO,GAAG;QACf,OAAOC;IACT;IAMA,0BACEC,YAAoB,EACpBC,YAAoB,EACL;QACf,MAAMF,MAAM,IAAI,CAAC,SAAS,CAAC,QAAQC;QACnC,IAAI,CAAC,cAAc,GAAGC;QACtB,IAAI,CAAC,OAAO,GAAG;QACf,OAAOF;IACT;IAQA,oBACEC,YAAoB,EACpBC,YAAoB,EACL;QACf,MAAMF,MAAM,IAAI,CAAC,yBAAyB,CAACC,cAAcC;QACzD,IAAI,CAAC,cAAc,GAAGF;QACtB,OAAOA;IACT;IAGA,iBAA4C;QAC1C,OACE,IAAI,CAAC,cAAc,IAAI;YACrB,MAAM;YACN,IAAI,IAAI,CAAC,GAAG;YACZ,YAAY,IAAI,CAAC,WAAW;YAC5B,UAAU,AAAiB,WAAjB,IAAI,CAAC,OAAO,GAAc,eAAe;YACnD,SAAS;QACX;IAEJ;IAGA,OAAO,aAAaG,KAAc,EAAsC;QACtE,OAAOC,AAAkC,SAAlCA,uBAAuBD;IAChC;IAEQ,UACNE,OAA0B,EAC1BJ,YAAqB,EACN;QACf,MAAMK,UAAuC;YAC3C,MAAM;YACN,IAAI,IAAI,CAAC,GAAG;YACZ,YAAY,IAAI,CAAC,WAAW;YAC5B,UAAU,AAAiB,WAAjB,IAAI,CAAC,OAAO,GAAc,eAAe;YACnDD;QACF;QACA,IAAIA,AAAY,WAAZA,SACF,OAAO;YACL,GAAGC,OAAO;YACVD;YACA,MAAMJ;QACR;QAEF,OAAOK;IACT;IAMA,IAAI,YAAoB;QACtB,OAAO,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,uCAAuC;IACpE;IA7KA,YAAoBC,EAAU,EAAEnB,MAAc,EAAEE,UAAkB,CAAE;QARpE,uBAAQ,OAAR;QACA,uBAAQ,WAAR;QACA,uBAAQ,WAAR;QACA,uBAAQ,eAAR;QACA,uBAAQ,kBAAuC;QAC/C,uBAAQ,kBAAgC;QACxC,uBAAQ,sBAAoC;QAG1C,IAAI,CAAC,GAAG,GAAGiB;QACX,IAAI,CAAC,OAAO,GAAGnB;QACf,IAAI,CAAC,OAAO,GAAGD,aAAaC;QAC5B,IAAI,CAAC,WAAW,GAAGE;IACrB;AAyKF"}

package/dist/es/service/index.mjs

@@ -4,9 +4,9 @@ import { AiLocateSection, buildSearchAreaConfig } from "../ai-model/inspect.mjs"
 import { elementDescriberInstruction } from "../ai-model/prompt/describe.mjs";
 import { expandSearchArea } from "../common.mjs";
 import { ServiceError } from "../types.mjs";
-import { compositeElementInfoImg, cropByRect } from "@midscene/shared/img";
-import { getDebug } from "@midscene/shared/logger";
-import { assert } from "@midscene/shared/utils";
+import { compositeElementInfoImg, cropByRect } from "@aiscene/shared/img";
+import { getDebug } from "@aiscene/shared/logger";
+import { assert } from "@aiscene/shared/utils";
 import { createServiceDump } from "./utils.mjs";
 function _define_property(obj, key, value) {
     if (key in obj) Object.defineProperty(obj, key, {

package/dist/es/service/index.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"service/index.mjs","sources":["../../../src/service/index.ts"],"sourcesContent":["import { isAutoGLM, isUITars } from '@/ai-model/auto-glm/util';\nimport {\n  AIResponseParseError,\n  AiExtractElementInfo,\n  AiLocateElement,\n  callAIWithObjectResponse,\n} from '@/ai-model/index';\nimport { AiLocateSection, buildSearchAreaConfig } from '@/ai-model/inspect';\nimport { elementDescriberInstruction } from '@/ai-model/prompt/describe';\nimport { type AIArgs, expandSearchArea } from '@/common';\nimport type {\n  AIDescribeElementResponse,\n  AIUsageInfo,\n  DetailedLocateParam,\n  LocateResultElement,\n  LocateResultWithDump,\n  PartialServiceDumpFromSDK,\n  PlanningLocateParam,\n  Rect,\n  ServiceExtractOption,\n  ServiceExtractParam,\n  ServiceExtractResult,\n  ServiceTaskInfo,\n  UIContext,\n} from '@/types';\nimport { ServiceError } from '@/types';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport { compositeElementInfoImg, cropByRect } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TMultimodalPrompt } from '../common';\nimport { createServiceDump } from './utils';\n\nexport interface LocateOpts {\n  context?: UIContext;\n  planLocatedElement?: LocateResultElement;\n}\n\nexport type AnyValue<T> = {\n  [K in keyof T]: unknown extends T[K] ? any : T[K];\n};\n\ninterface ServiceOptions {\n  taskInfo?: Omit<ServiceTaskInfo, 'durationMs'>;\n}\n\nconst debug = getDebug('ai:service');\nexport default class Service {\n  contextRetrieverFn: () => Promise<UIContext> | UIContext;\n\n  taskInfo?: Omit<ServiceTaskInfo, 'durationMs'>;\n\n  constructor(\n    context: UIContext | (() => Promise<UIContext> | UIContext),\n    opt?: ServiceOptions,\n  ) {\n    assert(context, 'context is required for Service');\n    if (typeof context === 'function') {\n      this.contextRetrieverFn = context;\n    } else {\n      this.contextRetrieverFn = () => Promise.resolve(context);\n    }\n\n    if (typeof opt?.taskInfo !== 'undefined') {\n      this.taskInfo = opt.taskInfo;\n    }\n  }\n\n  async locate(\n    query: PlanningLocateParam,\n    opt: LocateOpts,\n    modelConfig: IModelConfig,\n    abortSignal?: AbortSignal,\n  ): Promise<LocateResultWithDump> {\n    const queryPrompt = typeof query === 'string' ? query : query.prompt;\n    assert(queryPrompt, 'query is required for locate');\n\n    assert(typeof query === 'object', 'query should be an object for locate');\n\n    const hasPlanLocatedElement = !!opt?.planLocatedElement?.rect;\n\n    let searchAreaPrompt;\n    if (query.deepLocate && !hasPlanLocatedElement) {\n      searchAreaPrompt = query.prompt;\n    }\n\n    const { modelFamily } = modelConfig;\n\n    if (searchAreaPrompt && !modelFamily) {\n      console.warn(\n        'The \"deepLocate\" feature is not supported with multimodal LLM. Please config VL model for Midscene. https://midscenejs.com/model-config',\n      );\n      searchAreaPrompt = undefined;\n    }\n\n    if (searchAreaPrompt && isAutoGLM(modelFamily)) {\n      console.warn('The \"deepLocate\" feature is not supported with AutoGLM.');\n      searchAreaPrompt = undefined;\n    }\n\n    const context = opt?.context || (await this.contextRetrieverFn());\n\n    let searchArea: Rect | undefined = undefined;\n    let searchAreaRawResponse: string | undefined = undefined;\n    let searchAreaUsage: AIUsageInfo | undefined = undefined;\n    let searchAreaResponse:\n      | Awaited<ReturnType<typeof AiLocateSection>>\n      | undefined = undefined;\n    if (query.deepLocate && hasPlanLocatedElement) {\n      const searchAreaConfig = await buildSearchAreaConfig({\n        context,\n        baseRect: opt.planLocatedElement!.rect,\n        modelFamily,\n      });\n      searchArea = searchAreaConfig.rect;\n\n      searchAreaRawResponse = JSON.stringify({\n        source: 'plan-located-element',\n        rect: opt.planLocatedElement!.rect,\n      });\n      searchAreaResponse = {\n        rect: searchArea,\n        imageBase64: searchAreaConfig.imageBase64,\n        scale: searchAreaConfig.scale,\n        rawResponse: searchAreaRawResponse,\n      };\n    } else if (searchAreaPrompt) {\n      searchAreaResponse = await AiLocateSection({\n        context,\n        sectionDescription: searchAreaPrompt,\n        modelConfig,\n        abortSignal,\n      });\n      assert(\n        searchAreaResponse.rect,\n        `cannot find search area for \"${searchAreaPrompt}\"${\n          searchAreaResponse.error ? `: ${searchAreaResponse.error}` : ''\n        }`,\n      );\n      searchAreaRawResponse = searchAreaResponse.rawResponse;\n      searchAreaUsage = searchAreaResponse.usage;\n      searchArea = searchAreaResponse.rect;\n    }\n\n    const startTime = Date.now();\n    const { parseResult, rect, rawResponse, usage, reasoning_content } =\n      await AiLocateElement({\n        context,\n        targetElementDescription: queryPrompt,\n        searchConfig: searchAreaResponse,\n        modelConfig,\n        abortSignal,\n      });\n\n    const timeCost = Date.now() - startTime;\n    const taskInfo: ServiceTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse: JSON.stringify(rawResponse),\n      formatResponse: JSON.stringify(parseResult),\n      usage,\n      searchArea,\n      searchAreaRawResponse,\n      searchAreaUsage,\n      reasoning_content,\n    };\n\n    let errorLog: string | undefined;\n    if (parseResult.errors?.length) {\n      errorLog = `failed to locate element: \\n${parseResult.errors.join('\\n')}`;\n    }\n\n    const dumpData: PartialServiceDumpFromSDK = {\n      type: 'locate',\n      userQuery: {\n        element: queryPrompt,\n      },\n      matchedElement: [],\n      matchedRect: rect,\n      data: null,\n      taskInfo,\n      deepLocate: !!searchArea,\n      error: errorLog,\n    };\n\n    const elements = parseResult.elements || [];\n\n    const dump = createServiceDump({\n      ...dumpData,\n      matchedElement: elements,\n    });\n\n    if (errorLog) {\n      throw new ServiceError(errorLog, dump);\n    }\n\n    if (elements.length > 1) {\n      throw new ServiceError(\n        `locate: multiple elements found, length = ${elements.length}`,\n        dump,\n      );\n    }\n\n    if (elements.length === 1) {\n      return {\n        element: {\n          center: elements[0]!.center,\n          rect: elements[0]!.rect,\n          description: elements[0]!.description,\n        },\n        rect,\n        dump,\n      };\n    }\n\n    return {\n      element: null,\n      rect,\n      dump,\n    };\n  }\n\n  async extract<T>(\n    dataDemand: ServiceExtractParam,\n    modelConfig: IModelConfig,\n    opt?: ServiceExtractOption,\n    pageDescription?: string,\n    multimodalPrompt?: TMultimodalPrompt,\n    context?: UIContext,\n  ): Promise<ServiceExtractResult<T>> {\n    assert(context, 'context is required for extract');\n    assert(\n      typeof dataDemand === 'object' || typeof dataDemand === 'string',\n      `dataDemand should be object or string, but get ${typeof dataDemand}`,\n    );\n\n    const startTime = Date.now();\n\n    let parseResult: Awaited<\n      ReturnType<typeof AiExtractElementInfo<T>>\n    >['parseResult'];\n    let rawResponse: string;\n    let usage: Awaited<ReturnType<typeof AiExtractElementInfo<T>>>['usage'];\n    let reasoning_content: string | undefined;\n\n    try {\n      const result = await AiExtractElementInfo<T>({\n        context,\n        dataQuery: dataDemand,\n        multimodalPrompt,\n        extractOption: opt,\n        modelConfig,\n        pageDescription,\n      });\n      parseResult = result.parseResult;\n      rawResponse = result.rawResponse;\n      usage = result.usage;\n      reasoning_content = result.reasoning_content;\n    } catch (error) {\n      if (error instanceof AIResponseParseError) {\n        // Create dump with usage and rawResponse from the error\n        const timeCost = Date.now() - startTime;\n        const taskInfo: ServiceTaskInfo = {\n          ...(this.taskInfo ? this.taskInfo : {}),\n          durationMs: timeCost,\n          rawResponse: error.rawResponse,\n          usage: error.usage,\n        };\n        const dump = createServiceDump({\n          type: 'extract',\n          userQuery: { dataDemand },\n          matchedElement: [],\n          data: null,\n          taskInfo,\n          error: error.message,\n        });\n        throw new ServiceError(error.message, dump);\n      }\n      throw error;\n    }\n\n    const timeCost = Date.now() - startTime;\n    const taskInfo: ServiceTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse,\n      formatResponse: JSON.stringify(parseResult),\n      usage,\n      reasoning_content,\n    };\n\n    let errorLog: string | undefined;\n    if (parseResult.errors?.length) {\n      errorLog = `AI response error: \\n${parseResult.errors.join('\\n')}`;\n    }\n\n    const dumpData: PartialServiceDumpFromSDK = {\n      type: 'extract',\n      userQuery: {\n        dataDemand,\n      },\n      matchedElement: [],\n      data: null,\n      taskInfo,\n      error: errorLog,\n    };\n\n    const { data, thought } = parseResult || {};\n\n    const dump = createServiceDump({\n      ...dumpData,\n      data,\n    });\n\n    if (errorLog && !data) {\n      throw new ServiceError(errorLog, dump);\n    }\n\n    return {\n      data,\n      thought,\n      usage,\n      reasoning_content,\n      dump,\n    };\n  }\n\n  async describe(\n    target: Rect | [number, number],\n    modelConfig: IModelConfig,\n    opt?: {\n      deepLocate?: boolean;\n    },\n  ): Promise<Pick<AIDescribeElementResponse, 'description'>> {\n    assert(target, 'target is required for service.describe');\n    const context = await this.contextRetrieverFn();\n    const { shotSize } = context;\n    const screenshotBase64 = context.screenshot.base64;\n    assert(screenshotBase64, 'screenshot is required for service.describe');\n    // The result of the \"describe\" function will be used for positioning, so essentially it is a form of grounding.\n    const { modelFamily } = modelConfig;\n    const systemPrompt = elementDescriberInstruction();\n\n    // Convert [x,y] center point to Rect if needed\n    const defaultRectSize = 30;\n    const targetRect: Rect = Array.isArray(target)\n      ? {\n          left: Math.floor(target[0] - defaultRectSize / 2),\n          top: Math.floor(target[1] - defaultRectSize / 2),\n          width: defaultRectSize,\n          height: defaultRectSize,\n        }\n      : target;\n\n    let imagePayload = await compositeElementInfoImg({\n      inputImgBase64: screenshotBase64,\n      size: shotSize,\n      elementsPositionInfo: [\n        {\n          rect: targetRect,\n        },\n      ],\n      borderThickness: 3,\n    });\n\n    if (opt?.deepLocate) {\n      const searchArea = expandSearchArea(targetRect, shotSize);\n      // Always crop in describe mode. Unlike locate's deepLocate (where\n      // cropping too small loses context for finding elements), describe's\n      // deepLocate intentionally zooms in so the model produces a more\n      // precise description from a focused view. expandSearchArea already\n      // guarantees a minimum 400x400 area with surrounding context.\n      debug('describe: cropping to searchArea', searchArea);\n      const croppedResult = await cropByRect(\n        imagePayload,\n        searchArea,\n        modelFamily === 'qwen2.5-vl',\n      );\n      imagePayload = croppedResult.imageBase64;\n    }\n\n    const msgs: AIArgs = [\n      { role: 'system', content: systemPrompt },\n      {\n        role: 'user',\n        content: [\n          {\n            type: 'image_url',\n            image_url: {\n              url: imagePayload,\n              detail: 'high',\n            },\n          },\n        ],\n      },\n    ];\n\n    const res = await callAIWithObjectResponse<AIDescribeElementResponse>(\n      msgs,\n      modelConfig,\n    );\n\n    const { content } = res;\n    assert(!content.error, `describe failed: ${content.error}`);\n    assert(content.description, 'failed to describe the element');\n    return content;\n  }\n}\n"],"names":["debug","getDebug","Service","query","opt","modelConfig","abortSignal","queryPrompt","assert","hasPlanLocatedElement","searchAreaPrompt","modelFamily","console","undefined","isAutoGLM","context","searchArea","searchAreaRawResponse","searchAreaUsage","searchAreaResponse","searchAreaConfig","buildSearchAreaConfig","JSON","AiLocateSection","startTime","Date","parseResult","rect","rawResponse","usage","reasoning_content","AiLocateElement","timeCost","taskInfo","errorLog","dumpData","elements","dump","createServiceDump","ServiceError","dataDemand","pageDescription","multimodalPrompt","result","AiExtractElementInfo","error","AIResponseParseError","data","thought","target","shotSize","screenshotBase64","systemPrompt","elementDescriberInstruction","defaultRectSize","targetRect","Array","Math","imagePayload","compositeElementInfoImg","expandSearchArea","croppedResult","cropByRect","msgs","res","callAIWithObjectResponse","content","Promise"],"mappings":";;;;;;;;;;;;;;;;;;;;AA8CA,MAAMA,QAAQC,SAAS;AACR,MAAMC;IAqBnB,MAAM,OACJC,KAA0B,EAC1BC,GAAe,EACfC,WAAyB,EACzBC,WAAyB,EACM;QAC/B,MAAMC,cAAc,AAAiB,YAAjB,OAAOJ,QAAqBA,QAAQA,MAAM,MAAM;QACpEK,OAAOD,aAAa;QAEpBC,OAAO,AAAiB,YAAjB,OAAOL,OAAoB;QAElC,MAAMM,wBAAwB,CAAC,CAACL,KAAK,oBAAoB;QAEzD,IAAIM;QACJ,IAAIP,MAAM,UAAU,IAAI,CAACM,uBACvBC,mBAAmBP,MAAM,MAAM;QAGjC,MAAM,EAAEQ,WAAW,EAAE,GAAGN;QAExB,IAAIK,oBAAoB,CAACC,aAAa;YACpCC,QAAQ,IAAI,CACV;YAEFF,mBAAmBG;QACrB;QAEA,IAAIH,oBAAoBI,UAAUH,cAAc;YAC9CC,QAAQ,IAAI,CAAC;YACbF,mBAAmBG;QACrB;QAEA,MAAME,UAAUX,KAAK,WAAY,MAAM,IAAI,CAAC,kBAAkB;QAE9D,IAAIY;QACJ,IAAIC;QACJ,IAAIC;QACJ,IAAIC;QAGJ,IAAIhB,MAAM,UAAU,IAAIM,uBAAuB;YAC7C,MAAMW,mBAAmB,MAAMC,sBAAsB;gBACnDN;gBACA,UAAUX,IAAI,kBAAkB,CAAE,IAAI;gBACtCO;YACF;YACAK,aAAaI,iBAAiB,IAAI;YAElCH,wBAAwBK,KAAK,SAAS,CAAC;gBACrC,QAAQ;gBACR,MAAMlB,IAAI,kBAAkB,CAAE,IAAI;YACpC;YACAe,qBAAqB;gBACnB,MAAMH;gBACN,aAAaI,iBAAiB,WAAW;gBACzC,OAAOA,iBAAiB,KAAK;gBAC7B,aAAaH;YACf;QACF,OAAO,IAAIP,kBAAkB;YAC3BS,qBAAqB,MAAMI,gBAAgB;gBACzCR;gBACA,oBAAoBL;gBACpBL;gBACAC;YACF;YACAE,OACEW,mBAAmB,IAAI,EACvB,CAAC,6BAA6B,EAAET,iBAAiB,CAAC,EAChDS,mBAAmB,KAAK,GAAG,CAAC,EAAE,EAAEA,mBAAmB,KAAK,EAAE,GAAG,IAC7D;YAEJF,wBAAwBE,mBAAmB,WAAW;YACtDD,kBAAkBC,mBAAmB,KAAK;YAC1CH,aAAaG,mBAAmB,IAAI;QACtC;QAEA,MAAMK,YAAYC,KAAK,GAAG;QAC1B,MAAM,EAAEC,WAAW,EAAEC,IAAI,EAAEC,WAAW,EAAEC,KAAK,EAAEC,iBAAiB,EAAE,GAChE,MAAMC,gBAAgB;YACpBhB;YACA,0BAA0BR;YAC1B,cAAcY;YACdd;YACAC;QACF;QAEF,MAAM0B,WAAWP,KAAK,GAAG,KAAKD;QAC9B,MAAMS,WAA4B;YAChC,GAAI,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;YACtC,YAAYD;YACZ,aAAaV,KAAK,SAAS,CAACM;YAC5B,gBAAgBN,KAAK,SAAS,CAACI;YAC/BG;YACAb;YACAC;YACAC;YACAY;QACF;QAEA,IAAII;QACJ,IAAIR,YAAY,MAAM,EAAE,QACtBQ,WAAW,CAAC,4BAA4B,EAAER,YAAY,MAAM,CAAC,IAAI,CAAC,OAAO;QAG3E,MAAMS,WAAsC;YAC1C,MAAM;YACN,WAAW;gBACT,SAAS5B;YACX;YACA,gBAAgB,EAAE;YAClB,aAAaoB;YACb,MAAM;YACNM;YACA,YAAY,CAAC,CAACjB;YACd,OAAOkB;QACT;QAEA,MAAME,WAAWV,YAAY,QAAQ,IAAI,EAAE;QAE3C,MAAMW,OAAOC,kBAAkB;YAC7B,GAAGH,QAAQ;YACX,gBAAgBC;QAClB;QAEA,IAAIF,UACF,MAAM,IAAIK,aAAaL,UAAUG;QAGnC,IAAID,SAAS,MAAM,GAAG,GACpB,MAAM,IAAIG,aACR,CAAC,0CAA0C,EAAEH,SAAS,MAAM,EAAE,EAC9DC;QAIJ,IAAID,AAAoB,MAApBA,SAAS,MAAM,EACjB,OAAO;YACL,SAAS;gBACP,QAAQA,QAAQ,CAAC,EAAE,CAAE,MAAM;gBAC3B,MAAMA,QAAQ,CAAC,EAAE,CAAE,IAAI;gBACvB,aAAaA,QAAQ,CAAC,EAAE,CAAE,WAAW;YACvC;YACAT;YACAU;QACF;QAGF,OAAO;YACL,SAAS;YACTV;YACAU;QACF;IACF;IAEA,MAAM,QACJG,UAA+B,EAC/BnC,WAAyB,EACzBD,GAA0B,EAC1BqC,eAAwB,EACxBC,gBAAoC,EACpC3B,OAAmB,EACe;QAClCP,OAAOO,SAAS;QAChBP,OACE,AAAsB,YAAtB,OAAOgC,cAA2B,AAAsB,YAAtB,OAAOA,YACzC,CAAC,+CAA+C,EAAE,OAAOA,YAAY;QAGvE,MAAMhB,YAAYC,KAAK,GAAG;QAE1B,IAAIC;QAGJ,IAAIE;QACJ,IAAIC;QACJ,IAAIC;QAEJ,IAAI;YACF,MAAMa,SAAS,MAAMC,qBAAwB;gBAC3C7B;gBACA,WAAWyB;gBACXE;gBACA,eAAetC;gBACfC;gBACAoC;YACF;YACAf,cAAciB,OAAO,WAAW;YAChCf,cAAce,OAAO,WAAW;YAChCd,QAAQc,OAAO,KAAK;YACpBb,oBAAoBa,OAAO,iBAAiB;QAC9C,EAAE,OAAOE,OAAO;YACd,IAAIA,iBAAiBC,sBAAsB;gBAEzC,MAAMd,WAAWP,KAAK,GAAG,KAAKD;gBAC9B,MAAMS,WAA4B;oBAChC,GAAI,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;oBACtC,YAAYD;oBACZ,aAAaa,MAAM,WAAW;oBAC9B,OAAOA,MAAM,KAAK;gBACpB;gBACA,MAAMR,OAAOC,kBAAkB;oBAC7B,MAAM;oBACN,WAAW;wBAAEE;oBAAW;oBACxB,gBAAgB,EAAE;oBAClB,MAAM;oBACNP;oBACA,OAAOY,MAAM,OAAO;gBACtB;gBACA,MAAM,IAAIN,aAAaM,MAAM,OAAO,EAAER;YACxC;YACA,MAAMQ;QACR;QAEA,MAAMb,WAAWP,KAAK,GAAG,KAAKD;QAC9B,MAAMS,WAA4B;YAChC,GAAI,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;YACtC,YAAYD;YACZJ;YACA,gBAAgBN,KAAK,SAAS,CAACI;YAC/BG;YACAC;QACF;QAEA,IAAII;QACJ,IAAIR,YAAY,MAAM,EAAE,QACtBQ,WAAW,CAAC,qBAAqB,EAAER,YAAY,MAAM,CAAC,IAAI,CAAC,OAAO;QAGpE,MAAMS,WAAsC;YAC1C,MAAM;YACN,WAAW;gBACTK;YACF;YACA,gBAAgB,EAAE;YAClB,MAAM;YACNP;YACA,OAAOC;QACT;QAEA,MAAM,EAAEa,IAAI,EAAEC,OAAO,EAAE,GAAGtB,eAAe,CAAC;QAE1C,MAAMW,OAAOC,kBAAkB;YAC7B,GAAGH,QAAQ;YACXY;QACF;QAEA,IAAIb,YAAY,CAACa,MACf,MAAM,IAAIR,aAAaL,UAAUG;QAGnC,OAAO;YACLU;YACAC;YACAnB;YACAC;YACAO;QACF;IACF;IAEA,MAAM,SACJY,MAA+B,EAC/B5C,WAAyB,EACzBD,GAEC,EACwD;QACzDI,OAAOyC,QAAQ;QACf,MAAMlC,UAAU,MAAM,IAAI,CAAC,kBAAkB;QAC7C,MAAM,EAAEmC,QAAQ,EAAE,GAAGnC;QACrB,MAAMoC,mBAAmBpC,QAAQ,UAAU,CAAC,MAAM;QAClDP,OAAO2C,kBAAkB;QAEzB,MAAM,EAAExC,WAAW,EAAE,GAAGN;QACxB,MAAM+C,eAAeC;QAGrB,MAAMC,kBAAkB;QACxB,MAAMC,aAAmBC,MAAM,OAAO,CAACP,UACnC;YACE,MAAMQ,KAAK,KAAK,CAACR,MAAM,CAAC,EAAE,GAAGK,kBAAkB;YAC/C,KAAKG,KAAK,KAAK,CAACR,MAAM,CAAC,EAAE,GAAGK,kBAAkB;YAC9C,OAAOA;YACP,QAAQA;QACV,IACAL;QAEJ,IAAIS,eAAe,MAAMC,wBAAwB;YAC/C,gBAAgBR;YAChB,MAAMD;YACN,sBAAsB;gBACpB;oBACE,MAAMK;gBACR;aACD;YACD,iBAAiB;QACnB;QAEA,IAAInD,KAAK,YAAY;YACnB,MAAMY,aAAa4C,iBAAiBL,YAAYL;YAMhDlD,MAAM,oCAAoCgB;YAC1C,MAAM6C,gBAAgB,MAAMC,WAC1BJ,cACA1C,YACAL,AAAgB,iBAAhBA;YAEF+C,eAAeG,cAAc,WAAW;QAC1C;QAEA,MAAME,OAAe;YACnB;gBAAE,MAAM;gBAAU,SAASX;YAAa;YACxC;gBACE,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,WAAW;4BACT,KAAKM;4BACL,QAAQ;wBACV;oBACF;iBACD;YACH;SACD;QAED,MAAMM,MAAM,MAAMC,yBAChBF,MACA1D;QAGF,MAAM,EAAE6D,OAAO,EAAE,GAAGF;QACpBxD,OAAO,CAAC0D,QAAQ,KAAK,EAAE,CAAC,iBAAiB,EAAEA,QAAQ,KAAK,EAAE;QAC1D1D,OAAO0D,QAAQ,WAAW,EAAE;QAC5B,OAAOA;IACT;IAlWA,YACEnD,OAA2D,EAC3DX,GAAoB,CACpB;QAPF;QAEA;QAMEI,OAAOO,SAAS;QAChB,IAAI,AAAmB,cAAnB,OAAOA,SACT,IAAI,CAAC,kBAAkB,GAAGA;aAE1B,IAAI,CAAC,kBAAkB,GAAG,IAAMoD,QAAQ,OAAO,CAACpD;QAGlD,IAAI,AAAyB,WAAlBX,KAAK,UACd,IAAI,CAAC,QAAQ,GAAGA,IAAI,QAAQ;IAEhC;AAqVF"}
+{"version":3,"file":"service/index.mjs","sources":["../../../src/service/index.ts"],"sourcesContent":["import { isAutoGLM, isUITars } from '@/ai-model/auto-glm/util';\nimport {\n  AIResponseParseError,\n  AiExtractElementInfo,\n  AiLocateElement,\n  callAIWithObjectResponse,\n} from '@/ai-model/index';\nimport { AiLocateSection, buildSearchAreaConfig } from '@/ai-model/inspect';\nimport { elementDescriberInstruction } from '@/ai-model/prompt/describe';\nimport { type AIArgs, expandSearchArea } from '@/common';\nimport type {\n  AIDescribeElementResponse,\n  AIUsageInfo,\n  DetailedLocateParam,\n  LocateResultElement,\n  LocateResultWithDump,\n  PartialServiceDumpFromSDK,\n  PlanningLocateParam,\n  Rect,\n  ServiceExtractOption,\n  ServiceExtractParam,\n  ServiceExtractResult,\n  ServiceTaskInfo,\n  UIContext,\n} from '@/types';\nimport { ServiceError } from '@/types';\nimport type { IModelConfig } from '@aiscene/shared/env';\nimport { compositeElementInfoImg, cropByRect } from '@aiscene/shared/img';\nimport { getDebug } from '@aiscene/shared/logger';\nimport { assert } from '@aiscene/shared/utils';\nimport type { TMultimodalPrompt } from '../common';\nimport { createServiceDump } from './utils';\n\nexport interface LocateOpts {\n  context?: UIContext;\n  planLocatedElement?: LocateResultElement;\n}\n\nexport type AnyValue<T> = {\n  [K in keyof T]: unknown extends T[K] ? any : T[K];\n};\n\ninterface ServiceOptions {\n  taskInfo?: Omit<ServiceTaskInfo, 'durationMs'>;\n}\n\nconst debug = getDebug('ai:service');\nexport default class Service {\n  contextRetrieverFn: () => Promise<UIContext> | UIContext;\n\n  taskInfo?: Omit<ServiceTaskInfo, 'durationMs'>;\n\n  constructor(\n    context: UIContext | (() => Promise<UIContext> | UIContext),\n    opt?: ServiceOptions,\n  ) {\n    assert(context, 'context is required for Service');\n    if (typeof context === 'function') {\n      this.contextRetrieverFn = context;\n    } else {\n      this.contextRetrieverFn = () => Promise.resolve(context);\n    }\n\n    if (typeof opt?.taskInfo !== 'undefined') {\n      this.taskInfo = opt.taskInfo;\n    }\n  }\n\n  async locate(\n    query: PlanningLocateParam,\n    opt: LocateOpts,\n    modelConfig: IModelConfig,\n    abortSignal?: AbortSignal,\n  ): Promise<LocateResultWithDump> {\n    const queryPrompt = typeof query === 'string' ? query : query.prompt;\n    assert(queryPrompt, 'query is required for locate');\n\n    assert(typeof query === 'object', 'query should be an object for locate');\n\n    const hasPlanLocatedElement = !!opt?.planLocatedElement?.rect;\n\n    let searchAreaPrompt;\n    if (query.deepLocate && !hasPlanLocatedElement) {\n      searchAreaPrompt = query.prompt;\n    }\n\n    const { modelFamily } = modelConfig;\n\n    if (searchAreaPrompt && !modelFamily) {\n      console.warn(\n        'The \"deepLocate\" feature is not supported with multimodal LLM. Please config VL model for Midscene. https://midscenejs.com/model-config',\n      );\n      searchAreaPrompt = undefined;\n    }\n\n    if (searchAreaPrompt && isAutoGLM(modelFamily)) {\n      console.warn('The \"deepLocate\" feature is not supported with AutoGLM.');\n      searchAreaPrompt = undefined;\n    }\n\n    const context = opt?.context || (await this.contextRetrieverFn());\n\n    let searchArea: Rect | undefined = undefined;\n    let searchAreaRawResponse: string | undefined = undefined;\n    let searchAreaUsage: AIUsageInfo | undefined = undefined;\n    let searchAreaResponse:\n      | Awaited<ReturnType<typeof AiLocateSection>>\n      | undefined = undefined;\n    if (query.deepLocate && hasPlanLocatedElement) {\n      const searchAreaConfig = await buildSearchAreaConfig({\n        context,\n        baseRect: opt.planLocatedElement!.rect,\n        modelFamily,\n      });\n      searchArea = searchAreaConfig.rect;\n\n      searchAreaRawResponse = JSON.stringify({\n        source: 'plan-located-element',\n        rect: opt.planLocatedElement!.rect,\n      });\n      searchAreaResponse = {\n        rect: searchArea,\n        imageBase64: searchAreaConfig.imageBase64,\n        scale: searchAreaConfig.scale,\n        rawResponse: searchAreaRawResponse,\n      };\n    } else if (searchAreaPrompt) {\n      searchAreaResponse = await AiLocateSection({\n        context,\n        sectionDescription: searchAreaPrompt,\n        modelConfig,\n        abortSignal,\n      });\n      assert(\n        searchAreaResponse.rect,\n        `cannot find search area for \"${searchAreaPrompt}\"${\n          searchAreaResponse.error ? `: ${searchAreaResponse.error}` : ''\n        }`,\n      );\n      searchAreaRawResponse = searchAreaResponse.rawResponse;\n      searchAreaUsage = searchAreaResponse.usage;\n      searchArea = searchAreaResponse.rect;\n    }\n\n    const startTime = Date.now();\n    const { parseResult, rect, rawResponse, usage, reasoning_content } =\n      await AiLocateElement({\n        context,\n        targetElementDescription: queryPrompt,\n        searchConfig: searchAreaResponse,\n        modelConfig,\n        abortSignal,\n      });\n\n    const timeCost = Date.now() - startTime;\n    const taskInfo: ServiceTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse: JSON.stringify(rawResponse),\n      formatResponse: JSON.stringify(parseResult),\n      usage,\n      searchArea,\n      searchAreaRawResponse,\n      searchAreaUsage,\n      reasoning_content,\n    };\n\n    let errorLog: string | undefined;\n    if (parseResult.errors?.length) {\n      errorLog = `failed to locate element: \\n${parseResult.errors.join('\\n')}`;\n    }\n\n    const dumpData: PartialServiceDumpFromSDK = {\n      type: 'locate',\n      userQuery: {\n        element: queryPrompt,\n      },\n      matchedElement: [],\n      matchedRect: rect,\n      data: null,\n      taskInfo,\n      deepLocate: !!searchArea,\n      error: errorLog,\n    };\n\n    const elements = parseResult.elements || [];\n\n    const dump = createServiceDump({\n      ...dumpData,\n      matchedElement: elements,\n    });\n\n    if (errorLog) {\n      throw new ServiceError(errorLog, dump);\n    }\n\n    if (elements.length > 1) {\n      throw new ServiceError(\n        `locate: multiple elements found, length = ${elements.length}`,\n        dump,\n      );\n    }\n\n    if (elements.length === 1) {\n      return {\n        element: {\n          center: elements[0]!.center,\n          rect: elements[0]!.rect,\n          description: elements[0]!.description,\n        },\n        rect,\n        dump,\n      };\n    }\n\n    return {\n      element: null,\n      rect,\n      dump,\n    };\n  }\n\n  async extract<T>(\n    dataDemand: ServiceExtractParam,\n    modelConfig: IModelConfig,\n    opt?: ServiceExtractOption,\n    pageDescription?: string,\n    multimodalPrompt?: TMultimodalPrompt,\n    context?: UIContext,\n  ): Promise<ServiceExtractResult<T>> {\n    assert(context, 'context is required for extract');\n    assert(\n      typeof dataDemand === 'object' || typeof dataDemand === 'string',\n      `dataDemand should be object or string, but get ${typeof dataDemand}`,\n    );\n\n    const startTime = Date.now();\n\n    let parseResult: Awaited<\n      ReturnType<typeof AiExtractElementInfo<T>>\n    >['parseResult'];\n    let rawResponse: string;\n    let usage: Awaited<ReturnType<typeof AiExtractElementInfo<T>>>['usage'];\n    let reasoning_content: string | undefined;\n\n    try {\n      const result = await AiExtractElementInfo<T>({\n        context,\n        dataQuery: dataDemand,\n        multimodalPrompt,\n        extractOption: opt,\n        modelConfig,\n        pageDescription,\n      });\n      parseResult = result.parseResult;\n      rawResponse = result.rawResponse;\n      usage = result.usage;\n      reasoning_content = result.reasoning_content;\n    } catch (error) {\n      if (error instanceof AIResponseParseError) {\n        // Create dump with usage and rawResponse from the error\n        const timeCost = Date.now() - startTime;\n        const taskInfo: ServiceTaskInfo = {\n          ...(this.taskInfo ? this.taskInfo : {}),\n          durationMs: timeCost,\n          rawResponse: error.rawResponse,\n          usage: error.usage,\n        };\n        const dump = createServiceDump({\n          type: 'extract',\n          userQuery: { dataDemand },\n          matchedElement: [],\n          data: null,\n          taskInfo,\n          error: error.message,\n        });\n        throw new ServiceError(error.message, dump);\n      }\n      throw error;\n    }\n\n    const timeCost = Date.now() - startTime;\n    const taskInfo: ServiceTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse,\n      formatResponse: JSON.stringify(parseResult),\n      usage,\n      reasoning_content,\n    };\n\n    let errorLog: string | undefined;\n    if (parseResult.errors?.length) {\n      errorLog = `AI response error: \\n${parseResult.errors.join('\\n')}`;\n    }\n\n    const dumpData: PartialServiceDumpFromSDK = {\n      type: 'extract',\n      userQuery: {\n        dataDemand,\n      },\n      matchedElement: [],\n      data: null,\n      taskInfo,\n      error: errorLog,\n    };\n\n    const { data, thought } = parseResult || {};\n\n    const dump = createServiceDump({\n      ...dumpData,\n      data,\n    });\n\n    if (errorLog && !data) {\n      throw new ServiceError(errorLog, dump);\n    }\n\n    return {\n      data,\n      thought,\n      usage,\n      reasoning_content,\n      dump,\n    };\n  }\n\n  async describe(\n    target: Rect | [number, number],\n    modelConfig: IModelConfig,\n    opt?: {\n      deepLocate?: boolean;\n    },\n  ): Promise<Pick<AIDescribeElementResponse, 'description'>> {\n    assert(target, 'target is required for service.describe');\n    const context = await this.contextRetrieverFn();\n    const { shotSize } = context;\n    const screenshotBase64 = context.screenshot.base64;\n    assert(screenshotBase64, 'screenshot is required for service.describe');\n    // The result of the \"describe\" function will be used for positioning, so essentially it is a form of grounding.\n    const { modelFamily } = modelConfig;\n    const systemPrompt = elementDescriberInstruction();\n\n    // Convert [x,y] center point to Rect if needed\n    const defaultRectSize = 30;\n    const targetRect: Rect = Array.isArray(target)\n      ? {\n          left: Math.floor(target[0] - defaultRectSize / 2),\n          top: Math.floor(target[1] - defaultRectSize / 2),\n          width: defaultRectSize,\n          height: defaultRectSize,\n        }\n      : target;\n\n    let imagePayload = await compositeElementInfoImg({\n      inputImgBase64: screenshotBase64,\n      size: shotSize,\n      elementsPositionInfo: [\n        {\n          rect: targetRect,\n        },\n      ],\n      borderThickness: 3,\n    });\n\n    if (opt?.deepLocate) {\n      const searchArea = expandSearchArea(targetRect, shotSize);\n      // Always crop in describe mode. Unlike locate's deepLocate (where\n      // cropping too small loses context for finding elements), describe's\n      // deepLocate intentionally zooms in so the model produces a more\n      // precise description from a focused view. expandSearchArea already\n      // guarantees a minimum 400x400 area with surrounding context.\n      debug('describe: cropping to searchArea', searchArea);\n      const croppedResult = await cropByRect(\n        imagePayload,\n        searchArea,\n        modelFamily === 'qwen2.5-vl',\n      );\n      imagePayload = croppedResult.imageBase64;\n    }\n\n    const msgs: AIArgs = [\n      { role: 'system', content: systemPrompt },\n      {\n        role: 'user',\n        content: [\n          {\n            type: 'image_url',\n            image_url: {\n              url: imagePayload,\n              detail: 'high',\n            },\n          },\n        ],\n      },\n    ];\n\n    const res = await callAIWithObjectResponse<AIDescribeElementResponse>(\n      msgs,\n      modelConfig,\n    );\n\n    const { content } = res;\n    assert(!content.error, `describe failed: ${content.error}`);\n    assert(content.description, 'failed to describe the element');\n    return content;\n  }\n}\n"],"names":["debug","getDebug","Service","query","opt","modelConfig","abortSignal","queryPrompt","assert","hasPlanLocatedElement","searchAreaPrompt","modelFamily","console","undefined","isAutoGLM","context","searchArea","searchAreaRawResponse","searchAreaUsage","searchAreaResponse","searchAreaConfig","buildSearchAreaConfig","JSON","AiLocateSection","startTime","Date","parseResult","rect","rawResponse","usage","reasoning_content","AiLocateElement","timeCost","taskInfo","errorLog","dumpData","elements","dump","createServiceDump","ServiceError","dataDemand","pageDescription","multimodalPrompt","result","AiExtractElementInfo","error","AIResponseParseError","data","thought","target","shotSize","screenshotBase64","systemPrompt","elementDescriberInstruction","defaultRectSize","targetRect","Array","Math","imagePayload","compositeElementInfoImg","expandSearchArea","croppedResult","cropByRect","msgs","res","callAIWithObjectResponse","content","Promise"],"mappings":";;;;;;;;;;;;;;;;;;;;AA8CA,MAAMA,QAAQC,SAAS;AACR,MAAMC;IAqBnB,MAAM,OACJC,KAA0B,EAC1BC,GAAe,EACfC,WAAyB,EACzBC,WAAyB,EACM;QAC/B,MAAMC,cAAc,AAAiB,YAAjB,OAAOJ,QAAqBA,QAAQA,MAAM,MAAM;QACpEK,OAAOD,aAAa;QAEpBC,OAAO,AAAiB,YAAjB,OAAOL,OAAoB;QAElC,MAAMM,wBAAwB,CAAC,CAACL,KAAK,oBAAoB;QAEzD,IAAIM;QACJ,IAAIP,MAAM,UAAU,IAAI,CAACM,uBACvBC,mBAAmBP,MAAM,MAAM;QAGjC,MAAM,EAAEQ,WAAW,EAAE,GAAGN;QAExB,IAAIK,oBAAoB,CAACC,aAAa;YACpCC,QAAQ,IAAI,CACV;YAEFF,mBAAmBG;QACrB;QAEA,IAAIH,oBAAoBI,UAAUH,cAAc;YAC9CC,QAAQ,IAAI,CAAC;YACbF,mBAAmBG;QACrB;QAEA,MAAME,UAAUX,KAAK,WAAY,MAAM,IAAI,CAAC,kBAAkB;QAE9D,IAAIY;QACJ,IAAIC;QACJ,IAAIC;QACJ,IAAIC;QAGJ,IAAIhB,MAAM,UAAU,IAAIM,uBAAuB;YAC7C,MAAMW,mBAAmB,MAAMC,sBAAsB;gBACnDN;gBACA,UAAUX,IAAI,kBAAkB,CAAE,IAAI;gBACtCO;YACF;YACAK,aAAaI,iBAAiB,IAAI;YAElCH,wBAAwBK,KAAK,SAAS,CAAC;gBACrC,QAAQ;gBACR,MAAMlB,IAAI,kBAAkB,CAAE,IAAI;YACpC;YACAe,qBAAqB;gBACnB,MAAMH;gBACN,aAAaI,iBAAiB,WAAW;gBACzC,OAAOA,iBAAiB,KAAK;gBAC7B,aAAaH;YACf;QACF,OAAO,IAAIP,kBAAkB;YAC3BS,qBAAqB,MAAMI,gBAAgB;gBACzCR;gBACA,oBAAoBL;gBACpBL;gBACAC;YACF;YACAE,OACEW,mBAAmB,IAAI,EACvB,CAAC,6BAA6B,EAAET,iBAAiB,CAAC,EAChDS,mBAAmB,KAAK,GAAG,CAAC,EAAE,EAAEA,mBAAmB,KAAK,EAAE,GAAG,IAC7D;YAEJF,wBAAwBE,mBAAmB,WAAW;YACtDD,kBAAkBC,mBAAmB,KAAK;YAC1CH,aAAaG,mBAAmB,IAAI;QACtC;QAEA,MAAMK,YAAYC,KAAK,GAAG;QAC1B,MAAM,EAAEC,WAAW,EAAEC,IAAI,EAAEC,WAAW,EAAEC,KAAK,EAAEC,iBAAiB,EAAE,GAChE,MAAMC,gBAAgB;YACpBhB;YACA,0BAA0BR;YAC1B,cAAcY;YACdd;YACAC;QACF;QAEF,MAAM0B,WAAWP,KAAK,GAAG,KAAKD;QAC9B,MAAMS,WAA4B;YAChC,GAAI,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;YACtC,YAAYD;YACZ,aAAaV,KAAK,SAAS,CAACM;YAC5B,gBAAgBN,KAAK,SAAS,CAACI;YAC/BG;YACAb;YACAC;YACAC;YACAY;QACF;QAEA,IAAII;QACJ,IAAIR,YAAY,MAAM,EAAE,QACtBQ,WAAW,CAAC,4BAA4B,EAAER,YAAY,MAAM,CAAC,IAAI,CAAC,OAAO;QAG3E,MAAMS,WAAsC;YAC1C,MAAM;YACN,WAAW;gBACT,SAAS5B;YACX;YACA,gBAAgB,EAAE;YAClB,aAAaoB;YACb,MAAM;YACNM;YACA,YAAY,CAAC,CAACjB;YACd,OAAOkB;QACT;QAEA,MAAME,WAAWV,YAAY,QAAQ,IAAI,EAAE;QAE3C,MAAMW,OAAOC,kBAAkB;YAC7B,GAAGH,QAAQ;YACX,gBAAgBC;QAClB;QAEA,IAAIF,UACF,MAAM,IAAIK,aAAaL,UAAUG;QAGnC,IAAID,SAAS,MAAM,GAAG,GACpB,MAAM,IAAIG,aACR,CAAC,0CAA0C,EAAEH,SAAS,MAAM,EAAE,EAC9DC;QAIJ,IAAID,AAAoB,MAApBA,SAAS,MAAM,EACjB,OAAO;YACL,SAAS;gBACP,QAAQA,QAAQ,CAAC,EAAE,CAAE,MAAM;gBAC3B,MAAMA,QAAQ,CAAC,EAAE,CAAE,IAAI;gBACvB,aAAaA,QAAQ,CAAC,EAAE,CAAE,WAAW;YACvC;YACAT;YACAU;QACF;QAGF,OAAO;YACL,SAAS;YACTV;YACAU;QACF;IACF;IAEA,MAAM,QACJG,UAA+B,EAC/BnC,WAAyB,EACzBD,GAA0B,EAC1BqC,eAAwB,EACxBC,gBAAoC,EACpC3B,OAAmB,EACe;QAClCP,OAAOO,SAAS;QAChBP,OACE,AAAsB,YAAtB,OAAOgC,cAA2B,AAAsB,YAAtB,OAAOA,YACzC,CAAC,+CAA+C,EAAE,OAAOA,YAAY;QAGvE,MAAMhB,YAAYC,KAAK,GAAG;QAE1B,IAAIC;QAGJ,IAAIE;QACJ,IAAIC;QACJ,IAAIC;QAEJ,IAAI;YACF,MAAMa,SAAS,MAAMC,qBAAwB;gBAC3C7B;gBACA,WAAWyB;gBACXE;gBACA,eAAetC;gBACfC;gBACAoC;YACF;YACAf,cAAciB,OAAO,WAAW;YAChCf,cAAce,OAAO,WAAW;YAChCd,QAAQc,OAAO,KAAK;YACpBb,oBAAoBa,OAAO,iBAAiB;QAC9C,EAAE,OAAOE,OAAO;YACd,IAAIA,iBAAiBC,sBAAsB;gBAEzC,MAAMd,WAAWP,KAAK,GAAG,KAAKD;gBAC9B,MAAMS,WAA4B;oBAChC,GAAI,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;oBACtC,YAAYD;oBACZ,aAAaa,MAAM,WAAW;oBAC9B,OAAOA,MAAM,KAAK;gBACpB;gBACA,MAAMR,OAAOC,kBAAkB;oBAC7B,MAAM;oBACN,WAAW;wBAAEE;oBAAW;oBACxB,gBAAgB,EAAE;oBAClB,MAAM;oBACNP;oBACA,OAAOY,MAAM,OAAO;gBACtB;gBACA,MAAM,IAAIN,aAAaM,MAAM,OAAO,EAAER;YACxC;YACA,MAAMQ;QACR;QAEA,MAAMb,WAAWP,KAAK,GAAG,KAAKD;QAC9B,MAAMS,WAA4B;YAChC,GAAI,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;YACtC,YAAYD;YACZJ;YACA,gBAAgBN,KAAK,SAAS,CAACI;YAC/BG;YACAC;QACF;QAEA,IAAII;QACJ,IAAIR,YAAY,MAAM,EAAE,QACtBQ,WAAW,CAAC,qBAAqB,EAAER,YAAY,MAAM,CAAC,IAAI,CAAC,OAAO;QAGpE,MAAMS,WAAsC;YAC1C,MAAM;YACN,WAAW;gBACTK;YACF;YACA,gBAAgB,EAAE;YAClB,MAAM;YACNP;YACA,OAAOC;QACT;QAEA,MAAM,EAAEa,IAAI,EAAEC,OAAO,EAAE,GAAGtB,eAAe,CAAC;QAE1C,MAAMW,OAAOC,kBAAkB;YAC7B,GAAGH,QAAQ;YACXY;QACF;QAEA,IAAIb,YAAY,CAACa,MACf,MAAM,IAAIR,aAAaL,UAAUG;QAGnC,OAAO;YACLU;YACAC;YACAnB;YACAC;YACAO;QACF;IACF;IAEA,MAAM,SACJY,MAA+B,EAC/B5C,WAAyB,EACzBD,GAEC,EACwD;QACzDI,OAAOyC,QAAQ;QACf,MAAMlC,UAAU,MAAM,IAAI,CAAC,kBAAkB;QAC7C,MAAM,EAAEmC,QAAQ,EAAE,GAAGnC;QACrB,MAAMoC,mBAAmBpC,QAAQ,UAAU,CAAC,MAAM;QAClDP,OAAO2C,kBAAkB;QAEzB,MAAM,EAAExC,WAAW,EAAE,GAAGN;QACxB,MAAM+C,eAAeC;QAGrB,MAAMC,kBAAkB;QACxB,MAAMC,aAAmBC,MAAM,OAAO,CAACP,UACnC;YACE,MAAMQ,KAAK,KAAK,CAACR,MAAM,CAAC,EAAE,GAAGK,kBAAkB;YAC/C,KAAKG,KAAK,KAAK,CAACR,MAAM,CAAC,EAAE,GAAGK,kBAAkB;YAC9C,OAAOA;YACP,QAAQA;QACV,IACAL;QAEJ,IAAIS,eAAe,MAAMC,wBAAwB;YAC/C,gBAAgBR;YAChB,MAAMD;YACN,sBAAsB;gBACpB;oBACE,MAAMK;gBACR;aACD;YACD,iBAAiB;QACnB;QAEA,IAAInD,KAAK,YAAY;YACnB,MAAMY,aAAa4C,iBAAiBL,YAAYL;YAMhDlD,MAAM,oCAAoCgB;YAC1C,MAAM6C,gBAAgB,MAAMC,WAC1BJ,cACA1C,YACAL,AAAgB,iBAAhBA;YAEF+C,eAAeG,cAAc,WAAW;QAC1C;QAEA,MAAME,OAAe;YACnB;gBAAE,MAAM;gBAAU,SAASX;YAAa;YACxC;gBACE,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,WAAW;4BACT,KAAKM;4BACL,QAAQ;wBACV;oBACF;iBACD;YACH;SACD;QAED,MAAMM,MAAM,MAAMC,yBAChBF,MACA1D;QAGF,MAAM,EAAE6D,OAAO,EAAE,GAAGF;QACpBxD,OAAO,CAAC0D,QAAQ,KAAK,EAAE,CAAC,iBAAiB,EAAEA,QAAQ,KAAK,EAAE;QAC1D1D,OAAO0D,QAAQ,WAAW,EAAE;QAC5B,OAAOA;IACT;IAlWA,YACEnD,OAA2D,EAC3DX,GAAoB,CACpB;QAPF;QAEA;QAMEI,OAAOO,SAAS;QAChB,IAAI,AAAmB,cAAnB,OAAOA,SACT,IAAI,CAAC,kBAAkB,GAAGA;aAE1B,IAAI,CAAC,kBAAkB,GAAG,IAAMoD,QAAQ,OAAO,CAACpD;QAGlD,IAAI,AAAyB,WAAlBX,KAAK,UACd,IAAI,CAAC,QAAQ,GAAGA,IAAI,QAAQ;IAEhC;AAqVF"}

package/dist/es/service/utils.mjs

@@ -1,4 +1,4 @@
-import { uuid } from "@midscene/shared/utils";
+import { uuid } from "@aiscene/shared/utils";
 function createServiceDump(data) {
     const baseData = {
         logTime: Date.now()

package/dist/es/service/utils.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"service/utils.mjs","sources":["../../../src/service/utils.ts"],"sourcesContent":["import type { DumpMeta, PartialServiceDumpFromSDK, ServiceDump } from '@/types';\nimport { uuid } from '@midscene/shared/utils';\n\nexport function createServiceDump(\n  data: PartialServiceDumpFromSDK,\n): ServiceDump {\n  const baseData: DumpMeta = {\n    logTime: Date.now(),\n  };\n  const finalData: ServiceDump = {\n    logId: uuid(),\n    ...baseData,\n    ...data,\n  };\n\n  return finalData;\n}\n"],"names":["createServiceDump","data","baseData","Date","finalData","uuid"],"mappings":";AAGO,SAASA,kBACdC,IAA+B;IAE/B,MAAMC,WAAqB;QACzB,SAASC,KAAK,GAAG;IACnB;IACA,MAAMC,YAAyB;QAC7B,OAAOC;QACP,GAAGH,QAAQ;QACX,GAAGD,IAAI;IACT;IAEA,OAAOG;AACT"}
+{"version":3,"file":"service/utils.mjs","sources":["../../../src/service/utils.ts"],"sourcesContent":["import type { DumpMeta, PartialServiceDumpFromSDK, ServiceDump } from '@/types';\nimport { uuid } from '@aiscene/shared/utils';\n\nexport function createServiceDump(\n  data: PartialServiceDumpFromSDK,\n): ServiceDump {\n  const baseData: DumpMeta = {\n    logTime: Date.now(),\n  };\n  const finalData: ServiceDump = {\n    logId: uuid(),\n    ...baseData,\n    ...data,\n  };\n\n  return finalData;\n}\n"],"names":["createServiceDump","data","baseData","Date","finalData","uuid"],"mappings":";AAGO,SAASA,kBACdC,IAA+B;IAE/B,MAAMC,WAAqB;QACzB,SAASC,KAAK,GAAG;IACnB;IACA,MAAMC,YAAyB;QAC7B,OAAOC;QACP,GAAGH,QAAQ;QACX,GAAGD,IAAI;IACT;IAEA,OAAOG;AACT"}

package/dist/es/skill/index.mjs

@@ -1,5 +1,5 @@
-import { CLIError, runToolsCLI } from "@midscene/shared/cli";
-import { BaseMidsceneTools } from "@midscene/shared/mcp";
+import { CLIError, runToolsCLI } from "@aiscene/shared/cli";
+import { BaseMidsceneTools } from "@aiscene/shared/mcp";
 import { Agent } from "../agent/agent.mjs";
 function _define_property(obj, key, value) {
     if (key in obj) Object.defineProperty(obj, key, {

package/dist/es/skill/index.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"skill/index.mjs","sources":["../../../src/skill/index.ts"],"sourcesContent":["import { CLIError, runToolsCLI } from '@midscene/shared/cli';\nimport { BaseMidsceneTools } from '@midscene/shared/mcp';\nimport type { BaseAgent, BaseDevice } from '@midscene/shared/mcp';\nimport { Agent } from '../agent/agent';\nimport type { AbstractInterface } from '../device';\n\ntype DeviceClass = new (...args: any[]) => AbstractInterface;\n\n/**\n * Skill tools manager that lazily creates Agent from a Device class.\n * Used by runSkillCLI for CLI / Agent Skills scenarios where no agent exists at startup.\n */\nclass SkillMidsceneTools extends BaseMidsceneTools<BaseAgent> {\n  constructor(private DeviceClass: DeviceClass) {\n    super();\n  }\n\n  protected createTemporaryDevice(): BaseDevice {\n    return new this.DeviceClass() as unknown as BaseDevice;\n  }\n\n  protected async ensureAgent(): Promise<BaseAgent> {\n    if (!this.agent) {\n      const device = new this.DeviceClass();\n      this.agent = new Agent(device) as unknown as BaseAgent;\n    }\n    return this.agent;\n  }\n}\n\nexport interface SkillCLIOptions {\n  scriptName: string;\n  DeviceClass: DeviceClass;\n}\n\n/**\n * Launch a Skill CLI for a custom interface Device class.\n * This enables AI coding assistants (Claude Code, Cline, etc.) to control\n * your custom interface through CLI commands.\n *\n * @example\n * ```typescript\n * #!/usr/bin/env node\n * import { runSkillCLI } from '@midscene/core/skill';\n * import { SampleDevice } from './sample-device';\n *\n * runSkillCLI({\n *   DeviceClass: SampleDevice,\n *   scriptName: 'my-device',\n * });\n * ```\n */\nexport function runSkillCLI(options: SkillCLIOptions): Promise<void> {\n  const tools = new SkillMidsceneTools(options.DeviceClass);\n  return runToolsCLI(tools, options.scriptName).catch((e) => {\n    if (!(e instanceof CLIError)) console.error(e);\n    process.exit(e instanceof CLIError ? e.exitCode : 1);\n  });\n}\n"],"names":["SkillMidsceneTools","BaseMidsceneTools","device","Agent","DeviceClass","runSkillCLI","options","tools","runToolsCLI","e","CLIError","console","process"],"mappings":";;;;;;;;;;;;;AAYA,MAAMA,2BAA2BC;IAKrB,wBAAoC;QAC5C,OAAO,IAAI,IAAI,CAAC,WAAW;IAC7B;IAEA,MAAgB,cAAkC;QAChD,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE;YACf,MAAMC,SAAS,IAAI,IAAI,CAAC,WAAW;YACnC,IAAI,CAAC,KAAK,GAAG,IAAIC,MAAMD;QACzB;QACA,OAAO,IAAI,CAAC,KAAK;IACnB;IAdA,YAAoBE,WAAwB,CAAE;QAC5C,KAAK,wDADaA,WAAW,GAAXA;IAEpB;AAaF;AAwBO,SAASC,YAAYC,OAAwB;IAClD,MAAMC,QAAQ,IAAIP,mBAAmBM,QAAQ,WAAW;IACxD,OAAOE,YAAYD,OAAOD,QAAQ,UAAU,EAAE,KAAK,CAAC,CAACG;QACnD,IAAI,CAAEA,CAAAA,aAAaC,QAAO,GAAIC,QAAQ,KAAK,CAACF;QAC5CG,QAAQ,IAAI,CAACH,aAAaC,WAAWD,EAAE,QAAQ,GAAG;IACpD;AACF"}
+{"version":3,"file":"skill/index.mjs","sources":["../../../src/skill/index.ts"],"sourcesContent":["import { CLIError, runToolsCLI } from '@aiscene/shared/cli';\nimport { BaseMidsceneTools } from '@aiscene/shared/mcp';\nimport type { BaseAgent, BaseDevice } from '@aiscene/shared/mcp';\nimport { Agent } from '../agent/agent';\nimport type { AbstractInterface } from '../device';\n\ntype DeviceClass = new (...args: any[]) => AbstractInterface;\n\n/**\n * Skill tools manager that lazily creates Agent from a Device class.\n * Used by runSkillCLI for CLI / Agent Skills scenarios where no agent exists at startup.\n */\nclass SkillMidsceneTools extends BaseMidsceneTools<BaseAgent> {\n  constructor(private DeviceClass: DeviceClass) {\n    super();\n  }\n\n  protected createTemporaryDevice(): BaseDevice {\n    return new this.DeviceClass() as unknown as BaseDevice;\n  }\n\n  protected async ensureAgent(): Promise<BaseAgent> {\n    if (!this.agent) {\n      const device = new this.DeviceClass();\n      this.agent = new Agent(device) as unknown as BaseAgent;\n    }\n    return this.agent;\n  }\n}\n\nexport interface SkillCLIOptions {\n  scriptName: string;\n  DeviceClass: DeviceClass;\n}\n\n/**\n * Launch a Skill CLI for a custom interface Device class.\n * This enables AI coding assistants (Claude Code, Cline, etc.) to control\n * your custom interface through CLI commands.\n *\n * @example\n * ```typescript\n * #!/usr/bin/env node\n * import { runSkillCLI } from '@midscene/core/skill';\n * import { SampleDevice } from './sample-device';\n *\n * runSkillCLI({\n *   DeviceClass: SampleDevice,\n *   scriptName: 'my-device',\n * });\n * ```\n */\nexport function runSkillCLI(options: SkillCLIOptions): Promise<void> {\n  const tools = new SkillMidsceneTools(options.DeviceClass);\n  return runToolsCLI(tools, options.scriptName).catch((e) => {\n    if (!(e instanceof CLIError)) console.error(e);\n    process.exit(e instanceof CLIError ? e.exitCode : 1);\n  });\n}\n"],"names":["SkillMidsceneTools","BaseMidsceneTools","device","Agent","DeviceClass","runSkillCLI","options","tools","runToolsCLI","e","CLIError","console","process"],"mappings":";;;;;;;;;;;;;AAYA,MAAMA,2BAA2BC;IAKrB,wBAAoC;QAC5C,OAAO,IAAI,IAAI,CAAC,WAAW;IAC7B;IAEA,MAAgB,cAAkC;QAChD,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE;YACf,MAAMC,SAAS,IAAI,IAAI,CAAC,WAAW;YACnC,IAAI,CAAC,KAAK,GAAG,IAAIC,MAAMD;QACzB;QACA,OAAO,IAAI,CAAC,KAAK;IACnB;IAdA,YAAoBE,WAAwB,CAAE;QAC5C,KAAK,wDADaA,WAAW,GAAXA;IAEpB;AAaF;AAwBO,SAASC,YAAYC,OAAwB;IAClD,MAAMC,QAAQ,IAAIP,mBAAmBM,QAAQ,WAAW;IACxD,OAAOE,YAAYD,OAAOD,QAAQ,UAAU,EAAE,KAAK,CAAC,CAACG;QACnD,IAAI,CAAEA,CAAAA,aAAaC,QAAO,GAAIC,QAAQ,KAAK,CAACF;QAC5CG,QAAQ,IAAI,CAACH,aAAaC,WAAWD,EAAE,QAAQ,GAAG;IACpD;AACF"}

package/dist/es/task-runner.mjs

@@ -1,7 +1,7 @@
 import { setTimingFieldOnce } from "./task-timing.mjs";
 import { ExecutionDump } from "./types.mjs";
-import { getDebug } from "@midscene/shared/logger";
-import { assert, uuid } from "@midscene/shared/utils";
+import { getDebug } from "@aiscene/shared/logger";
+import { assert, uuid } from "@aiscene/shared/utils";
 function _define_property(obj, key, value) {
     if (key in obj) Object.defineProperty(obj, key, {
         value: value,

package/dist/es/task-runner.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"task-runner.mjs","sources":["../../src/task-runner.ts"],"sourcesContent":["import type { ScreenshotItem } from '@/screenshot-item';\nimport { setTimingFieldOnce } from '@/task-timing';\nimport {\n  ExecutionDump,\n  type ExecutionRecorderItem,\n  type ExecutionTask,\n  type ExecutionTaskActionApply,\n  type ExecutionTaskApply,\n  type ExecutionTaskPlanningLocateOutput,\n  type ExecutionTaskProgressOptions,\n  type ExecutionTaskReturn,\n  type ExecutorContext,\n  type PlanningActionParamError,\n  type UIContext,\n} from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert, uuid } from '@midscene/shared/utils';\n\nconst debug = getDebug('task-runner');\nconst UI_CONTEXT_CACHE_TTL_MS = 300;\n\ntype TaskRunnerInitOptions = ExecutionTaskProgressOptions & {\n  tasks?: ExecutionTaskApply[];\n  onTaskUpdate?: (\n    runner: TaskRunner,\n    error?: TaskExecutionError,\n  ) => Promise<void> | void;\n};\n\ntype TaskRunnerOperationOptions = {\n  allowWhenError?: boolean;\n};\n\nexport class TaskRunner {\n  readonly id: string;\n  name: string;\n\n  tasks: ExecutionTask[];\n\n  // status of runner\n  status: 'init' | 'pending' | 'running' | 'completed' | 'error';\n\n  onTaskStart?: ExecutionTaskProgressOptions['onTaskStart'];\n\n  private readonly uiContextBuilder: () => Promise<UIContext>;\n\n  private readonly onTaskUpdate?:\n    | ((runner: TaskRunner, error?: TaskExecutionError) => Promise<void> | void)\n    | undefined;\n\n  private readonly executionLogTime: number;\n\n  constructor(\n    name: string,\n    uiContextBuilder: () => Promise<UIContext>,\n    options?: TaskRunnerInitOptions,\n  ) {\n    this.id = uuid();\n    this.status =\n      options?.tasks && options.tasks.length > 0 ? 'pending' : 'init';\n    this.name = name;\n    this.tasks = (options?.tasks || []).map((item) =>\n      this.markTaskAsPending(item),\n    );\n    this.onTaskStart = options?.onTaskStart;\n    this.uiContextBuilder = uiContextBuilder;\n    this.onTaskUpdate = options?.onTaskUpdate;\n    this.executionLogTime = Date.now();\n  }\n\n  private async emitOnTaskUpdate(error?: TaskExecutionError): Promise<void> {\n    if (!this.onTaskUpdate) {\n      return;\n    }\n    await this.onTaskUpdate(this, error);\n  }\n\n  private lastUiContext?: {\n    context: UIContext;\n    capturedAt: number;\n  };\n\n  private async getUiContext(options?: { forceRefresh?: boolean }): Promise<\n    UIContext | undefined\n  > {\n    const now = Date.now();\n    const shouldReuse =\n      !options?.forceRefresh &&\n      this.lastUiContext &&\n      now - this.lastUiContext.capturedAt <= UI_CONTEXT_CACHE_TTL_MS;\n\n    if (shouldReuse && this.lastUiContext?.context) {\n      debug(\n        `reuse cached uiContext captured ${now - this.lastUiContext.capturedAt}ms ago`,\n      );\n      return this.lastUiContext?.context;\n    }\n\n    try {\n      const uiContext = await this.uiContextBuilder();\n      if (uiContext) {\n        this.lastUiContext = {\n          context: uiContext,\n          capturedAt: Date.now(),\n        };\n      } else {\n        this.lastUiContext = undefined;\n      }\n      return uiContext;\n    } catch (error) {\n      this.lastUiContext = undefined;\n      throw error;\n    }\n  }\n\n  private async captureScreenshot(): Promise<ScreenshotItem | undefined> {\n    try {\n      const uiContext = await this.getUiContext({ forceRefresh: true });\n      return uiContext?.screenshot;\n    } catch (error) {\n      console.error('error while capturing screenshot', error);\n    }\n    return undefined;\n  }\n\n  private attachRecorderItem(\n    task: ExecutionTask,\n    screenshot: ScreenshotItem | undefined,\n    phase: 'after-calling',\n  ): void {\n    if (!phase || !screenshot) {\n      return;\n    }\n\n    const recorderItem: ExecutionRecorderItem = {\n      type: 'screenshot',\n      ts: Date.now(),\n      screenshot,\n      timing: phase,\n    };\n\n    if (!task.recorder) {\n      task.recorder = [recorderItem];\n      return;\n    }\n    task.recorder.push(recorderItem);\n  }\n\n  private markTaskAsPending(task: ExecutionTaskApply): ExecutionTask {\n    return {\n      taskId: uuid(),\n      status: 'pending',\n      ...task,\n    };\n  }\n\n  private normalizeStatusFromError(\n    options?: TaskRunnerOperationOptions,\n    errorMessage?: string,\n  ): void {\n    if (this.status !== 'error') {\n      return;\n    }\n    assert(\n      options?.allowWhenError,\n      errorMessage ||\n        `task runner is in error state, cannot proceed\\nerror=${this.latestErrorTask()?.error}\\n${this.latestErrorTask()?.errorStack}`,\n    );\n    // reset runner state so new tasks can run\n    this.status = this.tasks.length > 0 ? 'pending' : 'init';\n  }\n\n  async append(\n    task: ExecutionTaskApply[] | ExecutionTaskApply,\n    options?: TaskRunnerOperationOptions,\n  ): Promise<void> {\n    this.normalizeStatusFromError(\n      options,\n      `task runner is in error state, cannot append task\\nerror=${this.latestErrorTask()?.error}\\n${this.latestErrorTask()?.errorStack}`,\n    );\n    if (Array.isArray(task)) {\n      this.tasks.push(...task.map((item) => this.markTaskAsPending(item)));\n    } else {\n      this.tasks.push(this.markTaskAsPending(task));\n    }\n    if (this.status !== 'running') {\n      this.status = 'pending';\n    }\n    await this.emitOnTaskUpdate();\n  }\n\n  async appendAndFlush(\n    task: ExecutionTaskApply[] | ExecutionTaskApply,\n    options?: TaskRunnerOperationOptions,\n  ): Promise<{ output: any; thought?: string } | undefined> {\n    await this.append(task, options);\n    return this.flush(options);\n  }\n\n  async flush(\n    options?: TaskRunnerOperationOptions,\n  ): Promise<{ output: any; thought?: string } | undefined> {\n    if (this.status === 'init' && this.tasks.length > 0) {\n      console.warn(\n        'illegal state for task runner, status is init but tasks are not empty',\n      );\n    }\n\n    this.normalizeStatusFromError(options, 'task runner is in error state');\n    assert(this.status !== 'running', 'task runner is already running');\n    assert(this.status !== 'completed', 'task runner is already completed');\n\n    const nextPendingIndex = this.tasks.findIndex(\n      (task) => task.status === 'pending',\n    );\n    if (nextPendingIndex < 0) {\n      // all tasks are completed\n      return;\n    }\n\n    this.status = 'running';\n    await this.emitOnTaskUpdate();\n    let taskIndex = nextPendingIndex;\n    let successfullyCompleted = true;\n\n    let previousFindOutput: ExecutionTaskPlanningLocateOutput | undefined;\n\n    while (taskIndex < this.tasks.length) {\n      const task = this.tasks[taskIndex];\n      assert(\n        task.status === 'pending',\n        `task status should be pending, but got: ${task.status}`,\n      );\n      task.timing = {\n        start: Date.now(),\n      };\n      try {\n        task.status = 'running';\n        await this.emitOnTaskUpdate();\n        try {\n          if (this.onTaskStart) {\n            await this.onTaskStart(task);\n          }\n        } catch (e) {\n          console.error('error in onTaskStart', e);\n        }\n        assert(\n          ['Insight', 'Action Space', 'Planning'].indexOf(task.type) >= 0,\n          `unsupported task type: ${task.type}`,\n        );\n\n        const { executor, param } = task;\n        assert(executor, `executor is required for task type: ${task.type}`);\n\n        let returnValue;\n        // For Insight tasks (Query/Assert/WaitFor), always get fresh context\n        // to ensure we have the latest UI state after any preceding actions\n        const forceRefresh = task.type === 'Insight';\n        setTimingFieldOnce(task.timing, 'getUiContextStart');\n        const uiContext = await this.getUiContext({ forceRefresh });\n        setTimingFieldOnce(task.timing, 'getUiContextEnd');\n\n        task.uiContext = uiContext;\n        const executorContext: ExecutorContext = {\n          task,\n          element: previousFindOutput?.element,\n          uiContext,\n        };\n\n        if (task.type === 'Insight') {\n          assert(\n            task.subType === 'Query' ||\n              task.subType === 'Assert' ||\n              task.subType === 'WaitFor' ||\n              task.subType === 'Boolean' ||\n              task.subType === 'Number' ||\n              task.subType === 'String',\n            `unsupported service subType: ${task.subType}`,\n          );\n          returnValue = await task.executor(param, executorContext);\n        } else if (task.type === 'Planning') {\n          returnValue = await task.executor(param, executorContext);\n          if (task.subType === 'Locate') {\n            previousFindOutput = (\n              returnValue as ExecutionTaskReturn<ExecutionTaskPlanningLocateOutput>\n            )?.output;\n          }\n        } else if (task.type === 'Action Space') {\n          returnValue = await task.executor(param, executorContext);\n        } else {\n          console.warn(\n            `unsupported task type: ${task.type}, will try to execute it directly`,\n          );\n          returnValue = await task.executor(param, executorContext);\n        }\n\n        const isLastTask = taskIndex === this.tasks.length - 1;\n\n        if (isLastTask) {\n          setTimingFieldOnce(task.timing, 'captureAfterCallingSnapshotStart');\n          const screenshot = await this.captureScreenshot();\n          this.attachRecorderItem(task, screenshot, 'after-calling');\n          setTimingFieldOnce(task.timing, 'captureAfterCallingSnapshotEnd');\n        }\n\n        Object.assign(task, returnValue);\n        task.status = 'finished';\n        task.timing.end = Date.now();\n        task.timing.cost = task.timing.end - task.timing.start;\n        await this.emitOnTaskUpdate();\n        taskIndex++;\n      } catch (e: any) {\n        successfullyCompleted = false;\n        task.error = e;\n        task.errorMessage =\n          e?.message || (typeof e === 'string' ? e : 'error-without-message');\n        task.errorStack = e.stack;\n\n        task.status = 'failed';\n        task.timing.end = Date.now();\n        task.timing.cost = task.timing.end - task.timing.start;\n        await this.emitOnTaskUpdate();\n        break;\n      }\n    }\n\n    // set all remaining tasks as cancelled\n    for (let i = taskIndex + 1; i < this.tasks.length; i++) {\n      this.tasks[i].status = 'cancelled';\n    }\n    if (taskIndex + 1 < this.tasks.length) {\n      await this.emitOnTaskUpdate();\n    }\n\n    let finalizeError: TaskExecutionError | undefined;\n    if (!successfullyCompleted) {\n      this.status = 'error';\n      const errorTask = this.latestErrorTask();\n      const messageBase =\n        errorTask?.errorMessage ||\n        (errorTask?.error ? String(errorTask.error) : 'Task execution failed');\n      const stack = errorTask?.errorStack;\n      const message = stack ? `${messageBase}\\n${stack}` : messageBase;\n      finalizeError = new TaskExecutionError(message, this, errorTask, {\n        cause: errorTask?.error,\n      });\n      await this.emitOnTaskUpdate(finalizeError);\n    } else {\n      this.status = 'completed';\n      await this.emitOnTaskUpdate();\n    }\n\n    if (finalizeError) {\n      throw finalizeError;\n    }\n\n    if (this.tasks.length) {\n      // return the last output\n      const outputIndex = Math.min(taskIndex, this.tasks.length - 1);\n      const { thought, output } = this.tasks[outputIndex];\n      return {\n        thought,\n        output,\n      };\n    }\n  }\n\n  isInErrorState(): boolean {\n    return this.status === 'error';\n  }\n\n  latestErrorTask(): ExecutionTask | null {\n    if (this.status !== 'error') {\n      return null;\n    }\n    // Find the LAST failed task (not the first one)\n    // This is important when using allowWhenError to continue after errors\n    for (let i = this.tasks.length - 1; i >= 0; i--) {\n      if (this.tasks[i].status === 'failed') {\n        return this.tasks[i];\n      }\n    }\n    return null;\n  }\n\n  dump(): ExecutionDump {\n    return new ExecutionDump({\n      id: this.id,\n      logTime: this.executionLogTime,\n      name: this.name,\n      tasks: this.tasks,\n    });\n  }\n\n  async appendErrorPlan(errorMsg: string): Promise<{\n    output: undefined;\n    runner: TaskRunner;\n  }> {\n    const errorTask: ExecutionTaskActionApply<PlanningActionParamError> = {\n      type: 'Action Space',\n      subType: 'Error',\n      param: {\n        thought: errorMsg,\n      },\n      thought: errorMsg,\n      executor: async () => {\n        throw new Error(errorMsg || 'error without thought');\n      },\n    };\n    await this.appendAndFlush(errorTask);\n\n    return {\n      output: undefined,\n      runner: this,\n    };\n  }\n}\n\nexport class TaskExecutionError extends Error {\n  runner: TaskRunner;\n\n  errorTask: ExecutionTask | null;\n\n  constructor(\n    message: string,\n    runner: TaskRunner,\n    errorTask: ExecutionTask | null,\n    options?: { cause?: unknown },\n  ) {\n    super(message, options);\n    this.runner = runner;\n    this.errorTask = errorTask;\n  }\n}\n"],"names":["debug","getDebug","UI_CONTEXT_CACHE_TTL_MS","TaskRunner","error","options","now","Date","shouldReuse","uiContext","undefined","console","task","screenshot","phase","recorderItem","uuid","errorMessage","assert","Array","item","nextPendingIndex","taskIndex","successfullyCompleted","previousFindOutput","e","executor","param","returnValue","forceRefresh","setTimingFieldOnce","executorContext","isLastTask","Object","i","finalizeError","errorTask","messageBase","String","stack","message","TaskExecutionError","outputIndex","Math","thought","output","ExecutionDump","errorMsg","Error","name","uiContextBuilder","runner"],"mappings":";;;;;;;;;;;;;;AAkBA,MAAMA,QAAQC,SAAS;AACvB,MAAMC,0BAA0B;AAczB,MAAMC;IAqCX,MAAc,iBAAiBC,KAA0B,EAAiB;QACxE,IAAI,CAAC,IAAI,CAAC,YAAY,EACpB;QAEF,MAAM,IAAI,CAAC,YAAY,CAAC,IAAI,EAAEA;IAChC;IAOA,MAAc,aAAaC,OAAoC,EAE7D;QACA,MAAMC,MAAMC,KAAK,GAAG;QACpB,MAAMC,cACJ,CAACH,SAAS,gBACV,IAAI,CAAC,aAAa,IAClBC,MAAM,IAAI,CAAC,aAAa,CAAC,UAAU,IAAIJ;QAEzC,IAAIM,eAAe,IAAI,CAAC,aAAa,EAAE,SAAS;YAC9CR,MACE,CAAC,gCAAgC,EAAEM,MAAM,IAAI,CAAC,aAAa,CAAC,UAAU,CAAC,MAAM,CAAC;YAEhF,OAAO,IAAI,CAAC,aAAa,EAAE;QAC7B;QAEA,IAAI;YACF,MAAMG,YAAY,MAAM,IAAI,CAAC,gBAAgB;YAC7C,IAAIA,WACF,IAAI,CAAC,aAAa,GAAG;gBACnB,SAASA;gBACT,YAAYF,KAAK,GAAG;YACtB;iBAEA,IAAI,CAAC,aAAa,GAAGG;YAEvB,OAAOD;QACT,EAAE,OAAOL,OAAO;YACd,IAAI,CAAC,aAAa,GAAGM;YACrB,MAAMN;QACR;IACF;IAEA,MAAc,oBAAyD;QACrE,IAAI;YACF,MAAMK,YAAY,MAAM,IAAI,CAAC,YAAY,CAAC;gBAAE,cAAc;YAAK;YAC/D,OAAOA,WAAW;QACpB,EAAE,OAAOL,OAAO;YACdO,QAAQ,KAAK,CAAC,oCAAoCP;QACpD;IAEF;IAEQ,mBACNQ,IAAmB,EACnBC,UAAsC,EACtCC,KAAsB,EAChB;QACN,IAAI,CAACA,SAAS,CAACD,YACb;QAGF,MAAME,eAAsC;YAC1C,MAAM;YACN,IAAIR,KAAK,GAAG;YACZM;YACA,QAAQC;QACV;QAEA,IAAI,CAACF,KAAK,QAAQ,EAAE;YAClBA,KAAK,QAAQ,GAAG;gBAACG;aAAa;YAC9B;QACF;QACAH,KAAK,QAAQ,CAAC,IAAI,CAACG;IACrB;IAEQ,kBAAkBH,IAAwB,EAAiB;QACjE,OAAO;YACL,QAAQI;YACR,QAAQ;YACR,GAAGJ,IAAI;QACT;IACF;IAEQ,yBACNP,OAAoC,EACpCY,YAAqB,EACf;QACN,IAAI,AAAgB,YAAhB,IAAI,CAAC,MAAM,EACb;QAEFC,OACEb,SAAS,gBACTY,gBACE,CAAC,qDAAqD,EAAE,IAAI,CAAC,eAAe,IAAI,MAAM,EAAE,EAAE,IAAI,CAAC,eAAe,IAAI,YAAY;QAGlI,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,YAAY;IACpD;IAEA,MAAM,OACJL,IAA+C,EAC/CP,OAAoC,EACrB;QACf,IAAI,CAAC,wBAAwB,CAC3BA,SACA,CAAC,yDAAyD,EAAE,IAAI,CAAC,eAAe,IAAI,MAAM,EAAE,EAAE,IAAI,CAAC,eAAe,IAAI,YAAY;QAEpI,IAAIc,MAAM,OAAO,CAACP,OAChB,IAAI,CAAC,KAAK,CAAC,IAAI,IAAIA,KAAK,GAAG,CAAC,CAACQ,OAAS,IAAI,CAAC,iBAAiB,CAACA;aAE7D,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAACR;QAEzC,IAAI,AAAgB,cAAhB,IAAI,CAAC,MAAM,EACb,IAAI,CAAC,MAAM,GAAG;QAEhB,MAAM,IAAI,CAAC,gBAAgB;IAC7B;IAEA,MAAM,eACJA,IAA+C,EAC/CP,OAAoC,EACoB;QACxD,MAAM,IAAI,CAAC,MAAM,CAACO,MAAMP;QACxB,OAAO,IAAI,CAAC,KAAK,CAACA;IACpB;IAEA,MAAM,MACJA,OAAoC,EACoB;QACxD,IAAI,AAAgB,WAAhB,IAAI,CAAC,MAAM,IAAe,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,GAChDM,QAAQ,IAAI,CACV;QAIJ,IAAI,CAAC,wBAAwB,CAACN,SAAS;QACvCa,OAAO,AAAgB,cAAhB,IAAI,CAAC,MAAM,EAAgB;QAClCA,OAAO,AAAgB,gBAAhB,IAAI,CAAC,MAAM,EAAkB;QAEpC,MAAMG,mBAAmB,IAAI,CAAC,KAAK,CAAC,SAAS,CAC3C,CAACT,OAASA,AAAgB,cAAhBA,KAAK,MAAM;QAEvB,IAAIS,mBAAmB,GAErB;QAGF,IAAI,CAAC,MAAM,GAAG;QACd,MAAM,IAAI,CAAC,gBAAgB;QAC3B,IAAIC,YAAYD;QAChB,IAAIE,wBAAwB;QAE5B,IAAIC;QAEJ,MAAOF,YAAY,IAAI,CAAC,KAAK,CAAC,MAAM,CAAE;YACpC,MAAMV,OAAO,IAAI,CAAC,KAAK,CAACU,UAAU;YAClCJ,OACEN,AAAgB,cAAhBA,KAAK,MAAM,EACX,CAAC,wCAAwC,EAAEA,KAAK,MAAM,EAAE;YAE1DA,KAAK,MAAM,GAAG;gBACZ,OAAOL,KAAK,GAAG;YACjB;YACA,IAAI;gBACFK,KAAK,MAAM,GAAG;gBACd,MAAM,IAAI,CAAC,gBAAgB;gBAC3B,IAAI;oBACF,IAAI,IAAI,CAAC,WAAW,EAClB,MAAM,IAAI,CAAC,WAAW,CAACA;gBAE3B,EAAE,OAAOa,GAAG;oBACVd,QAAQ,KAAK,CAAC,wBAAwBc;gBACxC;gBACAP,OACE;oBAAC;oBAAW;oBAAgB;iBAAW,CAAC,OAAO,CAACN,KAAK,IAAI,KAAK,GAC9D,CAAC,uBAAuB,EAAEA,KAAK,IAAI,EAAE;gBAGvC,MAAM,EAAEc,QAAQ,EAAEC,KAAK,EAAE,GAAGf;gBAC5BM,OAAOQ,UAAU,CAAC,oCAAoC,EAAEd,KAAK,IAAI,EAAE;gBAEnE,IAAIgB;gBAGJ,MAAMC,eAAejB,AAAc,cAAdA,KAAK,IAAI;gBAC9BkB,mBAAmBlB,KAAK,MAAM,EAAE;gBAChC,MAAMH,YAAY,MAAM,IAAI,CAAC,YAAY,CAAC;oBAAEoB;gBAAa;gBACzDC,mBAAmBlB,KAAK,MAAM,EAAE;gBAEhCA,KAAK,SAAS,GAAGH;gBACjB,MAAMsB,kBAAmC;oBACvCnB;oBACA,SAASY,oBAAoB;oBAC7Bf;gBACF;gBAEA,IAAIG,AAAc,cAAdA,KAAK,IAAI,EAAgB;oBAC3BM,OACEN,AAAiB,YAAjBA,KAAK,OAAO,IACVA,AAAiB,aAAjBA,KAAK,OAAO,IACZA,AAAiB,cAAjBA,KAAK,OAAO,IACZA,AAAiB,cAAjBA,KAAK,OAAO,IACZA,AAAiB,aAAjBA,KAAK,OAAO,IACZA,AAAiB,aAAjBA,KAAK,OAAO,EACd,CAAC,6BAA6B,EAAEA,KAAK,OAAO,EAAE;oBAEhDgB,cAAc,MAAMhB,KAAK,QAAQ,CAACe,OAAOI;gBAC3C,OAAO,IAAInB,AAAc,eAAdA,KAAK,IAAI,EAAiB;oBACnCgB,cAAc,MAAMhB,KAAK,QAAQ,CAACe,OAAOI;oBACzC,IAAInB,AAAiB,aAAjBA,KAAK,OAAO,EACdY,qBACEI,aACC;gBAEP,OAAO,IAAIhB,AAAc,mBAAdA,KAAK,IAAI,EAClBgB,cAAc,MAAMhB,KAAK,QAAQ,CAACe,OAAOI;qBACpC;oBACLpB,QAAQ,IAAI,CACV,CAAC,uBAAuB,EAAEC,KAAK,IAAI,CAAC,iCAAiC,CAAC;oBAExEgB,cAAc,MAAMhB,KAAK,QAAQ,CAACe,OAAOI;gBAC3C;gBAEA,MAAMC,aAAaV,cAAc,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG;gBAErD,IAAIU,YAAY;oBACdF,mBAAmBlB,KAAK,MAAM,EAAE;oBAChC,MAAMC,aAAa,MAAM,IAAI,CAAC,iBAAiB;oBAC/C,IAAI,CAAC,kBAAkB,CAACD,MAAMC,YAAY;oBAC1CiB,mBAAmBlB,KAAK,MAAM,EAAE;gBAClC;gBAEAqB,OAAO,MAAM,CAACrB,MAAMgB;gBACpBhB,KAAK,MAAM,GAAG;gBACdA,KAAK,MAAM,CAAC,GAAG,GAAGL,KAAK,GAAG;gBAC1BK,KAAK,MAAM,CAAC,IAAI,GAAGA,KAAK,MAAM,CAAC,GAAG,GAAGA,KAAK,MAAM,CAAC,KAAK;gBACtD,MAAM,IAAI,CAAC,gBAAgB;gBAC3BU;YACF,EAAE,OAAOG,GAAQ;gBACfF,wBAAwB;gBACxBX,KAAK,KAAK,GAAGa;gBACbb,KAAK,YAAY,GACfa,GAAG,WAAY,CAAa,YAAb,OAAOA,IAAiBA,IAAI,uBAAsB;gBACnEb,KAAK,UAAU,GAAGa,EAAE,KAAK;gBAEzBb,KAAK,MAAM,GAAG;gBACdA,KAAK,MAAM,CAAC,GAAG,GAAGL,KAAK,GAAG;gBAC1BK,KAAK,MAAM,CAAC,IAAI,GAAGA,KAAK,MAAM,CAAC,GAAG,GAAGA,KAAK,MAAM,CAAC,KAAK;gBACtD,MAAM,IAAI,CAAC,gBAAgB;gBAC3B;YACF;QACF;QAGA,IAAK,IAAIsB,IAAIZ,YAAY,GAAGY,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,EAAEA,IACjD,IAAI,CAAC,KAAK,CAACA,EAAE,CAAC,MAAM,GAAG;QAEzB,IAAIZ,YAAY,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,EACnC,MAAM,IAAI,CAAC,gBAAgB;QAG7B,IAAIa;QACJ,IAAKZ,uBAYE;YACL,IAAI,CAAC,MAAM,GAAG;YACd,MAAM,IAAI,CAAC,gBAAgB;QAC7B,OAf4B;YAC1B,IAAI,CAAC,MAAM,GAAG;YACd,MAAMa,YAAY,IAAI,CAAC,eAAe;YACtC,MAAMC,cACJD,WAAW,gBACVA,CAAAA,WAAW,QAAQE,OAAOF,UAAU,KAAK,IAAI,uBAAsB;YACtE,MAAMG,QAAQH,WAAW;YACzB,MAAMI,UAAUD,QAAQ,GAAGF,YAAY,EAAE,EAAEE,OAAO,GAAGF;YACrDF,gBAAgB,IAAIM,mBAAmBD,SAAS,IAAI,EAAEJ,WAAW;gBAC/D,OAAOA,WAAW;YACpB;YACA,MAAM,IAAI,CAAC,gBAAgB,CAACD;QAC9B;QAKA,IAAIA,eACF,MAAMA;QAGR,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE;YAErB,MAAMO,cAAcC,KAAK,GAAG,CAACrB,WAAW,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG;YAC5D,MAAM,EAAEsB,OAAO,EAAEC,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAACH,YAAY;YACnD,OAAO;gBACLE;gBACAC;YACF;QACF;IACF;IAEA,iBAA0B;QACxB,OAAO,AAAgB,YAAhB,IAAI,CAAC,MAAM;IACpB;IAEA,kBAAwC;QACtC,IAAI,AAAgB,YAAhB,IAAI,CAAC,MAAM,EACb,OAAO;QAIT,IAAK,IAAIX,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,GAAGA,KAAK,GAAGA,IAC1C,IAAI,AAAyB,aAAzB,IAAI,CAAC,KAAK,CAACA,EAAE,CAAC,MAAM,EACtB,OAAO,IAAI,CAAC,KAAK,CAACA,EAAE;QAGxB,OAAO;IACT;IAEA,OAAsB;QACpB,OAAO,IAAIY,cAAc;YACvB,IAAI,IAAI,CAAC,EAAE;YACX,SAAS,IAAI,CAAC,gBAAgB;YAC9B,MAAM,IAAI,CAAC,IAAI;YACf,OAAO,IAAI,CAAC,KAAK;QACnB;IACF;IAEA,MAAM,gBAAgBC,QAAgB,EAGnC;QACD,MAAMX,YAAgE;YACpE,MAAM;YACN,SAAS;YACT,OAAO;gBACL,SAASW;YACX;YACA,SAASA;YACT,UAAU;gBACR,MAAM,IAAIC,MAAMD,YAAY;YAC9B;QACF;QACA,MAAM,IAAI,CAAC,cAAc,CAACX;QAE1B,OAAO;YACL,QAAQ1B;YACR,QAAQ,IAAI;QACd;IACF;IA3WA,YACEuC,IAAY,EACZC,gBAA0C,EAC1C7C,OAA+B,CAC/B;QAtBF,uBAAS,MAAT;QACA;QAEA;QAGA;QAEA;QAEA,uBAAiB,oBAAjB;QAEA,uBAAiB,gBAAjB;QAIA,uBAAiB,oBAAjB;QA2BA,uBAAQ,iBAAR;QApBE,IAAI,CAAC,EAAE,GAAGW;QACV,IAAI,CAAC,MAAM,GACTX,SAAS,SAASA,QAAQ,KAAK,CAAC,MAAM,GAAG,IAAI,YAAY;QAC3D,IAAI,CAAC,IAAI,GAAG4C;QACZ,IAAI,CAAC,KAAK,GAAI5C,AAAAA,CAAAA,SAAS,SAAS,EAAC,EAAG,GAAG,CAAC,CAACe,OACvC,IAAI,CAAC,iBAAiB,CAACA;QAEzB,IAAI,CAAC,WAAW,GAAGf,SAAS;QAC5B,IAAI,CAAC,gBAAgB,GAAG6C;QACxB,IAAI,CAAC,YAAY,GAAG7C,SAAS;QAC7B,IAAI,CAAC,gBAAgB,GAAGE,KAAK,GAAG;IAClC;AA4VF;AAEO,MAAMkC,2BAA2BO;IAKtC,YACER,OAAe,EACfW,MAAkB,EAClBf,SAA+B,EAC/B/B,OAA6B,CAC7B;QACA,KAAK,CAACmC,SAASnC,UAVjB,0CAEA;QASE,IAAI,CAAC,MAAM,GAAG8C;QACd,IAAI,CAAC,SAAS,GAAGf;IACnB;AACF"}
+{"version":3,"file":"task-runner.mjs","sources":["../../src/task-runner.ts"],"sourcesContent":["import type { ScreenshotItem } from '@/screenshot-item';\nimport { setTimingFieldOnce } from '@/task-timing';\nimport {\n  ExecutionDump,\n  type ExecutionRecorderItem,\n  type ExecutionTask,\n  type ExecutionTaskActionApply,\n  type ExecutionTaskApply,\n  type ExecutionTaskPlanningLocateOutput,\n  type ExecutionTaskProgressOptions,\n  type ExecutionTaskReturn,\n  type ExecutorContext,\n  type PlanningActionParamError,\n  type UIContext,\n} from '@/types';\nimport { getDebug } from '@aiscene/shared/logger';\nimport { assert, uuid } from '@aiscene/shared/utils';\n\nconst debug = getDebug('task-runner');\nconst UI_CONTEXT_CACHE_TTL_MS = 300;\n\ntype TaskRunnerInitOptions = ExecutionTaskProgressOptions & {\n  tasks?: ExecutionTaskApply[];\n  onTaskUpdate?: (\n    runner: TaskRunner,\n    error?: TaskExecutionError,\n  ) => Promise<void> | void;\n};\n\ntype TaskRunnerOperationOptions = {\n  allowWhenError?: boolean;\n};\n\nexport class TaskRunner {\n  readonly id: string;\n  name: string;\n\n  tasks: ExecutionTask[];\n\n  // status of runner\n  status: 'init' | 'pending' | 'running' | 'completed' | 'error';\n\n  onTaskStart?: ExecutionTaskProgressOptions['onTaskStart'];\n\n  private readonly uiContextBuilder: () => Promise<UIContext>;\n\n  private readonly onTaskUpdate?:\n    | ((runner: TaskRunner, error?: TaskExecutionError) => Promise<void> | void)\n    | undefined;\n\n  private readonly executionLogTime: number;\n\n  constructor(\n    name: string,\n    uiContextBuilder: () => Promise<UIContext>,\n    options?: TaskRunnerInitOptions,\n  ) {\n    this.id = uuid();\n    this.status =\n      options?.tasks && options.tasks.length > 0 ? 'pending' : 'init';\n    this.name = name;\n    this.tasks = (options?.tasks || []).map((item) =>\n      this.markTaskAsPending(item),\n    );\n    this.onTaskStart = options?.onTaskStart;\n    this.uiContextBuilder = uiContextBuilder;\n    this.onTaskUpdate = options?.onTaskUpdate;\n    this.executionLogTime = Date.now();\n  }\n\n  private async emitOnTaskUpdate(error?: TaskExecutionError): Promise<void> {\n    if (!this.onTaskUpdate) {\n      return;\n    }\n    await this.onTaskUpdate(this, error);\n  }\n\n  private lastUiContext?: {\n    context: UIContext;\n    capturedAt: number;\n  };\n\n  private async getUiContext(options?: { forceRefresh?: boolean }): Promise<\n    UIContext | undefined\n  > {\n    const now = Date.now();\n    const shouldReuse =\n      !options?.forceRefresh &&\n      this.lastUiContext &&\n      now - this.lastUiContext.capturedAt <= UI_CONTEXT_CACHE_TTL_MS;\n\n    if (shouldReuse && this.lastUiContext?.context) {\n      debug(\n        `reuse cached uiContext captured ${now - this.lastUiContext.capturedAt}ms ago`,\n      );\n      return this.lastUiContext?.context;\n    }\n\n    try {\n      const uiContext = await this.uiContextBuilder();\n      if (uiContext) {\n        this.lastUiContext = {\n          context: uiContext,\n          capturedAt: Date.now(),\n        };\n      } else {\n        this.lastUiContext = undefined;\n      }\n      return uiContext;\n    } catch (error) {\n      this.lastUiContext = undefined;\n      throw error;\n    }\n  }\n\n  private async captureScreenshot(): Promise<ScreenshotItem | undefined> {\n    try {\n      const uiContext = await this.getUiContext({ forceRefresh: true });\n      return uiContext?.screenshot;\n    } catch (error) {\n      console.error('error while capturing screenshot', error);\n    }\n    return undefined;\n  }\n\n  private attachRecorderItem(\n    task: ExecutionTask,\n    screenshot: ScreenshotItem | undefined,\n    phase: 'after-calling',\n  ): void {\n    if (!phase || !screenshot) {\n      return;\n    }\n\n    const recorderItem: ExecutionRecorderItem = {\n      type: 'screenshot',\n      ts: Date.now(),\n      screenshot,\n      timing: phase,\n    };\n\n    if (!task.recorder) {\n      task.recorder = [recorderItem];\n      return;\n    }\n    task.recorder.push(recorderItem);\n  }\n\n  private markTaskAsPending(task: ExecutionTaskApply): ExecutionTask {\n    return {\n      taskId: uuid(),\n      status: 'pending',\n      ...task,\n    };\n  }\n\n  private normalizeStatusFromError(\n    options?: TaskRunnerOperationOptions,\n    errorMessage?: string,\n  ): void {\n    if (this.status !== 'error') {\n      return;\n    }\n    assert(\n      options?.allowWhenError,\n      errorMessage ||\n        `task runner is in error state, cannot proceed\\nerror=${this.latestErrorTask()?.error}\\n${this.latestErrorTask()?.errorStack}`,\n    );\n    // reset runner state so new tasks can run\n    this.status = this.tasks.length > 0 ? 'pending' : 'init';\n  }\n\n  async append(\n    task: ExecutionTaskApply[] | ExecutionTaskApply,\n    options?: TaskRunnerOperationOptions,\n  ): Promise<void> {\n    this.normalizeStatusFromError(\n      options,\n      `task runner is in error state, cannot append task\\nerror=${this.latestErrorTask()?.error}\\n${this.latestErrorTask()?.errorStack}`,\n    );\n    if (Array.isArray(task)) {\n      this.tasks.push(...task.map((item) => this.markTaskAsPending(item)));\n    } else {\n      this.tasks.push(this.markTaskAsPending(task));\n    }\n    if (this.status !== 'running') {\n      this.status = 'pending';\n    }\n    await this.emitOnTaskUpdate();\n  }\n\n  async appendAndFlush(\n    task: ExecutionTaskApply[] | ExecutionTaskApply,\n    options?: TaskRunnerOperationOptions,\n  ): Promise<{ output: any; thought?: string } | undefined> {\n    await this.append(task, options);\n    return this.flush(options);\n  }\n\n  async flush(\n    options?: TaskRunnerOperationOptions,\n  ): Promise<{ output: any; thought?: string } | undefined> {\n    if (this.status === 'init' && this.tasks.length > 0) {\n      console.warn(\n        'illegal state for task runner, status is init but tasks are not empty',\n      );\n    }\n\n    this.normalizeStatusFromError(options, 'task runner is in error state');\n    assert(this.status !== 'running', 'task runner is already running');\n    assert(this.status !== 'completed', 'task runner is already completed');\n\n    const nextPendingIndex = this.tasks.findIndex(\n      (task) => task.status === 'pending',\n    );\n    if (nextPendingIndex < 0) {\n      // all tasks are completed\n      return;\n    }\n\n    this.status = 'running';\n    await this.emitOnTaskUpdate();\n    let taskIndex = nextPendingIndex;\n    let successfullyCompleted = true;\n\n    let previousFindOutput: ExecutionTaskPlanningLocateOutput | undefined;\n\n    while (taskIndex < this.tasks.length) {\n      const task = this.tasks[taskIndex];\n      assert(\n        task.status === 'pending',\n        `task status should be pending, but got: ${task.status}`,\n      );\n      task.timing = {\n        start: Date.now(),\n      };\n      try {\n        task.status = 'running';\n        await this.emitOnTaskUpdate();\n        try {\n          if (this.onTaskStart) {\n            await this.onTaskStart(task);\n          }\n        } catch (e) {\n          console.error('error in onTaskStart', e);\n        }\n        assert(\n          ['Insight', 'Action Space', 'Planning'].indexOf(task.type) >= 0,\n          `unsupported task type: ${task.type}`,\n        );\n\n        const { executor, param } = task;\n        assert(executor, `executor is required for task type: ${task.type}`);\n\n        let returnValue;\n        // For Insight tasks (Query/Assert/WaitFor), always get fresh context\n        // to ensure we have the latest UI state after any preceding actions\n        const forceRefresh = task.type === 'Insight';\n        setTimingFieldOnce(task.timing, 'getUiContextStart');\n        const uiContext = await this.getUiContext({ forceRefresh });\n        setTimingFieldOnce(task.timing, 'getUiContextEnd');\n\n        task.uiContext = uiContext;\n        const executorContext: ExecutorContext = {\n          task,\n          element: previousFindOutput?.element,\n          uiContext,\n        };\n\n        if (task.type === 'Insight') {\n          assert(\n            task.subType === 'Query' ||\n              task.subType === 'Assert' ||\n              task.subType === 'WaitFor' ||\n              task.subType === 'Boolean' ||\n              task.subType === 'Number' ||\n              task.subType === 'String',\n            `unsupported service subType: ${task.subType}`,\n          );\n          returnValue = await task.executor(param, executorContext);\n        } else if (task.type === 'Planning') {\n          returnValue = await task.executor(param, executorContext);\n          if (task.subType === 'Locate') {\n            previousFindOutput = (\n              returnValue as ExecutionTaskReturn<ExecutionTaskPlanningLocateOutput>\n            )?.output;\n          }\n        } else if (task.type === 'Action Space') {\n          returnValue = await task.executor(param, executorContext);\n        } else {\n          console.warn(\n            `unsupported task type: ${task.type}, will try to execute it directly`,\n          );\n          returnValue = await task.executor(param, executorContext);\n        }\n\n        const isLastTask = taskIndex === this.tasks.length - 1;\n\n        if (isLastTask) {\n          setTimingFieldOnce(task.timing, 'captureAfterCallingSnapshotStart');\n          const screenshot = await this.captureScreenshot();\n          this.attachRecorderItem(task, screenshot, 'after-calling');\n          setTimingFieldOnce(task.timing, 'captureAfterCallingSnapshotEnd');\n        }\n\n        Object.assign(task, returnValue);\n        task.status = 'finished';\n        task.timing.end = Date.now();\n        task.timing.cost = task.timing.end - task.timing.start;\n        await this.emitOnTaskUpdate();\n        taskIndex++;\n      } catch (e: any) {\n        successfullyCompleted = false;\n        task.error = e;\n        task.errorMessage =\n          e?.message || (typeof e === 'string' ? e : 'error-without-message');\n        task.errorStack = e.stack;\n\n        task.status = 'failed';\n        task.timing.end = Date.now();\n        task.timing.cost = task.timing.end - task.timing.start;\n        await this.emitOnTaskUpdate();\n        break;\n      }\n    }\n\n    // set all remaining tasks as cancelled\n    for (let i = taskIndex + 1; i < this.tasks.length; i++) {\n      this.tasks[i].status = 'cancelled';\n    }\n    if (taskIndex + 1 < this.tasks.length) {\n      await this.emitOnTaskUpdate();\n    }\n\n    let finalizeError: TaskExecutionError | undefined;\n    if (!successfullyCompleted) {\n      this.status = 'error';\n      const errorTask = this.latestErrorTask();\n      const messageBase =\n        errorTask?.errorMessage ||\n        (errorTask?.error ? String(errorTask.error) : 'Task execution failed');\n      const stack = errorTask?.errorStack;\n      const message = stack ? `${messageBase}\\n${stack}` : messageBase;\n      finalizeError = new TaskExecutionError(message, this, errorTask, {\n        cause: errorTask?.error,\n      });\n      await this.emitOnTaskUpdate(finalizeError);\n    } else {\n      this.status = 'completed';\n      await this.emitOnTaskUpdate();\n    }\n\n    if (finalizeError) {\n      throw finalizeError;\n    }\n\n    if (this.tasks.length) {\n      // return the last output\n      const outputIndex = Math.min(taskIndex, this.tasks.length - 1);\n      const { thought, output } = this.tasks[outputIndex];\n      return {\n        thought,\n        output,\n      };\n    }\n  }\n\n  isInErrorState(): boolean {\n    return this.status === 'error';\n  }\n\n  latestErrorTask(): ExecutionTask | null {\n    if (this.status !== 'error') {\n      return null;\n    }\n    // Find the LAST failed task (not the first one)\n    // This is important when using allowWhenError to continue after errors\n    for (let i = this.tasks.length - 1; i >= 0; i--) {\n      if (this.tasks[i].status === 'failed') {\n        return this.tasks[i];\n      }\n    }\n    return null;\n  }\n\n  dump(): ExecutionDump {\n    return new ExecutionDump({\n      id: this.id,\n      logTime: this.executionLogTime,\n      name: this.name,\n      tasks: this.tasks,\n    });\n  }\n\n  async appendErrorPlan(errorMsg: string): Promise<{\n    output: undefined;\n    runner: TaskRunner;\n  }> {\n    const errorTask: ExecutionTaskActionApply<PlanningActionParamError> = {\n      type: 'Action Space',\n      subType: 'Error',\n      param: {\n        thought: errorMsg,\n      },\n      thought: errorMsg,\n      executor: async () => {\n        throw new Error(errorMsg || 'error without thought');\n      },\n    };\n    await this.appendAndFlush(errorTask);\n\n    return {\n      output: undefined,\n      runner: this,\n    };\n  }\n}\n\nexport class TaskExecutionError extends Error {\n  runner: TaskRunner;\n\n  errorTask: ExecutionTask | null;\n\n  constructor(\n    message: string,\n    runner: TaskRunner,\n    errorTask: ExecutionTask | null,\n    options?: { cause?: unknown },\n  ) {\n    super(message, options);\n    this.runner = runner;\n    this.errorTask = errorTask;\n  }\n}\n"],"names":["debug","getDebug","UI_CONTEXT_CACHE_TTL_MS","TaskRunner","error","options","now","Date","shouldReuse","uiContext","undefined","console","task","screenshot","phase","recorderItem","uuid","errorMessage","assert","Array","item","nextPendingIndex","taskIndex","successfullyCompleted","previousFindOutput","e","executor","param","returnValue","forceRefresh","setTimingFieldOnce","executorContext","isLastTask","Object","i","finalizeError","errorTask","messageBase","String","stack","message","TaskExecutionError","outputIndex","Math","thought","output","ExecutionDump","errorMsg","Error","name","uiContextBuilder","runner"],"mappings":";;;;;;;;;;;;;;AAkBA,MAAMA,QAAQC,SAAS;AACvB,MAAMC,0BAA0B;AAczB,MAAMC;IAqCX,MAAc,iBAAiBC,KAA0B,EAAiB;QACxE,IAAI,CAAC,IAAI,CAAC,YAAY,EACpB;QAEF,MAAM,IAAI,CAAC,YAAY,CAAC,IAAI,EAAEA;IAChC;IAOA,MAAc,aAAaC,OAAoC,EAE7D;QACA,MAAMC,MAAMC,KAAK,GAAG;QACpB,MAAMC,cACJ,CAACH,SAAS,gBACV,IAAI,CAAC,aAAa,IAClBC,MAAM,IAAI,CAAC,aAAa,CAAC,UAAU,IAAIJ;QAEzC,IAAIM,eAAe,IAAI,CAAC,aAAa,EAAE,SAAS;YAC9CR,MACE,CAAC,gCAAgC,EAAEM,MAAM,IAAI,CAAC,aAAa,CAAC,UAAU,CAAC,MAAM,CAAC;YAEhF,OAAO,IAAI,CAAC,aAAa,EAAE;QAC7B;QAEA,IAAI;YACF,MAAMG,YAAY,MAAM,IAAI,CAAC,gBAAgB;YAC7C,IAAIA,WACF,IAAI,CAAC,aAAa,GAAG;gBACnB,SAASA;gBACT,YAAYF,KAAK,GAAG;YACtB;iBAEA,IAAI,CAAC,aAAa,GAAGG;YAEvB,OAAOD;QACT,EAAE,OAAOL,OAAO;YACd,IAAI,CAAC,aAAa,GAAGM;YACrB,MAAMN;QACR;IACF;IAEA,MAAc,oBAAyD;QACrE,IAAI;YACF,MAAMK,YAAY,MAAM,IAAI,CAAC,YAAY,CAAC;gBAAE,cAAc;YAAK;YAC/D,OAAOA,WAAW;QACpB,EAAE,OAAOL,OAAO;YACdO,QAAQ,KAAK,CAAC,oCAAoCP;QACpD;IAEF;IAEQ,mBACNQ,IAAmB,EACnBC,UAAsC,EACtCC,KAAsB,EAChB;QACN,IAAI,CAACA,SAAS,CAACD,YACb;QAGF,MAAME,eAAsC;YAC1C,MAAM;YACN,IAAIR,KAAK,GAAG;YACZM;YACA,QAAQC;QACV;QAEA,IAAI,CAACF,KAAK,QAAQ,EAAE;YAClBA,KAAK,QAAQ,GAAG;gBAACG;aAAa;YAC9B;QACF;QACAH,KAAK,QAAQ,CAAC,IAAI,CAACG;IACrB;IAEQ,kBAAkBH,IAAwB,EAAiB;QACjE,OAAO;YACL,QAAQI;YACR,QAAQ;YACR,GAAGJ,IAAI;QACT;IACF;IAEQ,yBACNP,OAAoC,EACpCY,YAAqB,EACf;QACN,IAAI,AAAgB,YAAhB,IAAI,CAAC,MAAM,EACb;QAEFC,OACEb,SAAS,gBACTY,gBACE,CAAC,qDAAqD,EAAE,IAAI,CAAC,eAAe,IAAI,MAAM,EAAE,EAAE,IAAI,CAAC,eAAe,IAAI,YAAY;QAGlI,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,YAAY;IACpD;IAEA,MAAM,OACJL,IAA+C,EAC/CP,OAAoC,EACrB;QACf,IAAI,CAAC,wBAAwB,CAC3BA,SACA,CAAC,yDAAyD,EAAE,IAAI,CAAC,eAAe,IAAI,MAAM,EAAE,EAAE,IAAI,CAAC,eAAe,IAAI,YAAY;QAEpI,IAAIc,MAAM,OAAO,CAACP,OAChB,IAAI,CAAC,KAAK,CAAC,IAAI,IAAIA,KAAK,GAAG,CAAC,CAACQ,OAAS,IAAI,CAAC,iBAAiB,CAACA;aAE7D,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAACR;QAEzC,IAAI,AAAgB,cAAhB,IAAI,CAAC,MAAM,EACb,IAAI,CAAC,MAAM,GAAG;QAEhB,MAAM,IAAI,CAAC,gBAAgB;IAC7B;IAEA,MAAM,eACJA,IAA+C,EAC/CP,OAAoC,EACoB;QACxD,MAAM,IAAI,CAAC,MAAM,CAACO,MAAMP;QACxB,OAAO,IAAI,CAAC,KAAK,CAACA;IACpB;IAEA,MAAM,MACJA,OAAoC,EACoB;QACxD,IAAI,AAAgB,WAAhB,IAAI,CAAC,MAAM,IAAe,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,GAChDM,QAAQ,IAAI,CACV;QAIJ,IAAI,CAAC,wBAAwB,CAACN,SAAS;QACvCa,OAAO,AAAgB,cAAhB,IAAI,CAAC,MAAM,EAAgB;QAClCA,OAAO,AAAgB,gBAAhB,IAAI,CAAC,MAAM,EAAkB;QAEpC,MAAMG,mBAAmB,IAAI,CAAC,KAAK,CAAC,SAAS,CAC3C,CAACT,OAASA,AAAgB,cAAhBA,KAAK,MAAM;QAEvB,IAAIS,mBAAmB,GAErB;QAGF,IAAI,CAAC,MAAM,GAAG;QACd,MAAM,IAAI,CAAC,gBAAgB;QAC3B,IAAIC,YAAYD;QAChB,IAAIE,wBAAwB;QAE5B,IAAIC;QAEJ,MAAOF,YAAY,IAAI,CAAC,KAAK,CAAC,MAAM,CAAE;YACpC,MAAMV,OAAO,IAAI,CAAC,KAAK,CAACU,UAAU;YAClCJ,OACEN,AAAgB,cAAhBA,KAAK,MAAM,EACX,CAAC,wCAAwC,EAAEA,KAAK,MAAM,EAAE;YAE1DA,KAAK,MAAM,GAAG;gBACZ,OAAOL,KAAK,GAAG;YACjB;YACA,IAAI;gBACFK,KAAK,MAAM,GAAG;gBACd,MAAM,IAAI,CAAC,gBAAgB;gBAC3B,IAAI;oBACF,IAAI,IAAI,CAAC,WAAW,EAClB,MAAM,IAAI,CAAC,WAAW,CAACA;gBAE3B,EAAE,OAAOa,GAAG;oBACVd,QAAQ,KAAK,CAAC,wBAAwBc;gBACxC;gBACAP,OACE;oBAAC;oBAAW;oBAAgB;iBAAW,CAAC,OAAO,CAACN,KAAK,IAAI,KAAK,GAC9D,CAAC,uBAAuB,EAAEA,KAAK,IAAI,EAAE;gBAGvC,MAAM,EAAEc,QAAQ,EAAEC,KAAK,EAAE,GAAGf;gBAC5BM,OAAOQ,UAAU,CAAC,oCAAoC,EAAEd,KAAK,IAAI,EAAE;gBAEnE,IAAIgB;gBAGJ,MAAMC,eAAejB,AAAc,cAAdA,KAAK,IAAI;gBAC9BkB,mBAAmBlB,KAAK,MAAM,EAAE;gBAChC,MAAMH,YAAY,MAAM,IAAI,CAAC,YAAY,CAAC;oBAAEoB;gBAAa;gBACzDC,mBAAmBlB,KAAK,MAAM,EAAE;gBAEhCA,KAAK,SAAS,GAAGH;gBACjB,MAAMsB,kBAAmC;oBACvCnB;oBACA,SAASY,oBAAoB;oBAC7Bf;gBACF;gBAEA,IAAIG,AAAc,cAAdA,KAAK,IAAI,EAAgB;oBAC3BM,OACEN,AAAiB,YAAjBA,KAAK,OAAO,IACVA,AAAiB,aAAjBA,KAAK,OAAO,IACZA,AAAiB,cAAjBA,KAAK,OAAO,IACZA,AAAiB,cAAjBA,KAAK,OAAO,IACZA,AAAiB,aAAjBA,KAAK,OAAO,IACZA,AAAiB,aAAjBA,KAAK,OAAO,EACd,CAAC,6BAA6B,EAAEA,KAAK,OAAO,EAAE;oBAEhDgB,cAAc,MAAMhB,KAAK,QAAQ,CAACe,OAAOI;gBAC3C,OAAO,IAAInB,AAAc,eAAdA,KAAK,IAAI,EAAiB;oBACnCgB,cAAc,MAAMhB,KAAK,QAAQ,CAACe,OAAOI;oBACzC,IAAInB,AAAiB,aAAjBA,KAAK,OAAO,EACdY,qBACEI,aACC;gBAEP,OAAO,IAAIhB,AAAc,mBAAdA,KAAK,IAAI,EAClBgB,cAAc,MAAMhB,KAAK,QAAQ,CAACe,OAAOI;qBACpC;oBACLpB,QAAQ,IAAI,CACV,CAAC,uBAAuB,EAAEC,KAAK,IAAI,CAAC,iCAAiC,CAAC;oBAExEgB,cAAc,MAAMhB,KAAK,QAAQ,CAACe,OAAOI;gBAC3C;gBAEA,MAAMC,aAAaV,cAAc,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG;gBAErD,IAAIU,YAAY;oBACdF,mBAAmBlB,KAAK,MAAM,EAAE;oBAChC,MAAMC,aAAa,MAAM,IAAI,CAAC,iBAAiB;oBAC/C,IAAI,CAAC,kBAAkB,CAACD,MAAMC,YAAY;oBAC1CiB,mBAAmBlB,KAAK,MAAM,EAAE;gBAClC;gBAEAqB,OAAO,MAAM,CAACrB,MAAMgB;gBACpBhB,KAAK,MAAM,GAAG;gBACdA,KAAK,MAAM,CAAC,GAAG,GAAGL,KAAK,GAAG;gBAC1BK,KAAK,MAAM,CAAC,IAAI,GAAGA,KAAK,MAAM,CAAC,GAAG,GAAGA,KAAK,MAAM,CAAC,KAAK;gBACtD,MAAM,IAAI,CAAC,gBAAgB;gBAC3BU;YACF,EAAE,OAAOG,GAAQ;gBACfF,wBAAwB;gBACxBX,KAAK,KAAK,GAAGa;gBACbb,KAAK,YAAY,GACfa,GAAG,WAAY,CAAa,YAAb,OAAOA,IAAiBA,IAAI,uBAAsB;gBACnEb,KAAK,UAAU,GAAGa,EAAE,KAAK;gBAEzBb,KAAK,MAAM,GAAG;gBACdA,KAAK,MAAM,CAAC,GAAG,GAAGL,KAAK,GAAG;gBAC1BK,KAAK,MAAM,CAAC,IAAI,GAAGA,KAAK,MAAM,CAAC,GAAG,GAAGA,KAAK,MAAM,CAAC,KAAK;gBACtD,MAAM,IAAI,CAAC,gBAAgB;gBAC3B;YACF;QACF;QAGA,IAAK,IAAIsB,IAAIZ,YAAY,GAAGY,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,EAAEA,IACjD,IAAI,CAAC,KAAK,CAACA,EAAE,CAAC,MAAM,GAAG;QAEzB,IAAIZ,YAAY,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,EACnC,MAAM,IAAI,CAAC,gBAAgB;QAG7B,IAAIa;QACJ,IAAKZ,uBAYE;YACL,IAAI,CAAC,MAAM,GAAG;YACd,MAAM,IAAI,CAAC,gBAAgB;QAC7B,OAf4B;YAC1B,IAAI,CAAC,MAAM,GAAG;YACd,MAAMa,YAAY,IAAI,CAAC,eAAe;YACtC,MAAMC,cACJD,WAAW,gBACVA,CAAAA,WAAW,QAAQE,OAAOF,UAAU,KAAK,IAAI,uBAAsB;YACtE,MAAMG,QAAQH,WAAW;YACzB,MAAMI,UAAUD,QAAQ,GAAGF,YAAY,EAAE,EAAEE,OAAO,GAAGF;YACrDF,gBAAgB,IAAIM,mBAAmBD,SAAS,IAAI,EAAEJ,WAAW;gBAC/D,OAAOA,WAAW;YACpB;YACA,MAAM,IAAI,CAAC,gBAAgB,CAACD;QAC9B;QAKA,IAAIA,eACF,MAAMA;QAGR,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE;YAErB,MAAMO,cAAcC,KAAK,GAAG,CAACrB,WAAW,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG;YAC5D,MAAM,EAAEsB,OAAO,EAAEC,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAACH,YAAY;YACnD,OAAO;gBACLE;gBACAC;YACF;QACF;IACF;IAEA,iBAA0B;QACxB,OAAO,AAAgB,YAAhB,IAAI,CAAC,MAAM;IACpB;IAEA,kBAAwC;QACtC,IAAI,AAAgB,YAAhB,IAAI,CAAC,MAAM,EACb,OAAO;QAIT,IAAK,IAAIX,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,GAAGA,KAAK,GAAGA,IAC1C,IAAI,AAAyB,aAAzB,IAAI,CAAC,KAAK,CAACA,EAAE,CAAC,MAAM,EACtB,OAAO,IAAI,CAAC,KAAK,CAACA,EAAE;QAGxB,OAAO;IACT;IAEA,OAAsB;QACpB,OAAO,IAAIY,cAAc;YACvB,IAAI,IAAI,CAAC,EAAE;YACX,SAAS,IAAI,CAAC,gBAAgB;YAC9B,MAAM,IAAI,CAAC,IAAI;YACf,OAAO,IAAI,CAAC,KAAK;QACnB;IACF;IAEA,MAAM,gBAAgBC,QAAgB,EAGnC;QACD,MAAMX,YAAgE;YACpE,MAAM;YACN,SAAS;YACT,OAAO;gBACL,SAASW;YACX;YACA,SAASA;YACT,UAAU;gBACR,MAAM,IAAIC,MAAMD,YAAY;YAC9B;QACF;QACA,MAAM,IAAI,CAAC,cAAc,CAACX;QAE1B,OAAO;YACL,QAAQ1B;YACR,QAAQ,IAAI;QACd;IACF;IA3WA,YACEuC,IAAY,EACZC,gBAA0C,EAC1C7C,OAA+B,CAC/B;QAtBF,uBAAS,MAAT;QACA;QAEA;QAGA;QAEA;QAEA,uBAAiB,oBAAjB;QAEA,uBAAiB,gBAAjB;QAIA,uBAAiB,oBAAjB;QA2BA,uBAAQ,iBAAR;QApBE,IAAI,CAAC,EAAE,GAAGW;QACV,IAAI,CAAC,MAAM,GACTX,SAAS,SAASA,QAAQ,KAAK,CAAC,MAAM,GAAG,IAAI,YAAY;QAC3D,IAAI,CAAC,IAAI,GAAG4C;QACZ,IAAI,CAAC,KAAK,GAAI5C,AAAAA,CAAAA,SAAS,SAAS,EAAC,EAAG,GAAG,CAAC,CAACe,OACvC,IAAI,CAAC,iBAAiB,CAACA;QAEzB,IAAI,CAAC,WAAW,GAAGf,SAAS;QAC5B,IAAI,CAAC,gBAAgB,GAAG6C;QACxB,IAAI,CAAC,YAAY,GAAG7C,SAAS;QAC7B,IAAI,CAAC,gBAAgB,GAAGE,KAAK,GAAG;IAClC;AA4VF;AAEO,MAAMkC,2BAA2BO;IAKtC,YACER,OAAe,EACfW,MAAkB,EAClBf,SAA+B,EAC/B/B,OAA6B,CAC7B;QACA,KAAK,CAACmC,SAASnC,UAVjB,0CAEA;QASE,IAAI,CAAC,MAAM,GAAG8C;QACd,IAAI,CAAC,SAAS,GAAGf;IACnB;AACF"}

package/dist/es/task-timing.mjs

@@ -1,4 +1,4 @@
-import { getDebug } from "@midscene/shared/logger";
+import { getDebug } from "@aiscene/shared/logger";
 const debugTiming = getDebug('task-timing');
 function setTimingFieldOnce(timing, field) {
     if (!timing) return void debugTiming(`[warning] timing object missing, skip set. field=${field}`);

package/dist/es/task-timing.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"task-timing.mjs","sources":["../../src/task-timing.ts"],"sourcesContent":["import type { ExecutionTask } from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\n\nconst debugTiming = getDebug('task-timing');\n\ntype ExecutionTaskTiming = NonNullable<ExecutionTask['timing']>;\n\ntype NumericTimingField = {\n  [K in keyof ExecutionTaskTiming]-?: ExecutionTaskTiming[K] extends\n    | number\n    | undefined\n    ? K\n    : never;\n}[keyof ExecutionTaskTiming];\n\nexport type TimingSettableField = Exclude<\n  NumericTimingField,\n  'start' | 'end' | 'cost'\n>;\n\nexport function setTimingFieldOnce(\n  timing: ExecutionTaskTiming | undefined,\n  field: TimingSettableField,\n): void {\n  if (!timing) {\n    debugTiming(`[warning] timing object missing, skip set. field=${field}`);\n    return;\n  }\n\n  const value = Date.now();\n  const existingValue = timing[field];\n  if (existingValue !== undefined) {\n    debugTiming(\n      `[warning] duplicate timing field set ignored. field=${field}, existing=${existingValue}, incoming=${value}`,\n    );\n    return;\n  }\n\n  timing[field] = value;\n}\n"],"names":["debugTiming","getDebug","setTimingFieldOnce","timing","field","value","Date","existingValue","undefined"],"mappings":";AAGA,MAAMA,cAAcC,SAAS;AAiBtB,SAASC,mBACdC,MAAuC,EACvCC,KAA0B;IAE1B,IAAI,CAACD,QAAQ,YACXH,YAAY,CAAC,iDAAiD,EAAEI,OAAO;IAIzE,MAAMC,QAAQC,KAAK,GAAG;IACtB,MAAMC,gBAAgBJ,MAAM,CAACC,MAAM;IACnC,IAAIG,AAAkBC,WAAlBD,eAA6B,YAC/BP,YACE,CAAC,oDAAoD,EAAEI,MAAM,WAAW,EAAEG,cAAc,WAAW,EAAEF,OAAO;IAKhHF,MAAM,CAACC,MAAM,GAAGC;AAClB"}
+{"version":3,"file":"task-timing.mjs","sources":["../../src/task-timing.ts"],"sourcesContent":["import type { ExecutionTask } from '@/types';\nimport { getDebug } from '@aiscene/shared/logger';\n\nconst debugTiming = getDebug('task-timing');\n\ntype ExecutionTaskTiming = NonNullable<ExecutionTask['timing']>;\n\ntype NumericTimingField = {\n  [K in keyof ExecutionTaskTiming]-?: ExecutionTaskTiming[K] extends\n    | number\n    | undefined\n    ? K\n    : never;\n}[keyof ExecutionTaskTiming];\n\nexport type TimingSettableField = Exclude<\n  NumericTimingField,\n  'start' | 'end' | 'cost'\n>;\n\nexport function setTimingFieldOnce(\n  timing: ExecutionTaskTiming | undefined,\n  field: TimingSettableField,\n): void {\n  if (!timing) {\n    debugTiming(`[warning] timing object missing, skip set. field=${field}`);\n    return;\n  }\n\n  const value = Date.now();\n  const existingValue = timing[field];\n  if (existingValue !== undefined) {\n    debugTiming(\n      `[warning] duplicate timing field set ignored. field=${field}, existing=${existingValue}, incoming=${value}`,\n    );\n    return;\n  }\n\n  timing[field] = value;\n}\n"],"names":["debugTiming","getDebug","setTimingFieldOnce","timing","field","value","Date","existingValue","undefined"],"mappings":";AAGA,MAAMA,cAAcC,SAAS;AAiBtB,SAASC,mBACdC,MAAuC,EACvCC,KAA0B;IAE1B,IAAI,CAACD,QAAQ,YACXH,YAAY,CAAC,iDAAiD,EAAEI,OAAO;IAIzE,MAAMC,QAAQC,KAAK,GAAG;IACtB,MAAMC,gBAAgBJ,MAAM,CAACC,MAAM;IACnC,IAAIG,AAAkBC,WAAlBD,eAA6B,YAC/BP,YACE,CAAC,oDAAoD,EAAEI,MAAM,WAAW,EAAEG,cAAc,WAAW,EAAEF,OAAO;IAKhHF,MAAM,CAACC,MAAM,GAAGC;AAClB"}

package/dist/es/tree.mjs

@@ -1,4 +1,4 @@
-import { descriptionOfTree, treeToList, trimAttributes, truncateText } from "@midscene/shared/extractor";
+import { descriptionOfTree, treeToList, trimAttributes, truncateText } from "@aiscene/shared/extractor";
 const ELEMENT_COUNT_WARNING_THRESHOLD = 5000;
 const TREE_SIZE_WARNING_MESSAGE = 'The number of elements is too large, it may cause the prompt to be too long, please use domIncluded: "visible-only" to reduce the number of elements';
 function tree_descriptionOfTree(tree, truncateTextLength, filterNonTextContent = false, visibleOnly = true) {

package/dist/es/tree.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"tree.mjs","sources":["../../src/tree.ts"],"sourcesContent":["import type { BaseElement, ElementTreeNode } from '@midscene/shared/types';\n\nimport {\n  descriptionOfTree as sharedDescriptionOfTree,\n  treeToList,\n  trimAttributes,\n  truncateText,\n} from '@midscene/shared/extractor';\n\nconst ELEMENT_COUNT_WARNING_THRESHOLD = 5000;\nconst TREE_SIZE_WARNING_MESSAGE =\n  'The number of elements is too large, it may cause the prompt to be too long, please use domIncluded: \"visible-only\" to reduce the number of elements';\n\nexport { trimAttributes, truncateText };\n\nexport function descriptionOfTree<\n  ElementType extends BaseElement = BaseElement,\n>(\n  tree: ElementTreeNode<ElementType>,\n  truncateTextLength?: number,\n  filterNonTextContent = false,\n  visibleOnly = true,\n) {\n  if (!visibleOnly) {\n    const flatElements = treeToList(tree);\n    if (flatElements.length >= ELEMENT_COUNT_WARNING_THRESHOLD) {\n      console.warn(TREE_SIZE_WARNING_MESSAGE);\n    }\n  }\n\n  return sharedDescriptionOfTree(\n    tree,\n    truncateTextLength,\n    filterNonTextContent,\n    visibleOnly,\n  );\n}\n"],"names":["ELEMENT_COUNT_WARNING_THRESHOLD","TREE_SIZE_WARNING_MESSAGE","descriptionOfTree","tree","truncateTextLength","filterNonTextContent","visibleOnly","flatElements","treeToList","console","sharedDescriptionOfTree"],"mappings":";AASA,MAAMA,kCAAkC;AACxC,MAAMC,4BACJ;AAIK,SAASC,uBAGdC,IAAkC,EAClCC,kBAA2B,EAC3BC,uBAAuB,KAAK,EAC5BC,cAAc,IAAI;IAElB,IAAI,CAACA,aAAa;QAChB,MAAMC,eAAeC,WAAWL;QAChC,IAAII,aAAa,MAAM,IAAIP,iCACzBS,QAAQ,IAAI,CAACR;IAEjB;IAEA,OAAOS,kBACLP,MACAC,oBACAC,sBACAC;AAEJ"}
+{"version":3,"file":"tree.mjs","sources":["../../src/tree.ts"],"sourcesContent":["import type { BaseElement, ElementTreeNode } from '@aiscene/shared/types';\n\nimport {\n  descriptionOfTree as sharedDescriptionOfTree,\n  treeToList,\n  trimAttributes,\n  truncateText,\n} from '@aiscene/shared/extractor';\n\nconst ELEMENT_COUNT_WARNING_THRESHOLD = 5000;\nconst TREE_SIZE_WARNING_MESSAGE =\n  'The number of elements is too large, it may cause the prompt to be too long, please use domIncluded: \"visible-only\" to reduce the number of elements';\n\nexport { trimAttributes, truncateText };\n\nexport function descriptionOfTree<\n  ElementType extends BaseElement = BaseElement,\n>(\n  tree: ElementTreeNode<ElementType>,\n  truncateTextLength?: number,\n  filterNonTextContent = false,\n  visibleOnly = true,\n) {\n  if (!visibleOnly) {\n    const flatElements = treeToList(tree);\n    if (flatElements.length >= ELEMENT_COUNT_WARNING_THRESHOLD) {\n      console.warn(TREE_SIZE_WARNING_MESSAGE);\n    }\n  }\n\n  return sharedDescriptionOfTree(\n    tree,\n    truncateTextLength,\n    filterNonTextContent,\n    visibleOnly,\n  );\n}\n"],"names":["ELEMENT_COUNT_WARNING_THRESHOLD","TREE_SIZE_WARNING_MESSAGE","descriptionOfTree","tree","truncateTextLength","filterNonTextContent","visibleOnly","flatElements","treeToList","console","sharedDescriptionOfTree"],"mappings":";AASA,MAAMA,kCAAkC;AACxC,MAAMC,4BACJ;AAIK,SAASC,uBAGdC,IAAkC,EAClCC,kBAA2B,EAC3BC,uBAAuB,KAAK,EAC5BC,cAAc,IAAI;IAElB,IAAI,CAACA,aAAa;QAChB,MAAMC,eAAeC,WAAWL;QAChC,IAAII,aAAa,MAAM,IAAIP,iCACzBS,QAAQ,IAAI,CAACR;IAEjB;IAEA,OAAOS,kBACLP,MACAC,oBACAC,sBACAC;AAEJ"}

package/dist/es/types.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"types.mjs","sources":["../../src/types.ts"],"sourcesContent":["/* eslint-disable @typescript-eslint/no-explicit-any */\n\nimport {\n  existsSync,\n  mkdirSync,\n  readFileSync,\n  rmSync,\n  writeFileSync,\n} from 'node:fs';\nimport { join } from 'node:path';\nimport type { NodeType } from '@midscene/shared/constants';\nimport type { CreateOpenAIClientFn, TModelConfig } from '@midscene/shared/env';\nimport type {\n  BaseElement,\n  LocateResultElement,\n  Rect,\n  Size,\n} from '@midscene/shared/types';\nimport type { z } from 'zod';\nimport type { TUserPrompt } from './common';\nimport { restoreImageReferences } from './dump/screenshot-restoration';\nimport { ScreenshotStore } from './dump/screenshot-store';\nimport { ScreenshotItem } from './screenshot-item';\nimport type {\n  DetailedLocateParam,\n  MidsceneYamlFlowItem,\n  ServiceExtractOption,\n} from './yaml';\n\nexport type {\n  ElementTreeNode,\n  BaseElement,\n  Rect,\n  Size,\n  Point,\n} from '@midscene/shared/types';\nexport * from './yaml';\n\nexport type AIUsageInfo = Record<string, any> & {\n  prompt_tokens: number | undefined;\n  completion_tokens: number | undefined;\n  total_tokens: number | undefined;\n  cached_input: number | undefined;\n  time_cost: number | undefined;\n  model_name: string | undefined;\n  model_description: string | undefined;\n  intent: string | undefined;\n  request_id: string | undefined;\n};\n\nexport type { LocateResultElement };\n\nexport type AISingleElementResponseByPosition = {\n  position?: {\n    x: number;\n    y: number;\n  };\n  bbox?: [number, number, number, number];\n  reason: string;\n  text: string;\n};\n\nexport interface AIElementCoordinatesResponse {\n  bbox: [number, number, number, number];\n  errors?: string[];\n}\n\nexport type AIElementResponse = AIElementCoordinatesResponse;\n\nexport interface AIDataExtractionResponse<DataDemand> {\n  data: DataDemand;\n  errors?: string[];\n  thought?: string;\n}\n\nexport interface AISectionLocatorResponse {\n  bbox: [number, number, number, number];\n  references_bbox?: [number, number, number, number][];\n  error?: string;\n}\n\nexport interface AIAssertionResponse {\n  pass: boolean;\n  thought: string;\n}\n\nexport interface AIDescribeElementResponse {\n  description: string;\n  error?: string;\n}\n\nexport interface LocatorValidatorOption {\n  centerDistanceThreshold?: number;\n}\n\nexport interface LocateValidatorResult {\n  pass: boolean;\n  rect: Rect;\n  center: [number, number];\n  centerDistance?: number;\n}\n\nexport interface AgentDescribeElementAtPointResult {\n  prompt: string;\n  deepLocate: boolean;\n  verifyResult?: LocateValidatorResult;\n}\n\n/**\n * context\n */\n\nexport abstract class UIContext {\n  /**\n   * screenshot of the current UI state. which size is shotSize(be shrunk by screenshotShrinkFactor),\n   */\n  abstract screenshot: ScreenshotItem;\n\n  /**\n   * screenshot size after shrinking\n   */\n  abstract shotSize: Size;\n\n  /**\n   * The ratio for converting shrunk screenshot coordinates to logical coordinates.\n   *\n   * Example:\n   * - Physical screen width: 3000px, dpr=6\n   * - Logical width: 500px\n   * - User-defined screenshotShrinkFactor: 2\n   * - Actual shrunk screenshot width: 3000 / 2 = 1500px\n   * - shrunkShotToLogicalRatio: dpr / screenshotShrinkFactor = 6 / 2 = 3\n   * - To map back to logical coordinates: 1500 / shrunkShotToLogicalRatio = 500px\n   */\n  abstract shrunkShotToLogicalRatio: number;\n\n  abstract _isFrozen?: boolean;\n\n  // @deprecated - backward compatibility for aiLocate\n  abstract deprecatedDpr?: number;\n}\n\nexport type EnsureObject<T> = { [K in keyof T]: any };\n\nexport type ServiceAction = 'locate' | 'extract' | 'assert' | 'describe';\n\nexport type ServiceExtractParam = string | Record<string, string>;\n\nexport type ElementCacheFeature = Record<string, unknown>;\n\nexport interface LocateResult {\n  element: LocateResultElement | null;\n  rect?: Rect;\n}\n\nexport type ThinkingLevel = 'off' | 'medium' | 'high';\n\nexport type DeepThinkOption = 'unset' | true | false;\n\nexport interface ServiceTaskInfo {\n  durationMs: number;\n  formatResponse?: string;\n  rawResponse?: string;\n  usage?: AIUsageInfo;\n  searchArea?: Rect;\n  searchAreaRawResponse?: string;\n  searchAreaUsage?: AIUsageInfo;\n  reasoning_content?: string;\n}\n\nexport interface DumpMeta {\n  logTime: number;\n}\n\nexport type ReportAttributes = Record<\n  string,\n  string | number | boolean | null | undefined\n>;\n\nexport interface ReportDumpWithAttributes {\n  dumpString: string;\n  attributes?: ReportAttributes;\n}\n\nexport interface ServiceDump extends DumpMeta {\n  type: 'locate' | 'extract' | 'assert';\n  logId: string;\n  userQuery: {\n    element?: TUserPrompt;\n    dataDemand?: ServiceExtractParam;\n    assertion?: TUserPrompt;\n  };\n  matchedElement: LocateResultElement[];\n  matchedRect?: Rect;\n  deepLocate?: boolean;\n  data: any;\n  assertionPass?: boolean;\n  assertionThought?: string;\n  taskInfo: ServiceTaskInfo;\n  error?: string;\n  output?: any;\n}\n\nexport type PartialServiceDumpFromSDK = Omit<\n  ServiceDump,\n  'logTime' | 'logId' | 'model_name'\n>;\n\nexport interface ServiceResultBase {\n  dump: ServiceDump;\n}\n\nexport type LocateResultWithDump = LocateResult & ServiceResultBase;\n\nexport interface ServiceExtractResult<T> extends ServiceResultBase {\n  data: T;\n  thought?: string;\n  usage?: AIUsageInfo;\n  reasoning_content?: string;\n}\n\nexport class ServiceError extends Error {\n  dump: ServiceDump;\n\n  constructor(message: string, dump: ServiceDump) {\n    super(message);\n    this.name = 'ServiceError';\n    this.dump = dump;\n  }\n}\n\n// intermediate variables to optimize the return value by AI\nexport interface LiteUISection {\n  name: string;\n  description: string;\n  sectionCharacteristics: string;\n  textIds: string[];\n}\n\nexport type ElementById = (id: string) => BaseElement | null;\n\nexport type ServiceAssertionResponse = AIAssertionResponse & {\n  usage?: AIUsageInfo;\n};\n\n/**\n * agent\n */\n\nexport type OnTaskStartTip = (tip: string) => Promise<void> | void;\n\nexport interface AgentWaitForOpt extends ServiceExtractOption {\n  checkIntervalMs?: number;\n  timeoutMs?: number;\n}\n\nexport interface AgentAssertOpt {\n  keepRawResponse?: boolean;\n}\n\n/**\n * planning\n *\n */\n\nexport interface PlanningLocateParam extends DetailedLocateParam {\n  bbox?: [number, number, number, number];\n}\n\nexport interface PlanningAction<ParamType = any> {\n  thought?: string;\n  log?: string; // a brief preamble to the user explaining what you’re about to do\n  type: string;\n  param: ParamType;\n}\n\nexport type SubGoalStatus = 'pending' | 'running' | 'finished';\n\nexport interface SubGoal {\n  index: number;\n  status: SubGoalStatus;\n  description: string;\n  logs?: string[];\n}\n\nexport interface RawResponsePlanningAIResponse {\n  action: PlanningAction;\n  thought?: string;\n  log: string;\n  memory?: string;\n  error?: string;\n  finalizeMessage?: string;\n  finalizeSuccess?: boolean;\n  updateSubGoals?: SubGoal[];\n  markFinishedIndexes?: number[];\n}\n\nexport interface PlanningAIResponse\n  extends Omit<RawResponsePlanningAIResponse, 'action'> {\n  actions?: PlanningAction[];\n  usage?: AIUsageInfo;\n  rawResponse?: string;\n  yamlFlow?: MidsceneYamlFlowItem[];\n  yamlString?: string;\n  error?: string;\n  reasoning_content?: string;\n  shouldContinuePlanning: boolean;\n  output?: string; // Output message from <complete> tag (same as finalizeMessage)\n}\n\nexport interface PlanningActionParamSleep {\n  timeMs: number;\n}\n\nexport interface PlanningActionParamError {\n  thought: string;\n}\n\nexport type PlanningActionParamWaitFor = AgentWaitForOpt & {};\n\nexport interface LongPressParam {\n  duration?: number;\n}\n\nexport interface PullParam {\n  direction: 'up' | 'down';\n  distance?: number;\n  duration?: number;\n}\n/**\n * misc\n */\n\nexport interface Color {\n  name: string;\n  hex: string;\n}\n\nexport interface BaseAgentParserOpt {\n  selector?: string;\n}\n// eslint-disable-next-line @typescript-eslint/no-empty-interface\nexport interface PuppeteerParserOpt extends BaseAgentParserOpt {}\n\n// eslint-disable-next-line @typescript-eslint/no-empty-interface\nexport interface PlaywrightParserOpt extends BaseAgentParserOpt {}\n\n/*\naction\n*/\nexport interface ExecutionTaskProgressOptions {\n  onTaskStart?: (task: ExecutionTask) => Promise<void> | void;\n}\n\nexport interface ExecutionRecorderItem {\n  type: 'screenshot';\n  ts: number;\n  screenshot?: ScreenshotItem;\n  timing?: string;\n}\n\nexport type ExecutionTaskType = 'Planning' | 'Insight' | 'Action Space' | 'Log';\n\nexport interface ExecutorContext {\n  task: ExecutionTask;\n  element?: LocateResultElement | null;\n  uiContext?: UIContext;\n}\n\nexport interface ExecutionTaskApply<\n  Type extends ExecutionTaskType = any,\n  TaskParam = any,\n  TaskOutput = any,\n  TaskLog = any,\n> {\n  type: Type;\n  subType?: string;\n  param?: TaskParam;\n  thought?: string;\n  uiContext?: UIContext;\n  executor: (\n    param: TaskParam,\n    context: ExecutorContext,\n  ) => // biome-ignore lint/suspicious/noConfusingVoidType: void is intentionally allowed as some executors may not return a value\n    | Promise<ExecutionTaskReturn<TaskOutput, TaskLog> | undefined | void>\n    | undefined\n    | void;\n}\n\nexport interface ExecutionTaskHitBy {\n  from: string;\n  context: Record<string, any>;\n}\n\nexport interface ExecutionTaskReturn<TaskOutput = unknown, TaskLog = unknown> {\n  output?: TaskOutput;\n  log?: TaskLog;\n  recorder?: ExecutionRecorderItem[];\n  hitBy?: ExecutionTaskHitBy;\n}\n\nexport type ExecutionTask<\n  E extends ExecutionTaskApply<any, any, any> = ExecutionTaskApply<\n    any,\n    any,\n    any\n  >,\n> = E &\n  ExecutionTaskReturn<\n    E extends ExecutionTaskApply<any, any, infer TaskOutput, any>\n      ? TaskOutput\n      : unknown,\n    E extends ExecutionTaskApply<any, any, any, infer TaskLog>\n      ? TaskLog\n      : unknown\n  > & {\n    taskId: string;\n    status: 'pending' | 'running' | 'finished' | 'failed' | 'cancelled';\n    error?: Error;\n    errorMessage?: string;\n    errorStack?: string;\n    timing?: {\n      start: number;\n      getUiContextStart?: number;\n      getUiContextEnd?: number;\n      callAiStart?: number;\n      callAiEnd?: number;\n      beforeInvokeActionHookStart?: number;\n      beforeInvokeActionHookEnd?: number;\n      callActionStart?: number;\n      callActionEnd?: number;\n      afterInvokeActionHookStart?: number;\n      afterInvokeActionHookEnd?: number;\n      captureAfterCallingSnapshotStart?: number;\n      captureAfterCallingSnapshotEnd?: number;\n      end?: number;\n      cost?: number;\n    };\n    usage?: AIUsageInfo;\n    searchAreaUsage?: AIUsageInfo;\n    reasoning_content?: string;\n  };\n\nexport interface IExecutionDump extends DumpMeta {\n  /** Stable unique identifier for this execution run */\n  id?: string;\n  name: string;\n  description?: string;\n  tasks: ExecutionTask[];\n  aiActContext?: string;\n}\n\n/**\n * Replacer function for JSON serialization that handles Page, Browser objects and ScreenshotItem\n */\nfunction replacerForDumpSerialization(_key: string, value: any): any {\n  if (value && value.constructor?.name === 'Page') {\n    return '[Page object]';\n  }\n  if (value && value.constructor?.name === 'Browser') {\n    return '[Browser object]';\n  }\n  // Handle ScreenshotItem serialization\n  if (value && typeof value.toSerializable === 'function') {\n    return value.toSerializable();\n  }\n  return value;\n}\n\n/**\n * Reviver function for JSON deserialization that keeps screenshot references\n * as plain objects. Resolution is handled lazily by restoreImageReferences.\n *\n * @param key - JSON key being processed\n * @param value - JSON value being processed\n * @returns Restored value\n */\nfunction reviverForDumpDeserialization(key: string, value: any): any {\n  // Only process screenshot fields\n  if (key !== 'screenshot' || typeof value !== 'object' || value === null) {\n    return value;\n  }\n\n  if (ScreenshotItem.isSerialized(value)) {\n    return value;\n  }\n\n  return value;\n}\n\n/**\n * ExecutionDump class for serializing and deserializing execution dumps\n */\nexport class ExecutionDump implements IExecutionDump {\n  id?: string;\n  logTime: number;\n  name: string;\n  description?: string;\n  tasks: ExecutionTask[];\n  aiActContext?: string;\n\n  constructor(data: IExecutionDump) {\n    this.id = data.id;\n    this.logTime = data.logTime;\n    this.name = data.name;\n    this.description = data.description;\n    this.tasks = data.tasks;\n    this.aiActContext = data.aiActContext;\n  }\n\n  /**\n   * Serialize the ExecutionDump to a JSON string\n   */\n  serialize(indents?: number): string {\n    return JSON.stringify(this.toJSON(), replacerForDumpSerialization, indents);\n  }\n\n  /**\n   * Convert to a plain object for JSON serialization\n   */\n  toJSON(): IExecutionDump {\n    return {\n      id: this.id,\n      logTime: this.logTime,\n      name: this.name,\n      description: this.description,\n      tasks: this.tasks.map((task) => ({\n        ...task,\n        recorder: task.recorder || [],\n      })),\n      aiActContext: this.aiActContext,\n    };\n  }\n\n  /**\n   * Create an ExecutionDump instance from a serialized JSON string\n   */\n  static fromSerializedString(serialized: string): ExecutionDump {\n    const parsed = JSON.parse(\n      serialized,\n      reviverForDumpDeserialization,\n    ) as IExecutionDump;\n    return new ExecutionDump(parsed);\n  }\n\n  /**\n   * Create an ExecutionDump instance from a plain object\n   */\n  static fromJSON(data: IExecutionDump): ExecutionDump {\n    return new ExecutionDump(data);\n  }\n\n  /**\n   * Collect all ScreenshotItem instances from tasks.\n   * Scans through uiContext and recorder items to find screenshots.\n   *\n   * @returns Array of ScreenshotItem instances\n   */\n  collectScreenshots(): ScreenshotItem[] {\n    const screenshots: ScreenshotItem[] = [];\n\n    for (const task of this.tasks) {\n      // Collect uiContext.screenshot if present\n      if (task.uiContext?.screenshot instanceof ScreenshotItem) {\n        screenshots.push(task.uiContext.screenshot);\n      }\n\n      // Collect recorder screenshots\n      if (task.recorder) {\n        for (const record of task.recorder) {\n          if (record.screenshot instanceof ScreenshotItem) {\n            screenshots.push(record.screenshot);\n          }\n        }\n      }\n    }\n\n    return screenshots;\n  }\n}\n\n/*\ntask - service-locate\n*/\nexport type ExecutionTaskInsightLocateParam = PlanningLocateParam;\n\nexport interface ExecutionTaskInsightLocateOutput {\n  element: LocateResultElement | null;\n}\n\nexport type ExecutionTaskInsightDump = ServiceDump;\n\nexport type ExecutionTaskInsightLocateApply = ExecutionTaskApply<\n  'Insight',\n  ExecutionTaskInsightLocateParam,\n  ExecutionTaskInsightLocateOutput,\n  ExecutionTaskInsightDump\n>;\n\nexport type ExecutionTaskInsightLocate =\n  ExecutionTask<ExecutionTaskInsightLocateApply>;\n\n/*\ntask - service-query\n*/\nexport interface ExecutionTaskInsightQueryParam {\n  dataDemand: ServiceExtractParam;\n  domIncluded?: boolean | 'visible-only';\n}\n\nexport interface ExecutionTaskInsightQueryOutput {\n  data: any;\n}\n\nexport type ExecutionTaskInsightQueryApply = ExecutionTaskApply<\n  'Insight',\n  ExecutionTaskInsightQueryParam,\n  any,\n  ExecutionTaskInsightDump\n>;\n\nexport type ExecutionTaskInsightQuery =\n  ExecutionTask<ExecutionTaskInsightQueryApply>;\n\n/*\ntask - assertion\n*/\nexport interface ExecutionTaskInsightAssertionParam {\n  assertion: string;\n}\n\nexport type ExecutionTaskInsightAssertionApply = ExecutionTaskApply<\n  'Insight',\n  ExecutionTaskInsightAssertionParam,\n  ServiceAssertionResponse,\n  ExecutionTaskInsightDump\n>;\n\nexport type ExecutionTaskInsightAssertion =\n  ExecutionTask<ExecutionTaskInsightAssertionApply>;\n\n/*\ntask - action (i.e. interact) \n*/\nexport type ExecutionTaskActionApply<ActionParam = any> = ExecutionTaskApply<\n  'Action Space',\n  ActionParam,\n  void,\n  void\n>;\n\nexport type ExecutionTaskAction = ExecutionTask<ExecutionTaskActionApply>;\n\n/*\ntask - Log\n*/\n\nexport type ExecutionTaskLogApply<\n  LogParam = {\n    content: string;\n  },\n> = ExecutionTaskApply<'Log', LogParam, void, void>;\n\nexport type ExecutionTaskLog = ExecutionTask<ExecutionTaskLogApply>;\n\n/*\ntask - planning\n*/\n\nexport type ExecutionTaskPlanningApply = ExecutionTaskApply<\n  'Planning',\n  {\n    userInstruction: string;\n    aiActContext?: string;\n  },\n  PlanningAIResponse\n>;\n\nexport type ExecutionTaskPlanning = ExecutionTask<ExecutionTaskPlanningApply>;\n\n/*\ntask - planning-locate\n*/\nexport type ExecutionTaskPlanningLocateParam = PlanningLocateParam;\n\nexport interface ExecutionTaskPlanningLocateOutput {\n  element: LocateResultElement | null;\n}\n\nexport type ExecutionTaskPlanningDump = ServiceDump;\n\nexport type ExecutionTaskPlanningLocateApply = ExecutionTaskApply<\n  'Planning',\n  ExecutionTaskPlanningLocateParam,\n  ExecutionTaskPlanningLocateOutput,\n  ExecutionTaskPlanningDump\n>;\n\nexport type ExecutionTaskPlanningLocate =\n  ExecutionTask<ExecutionTaskPlanningLocateApply>;\n\n/*\nReport metadata - extracted from ReportActionDump for per-execution writes\n*/\nexport interface ReportMeta {\n  groupName: string;\n  groupDescription?: string;\n  sdkVersion: string;\n  modelBriefs: ModelBrief[];\n  deviceType?: string;\n}\n\n// Backward-compatible aliases for existing external consumers.\nexport type GroupMeta = ReportMeta;\n\n/*\nReport dump\n*/\nexport interface IReportActionDump {\n  sdkVersion: string;\n  groupName: string;\n  groupDescription?: string;\n  modelBriefs: ModelBrief[];\n  executions: IExecutionDump[];\n  deviceType?: string;\n}\n\n// Backward-compatible aliases for existing external consumers.\nexport type IGroupedActionDump = IReportActionDump;\n\nexport interface ModelBrief {\n  /**\n   * The intent/category of the model call, for example \"planning\" or \"insight\".\n   */\n  intent?: string;\n\n  /**\n   * The model name returned by usage metadata, for example \"gpt-4o\".\n   */\n  name?: string;\n\n  /**\n   * Optional human-readable model description, for example \"qwen2.5-vl mode\".\n   */\n  modelDescription?: string;\n}\n\n/**\n * ReportActionDump class for serializing and deserializing report action dumps\n */\nexport class ReportActionDump implements IReportActionDump {\n  sdkVersion: string;\n  groupName: string;\n  groupDescription?: string;\n  modelBriefs: ModelBrief[];\n  executions: ExecutionDump[];\n  deviceType?: string;\n\n  constructor(data: IReportActionDump) {\n    this.sdkVersion = data.sdkVersion;\n    this.groupName = data.groupName;\n    this.groupDescription = data.groupDescription;\n    this.modelBriefs = data.modelBriefs;\n    this.executions = data.executions.map((exec) =>\n      exec instanceof ExecutionDump ? exec : ExecutionDump.fromJSON(exec),\n    );\n    this.deviceType = data.deviceType;\n  }\n\n  /**\n   * Serialize the ReportActionDump to a JSON string\n   * Uses compact { $screenshot: id } format\n   */\n  serialize(indents?: number): string {\n    return JSON.stringify(this.toJSON(), replacerForDumpSerialization, indents);\n  }\n\n  /**\n   * Serialize the ReportActionDump with inline screenshots to a JSON string.\n   * Each ScreenshotItem is replaced with { base64: \"...\", capturedAt }.\n   */\n  serializeWithInlineScreenshots(indents?: number): string {\n    const processValue = (obj: unknown): unknown => {\n      if (obj instanceof ScreenshotItem) {\n        return { base64: obj.base64, capturedAt: obj.capturedAt };\n      }\n      if (Array.isArray(obj)) {\n        return obj.map(processValue);\n      }\n      if (obj && typeof obj === 'object') {\n        const entries = Object.entries(obj).map(([key, value]) => [\n          key,\n          processValue(value),\n        ]);\n        return Object.fromEntries(entries);\n      }\n      return obj;\n    };\n\n    const data = processValue(this.toJSON());\n    return JSON.stringify(data, null, indents);\n  }\n\n  /**\n   * Convert to a plain object for JSON serialization\n   */\n  toJSON(): IReportActionDump {\n    return {\n      sdkVersion: this.sdkVersion,\n      groupName: this.groupName,\n      groupDescription: this.groupDescription,\n      modelBriefs: this.modelBriefs,\n      executions: this.executions.map((exec) => exec.toJSON()),\n      deviceType: this.deviceType,\n    };\n  }\n\n  /**\n   * Create a ReportActionDump instance from a serialized JSON string\n   */\n  static fromSerializedString(serialized: string): ReportActionDump {\n    const parsed = JSON.parse(\n      serialized,\n      reviverForDumpDeserialization,\n    ) as IReportActionDump;\n    return new ReportActionDump(parsed);\n  }\n\n  /**\n   * Create a ReportActionDump instance from a plain object\n   */\n  static fromJSON(data: IReportActionDump): ReportActionDump {\n    return new ReportActionDump(data);\n  }\n\n  /**\n   * Collect all ScreenshotItem instances from all executions.\n   *\n   * @returns Array of all ScreenshotItem instances across all executions\n   */\n  collectAllScreenshots(): ScreenshotItem[] {\n    const screenshots: ScreenshotItem[] = [];\n    for (const execution of this.executions) {\n      screenshots.push(...execution.collectScreenshots());\n    }\n    return screenshots;\n  }\n\n  /**\n   * Serialize the dump to files with screenshots as separate PNG files.\n   * Creates:\n   * - {basePath} - dump JSON with { $screenshot: id } references\n   * - {basePath}.screenshots/ - PNG files\n   *\n   * @param basePath - Base path for the dump file\n   */\n  serializeToFiles(basePath: string): void {\n    const screenshotsDir = `${basePath}.screenshots`;\n    if (!existsSync(screenshotsDir)) {\n      mkdirSync(screenshotsDir, { recursive: true });\n    }\n\n    const screenshots = this.collectAllScreenshots();\n\n    for (const screenshot of screenshots) {\n      const imagePath = join(\n        screenshotsDir,\n        `${screenshot.id}.${screenshot.extension}`,\n      );\n      if (existsSync(imagePath)) {\n        continue;\n      }\n\n      const rawBase64 = screenshot.rawBase64;\n      writeFileSync(imagePath, Buffer.from(rawBase64, 'base64'));\n    }\n\n    // Write dump JSON with references\n    writeFileSync(basePath, this.serialize(), 'utf-8');\n  }\n\n  /**\n   * Read dump from files and return JSON string with inline screenshots.\n   * Reads the dump JSON and screenshot files, then inlines the base64 data.\n   *\n   * @param basePath - Base path for the dump file\n   * @returns JSON string with inline screenshots ({ base64: \"...\" } format)\n   */\n  static fromFilesAsInlineJson(basePath: string): string {\n    const dumpString = readFileSync(basePath, 'utf-8');\n    const screenshotsDir = `${basePath}.screenshots`;\n\n    const loadFromExecutionScreenshotDir = (id: string, mimeType: string) => {\n      const ext = mimeType === 'image/jpeg' ? 'jpeg' : 'png';\n      const filePath = join(screenshotsDir, `${id}.${ext}`);\n      if (!existsSync(filePath)) {\n        return '';\n      }\n      const data = readFileSync(filePath);\n      return `data:image/${ext};base64,${data.toString('base64')}`;\n    };\n\n    // Restore image references\n    const dumpData = JSON.parse(dumpString);\n    const store = new ScreenshotStore({\n      mode: 'directory',\n      reportPath: basePath,\n    });\n    const processedData = restoreImageReferences(dumpData, (ref) => {\n      const executionFileImage = loadFromExecutionScreenshotDir(\n        ref.id,\n        ref.mimeType,\n      );\n      if (executionFileImage) {\n        return executionFileImage;\n      }\n\n      if (ref.storage === 'inline') {\n        return '';\n      }\n      return store.loadBase64(ref);\n    });\n    return JSON.stringify(processedData);\n  }\n\n  /**\n   * Clean up all files associated with a serialized dump.\n   *\n   * @param basePath - Base path for the dump file\n   */\n  static cleanupFiles(basePath: string): void {\n    const filesToClean = [basePath, `${basePath}.screenshots`];\n\n    for (const filePath of filesToClean) {\n      try {\n        rmSync(filePath, { force: true, recursive: true });\n      } catch {\n        // Ignore errors - file may already be deleted\n      }\n    }\n  }\n\n  /**\n   * Get all file paths associated with a serialized dump.\n   *\n   * @param basePath - Base path for the dump file\n   * @returns Array of all associated file paths\n   */\n  static getFilePaths(basePath: string): string[] {\n    return [basePath, `${basePath}.screenshots`];\n  }\n}\n\n// Backward-compatible aliases for existing external consumers.\nexport type GroupedActionDump = ReportActionDump;\nexport const GroupedActionDump = ReportActionDump;\n\nexport type InterfaceType =\n  | 'puppeteer'\n  | 'playwright'\n  | 'static'\n  | 'chrome-extension-proxy'\n  | 'android'\n  | string;\n\nexport interface StreamingCodeGenerationOptions {\n  /** Whether to enable streaming output */\n  stream?: boolean;\n  /** Callback function to handle streaming chunks */\n  onChunk?: StreamingCallback;\n  /** Callback function to handle streaming completion */\n  onComplete?: (finalCode: string) => void;\n  /** Callback function to handle streaming errors */\n  onError?: (error: Error) => void;\n}\n\nexport type StreamingCallback = (chunk: CodeGenerationChunk) => void;\n\nexport interface CodeGenerationChunk {\n  /** The incremental content chunk */\n  content: string;\n  /** The reasoning content */\n  reasoning_content: string;\n  /** The accumulated content so far */\n  accumulated: string;\n  /** Whether this is the final chunk */\n  isComplete: boolean;\n  /** Token usage information if available */\n  usage?: AIUsageInfo;\n}\n\nexport interface StreamingAIResponse {\n  /** The final accumulated content */\n  content: string;\n  /** Token usage information */\n  usage?: AIUsageInfo;\n  /** Whether the response was streamed */\n  isStreamed: boolean;\n}\n\nexport interface DeviceAction<TParam = any, TReturn = any> {\n  name: string;\n  description?: string;\n  interfaceAlias?: string;\n  paramSchema?: z.ZodType<TParam>;\n  call: (param: TParam, context: ExecutorContext) => Promise<TReturn> | TReturn;\n  delayAfterRunner?: number;\n  /**\n   * An example param object for this action.\n   * Locate fields with { prompt } will automatically get bbox injected when needed.\n   */\n  sample?: { [K in keyof TParam]?: any };\n}\n\n/**\n * Type utilities for extracting types from DeviceAction definitions\n */\n\n/**\n * Extract parameter type from a DeviceAction\n */\nexport type ActionParam<Action extends DeviceAction<any, any>> =\n  Action extends DeviceAction<infer P, any> ? P : never;\n\n/**\n * Extract return type from a DeviceAction\n */\nexport type ActionReturn<Action extends DeviceAction<any, any>> =\n  Action extends DeviceAction<any, infer R> ? R : never;\n\n/**\n * Web-specific types\n */\nexport interface WebElementInfo extends BaseElement {\n  id: string;\n  attributes: {\n    nodeType: NodeType;\n    [key: string]: string;\n  };\n}\n\n/**\n * Agent\n */\n\nexport type CacheConfig = {\n  strategy?: 'read-only' | 'read-write' | 'write-only';\n  id: string;\n};\n\nexport type Cache =\n  | false // No read, no write\n  | true // Will throw error at runtime - deprecated\n  | CacheConfig; // Object configuration (requires explicit id)\n\nexport interface AgentOpt {\n  // @deprecated Use `reportFileName` and `cache.id` instead.\n  testId?: string;\n  // @deprecated\n  cacheId?: string; // Keep backward compatibility, but marked as deprecated\n  groupName?: string;\n  groupDescription?: string;\n  /* if auto generate report, default true */\n  generateReport?: boolean;\n  /* if persist per-execution dump files next to the report, default false */\n  persistExecutionDump?: boolean;\n  /* if auto print report msg, default true */\n  autoPrintReportMsg?: boolean;\n\n  /**\n   * Use directory-based report format with separate image files.\n   *\n   * When enabled:\n   * - Screenshots are saved as PNG files in a `screenshots/` subdirectory\n   * - Report is generated as `index.html` with relative image paths\n   * - Reduces memory usage and report file size\n   *\n   * IMPORTANT: 'html-and-external-assets' reports must be served via HTTP server\n   * (e.g., `npx serve ./report-dir`). The file:// protocol will not\n   * work due to browser CORS restrictions.\n   *\n   * @default 'single-html'\n   */\n  outputFormat?: 'single-html' | 'html-and-external-assets';\n\n  onTaskStartTip?: OnTaskStartTip;\n  aiActContext?: string;\n  aiActionContext?: string;\n  /* custom report file name */\n  reportFileName?: string;\n  modelConfig?: TModelConfig;\n  cache?: Cache;\n  /**\n   * Maximum number of replanning cycles for aiAct.\n   * Defaults to 20 (40 for `vlm-ui-tars`) when not provided.\n   * If omitted, the agent will also read `MIDSCENE_REPLANNING_CYCLE_LIMIT` for backward compatibility.\n   */\n  replanningCycleLimit?: number;\n\n  /**\n   * Wait time in milliseconds after each action execution.\n   * This allows the UI to settle and stabilize before the next action.\n   * Defaults to 300ms when not provided.\n   */\n  waitAfterAction?: number;\n\n  /**\n   * When set to true, Midscene will use the target device's time (Android/iOS)\n   * instead of the system time. Useful when the device time differs from the\n   * host machine. Default: false\n   */\n  useDeviceTimestamp?: boolean;\n\n  /**\n   * Custom screenshot shrink factor to reduce AI token usage.\n   * When set, the screenshot will be scaled down by this factor from the physical resolution.\n   *\n   * Example:\n   * - Physical screen width: 3000px, dpr=6\n   * - Logical width: 500px\n   * - screenshotShrinkFactor: 2\n   * - Actual shrunk screenshot width: 3000 / 2 = 1500px\n   * - AI analyzes the 1500px screenshot\n   * - Coordinates are transformed back to logical (500px) before actions execute\n   *\n   * Benefits:\n   * - Reduces token usage for high-resolution screenshots\n   * - Maintains accuracy by scaling coordinates appropriately\n   *\n   * Must be >= 1 (shrinking only, enlarging is not supported).\n   *\n   * @default 1 (no shrinking, uses original physical screenshot)\n   */\n  screenshotShrinkFactor?: number;\n\n  /**\n   * Custom OpenAI client factory function\n   *\n   * If provided, this function will be called to create OpenAI client instances\n   * for each AI call, allowing you to:\n   * - Wrap clients with observability tools (langsmith, langfuse)\n   * - Use custom OpenAI-compatible clients\n   * - Apply different configurations based on intent\n   *\n   * @param config - Resolved model configuration\n   * @returns OpenAI client instance (original or wrapped)\n   *\n   * @example\n   * ```typescript\n   * createOpenAIClient: async (openai, opts) => {\n   *   // Wrap with langsmith for planning tasks\n   *   if (opts.baseURL?.includes('planning')) {\n   *     return wrapOpenAI(openai, { metadata: { task: 'planning' } });\n   *   }\n   *\n   *   return openai;\n   * }\n   * ```\n   */\n  createOpenAIClient?: CreateOpenAIClientFn;\n}\n\nexport type TestStatus =\n  | 'passed'\n  | 'failed'\n  | 'timedOut'\n  | 'skipped'\n  | 'interrupted';\n\nexport interface ReportFileAttributes {\n  testDuration: number;\n  testStatus: TestStatus;\n  testTitle: string;\n  testId: string;\n  testDescription: string;\n}\n\nexport type ReportFileWithAttributes =\n  | {\n      reportFilePath: string;\n      reportAttributes: ReportFileAttributes;\n    }\n  | {\n      reportFilePath?: string;\n      reportAttributes: ReportFileAttributes & { testStatus: 'skipped' };\n    };\n"],"names":["UIContext","ServiceError","Error","message","dump","replacerForDumpSerialization","_key","value","reviverForDumpDeserialization","key","ScreenshotItem","ExecutionDump","indents","JSON","task","serialized","parsed","data","screenshots","record","ReportActionDump","processValue","obj","Array","entries","Object","exec","execution","basePath","screenshotsDir","existsSync","mkdirSync","screenshot","imagePath","join","rawBase64","writeFileSync","Buffer","dumpString","readFileSync","loadFromExecutionScreenshotDir","id","mimeType","ext","filePath","dumpData","store","ScreenshotStore","processedData","restoreImageReferences","ref","executionFileImage","filesToClean","rmSync","GroupedActionDump"],"mappings":";;;;;;AAAqD;;;;;;;;;;AAgH9C,MAAeA;AA4BtB;AAiFO,MAAMC,qBAAqBC;IAGhC,YAAYC,OAAe,EAAEC,IAAiB,CAAE;QAC9C,KAAK,CAACD,UAHR;QAIE,IAAI,CAAC,IAAI,GAAG;QACZ,IAAI,CAAC,IAAI,GAAGC;IACd;AACF;AAkOA,SAASC,6BAA6BC,IAAY,EAAEC,KAAU;IAC5D,IAAIA,SAASA,MAAM,WAAW,EAAE,SAAS,QACvC,OAAO;IAET,IAAIA,SAASA,MAAM,WAAW,EAAE,SAAS,WACvC,OAAO;IAGT,IAAIA,SAAS,AAAgC,cAAhC,OAAOA,MAAM,cAAc,EACtC,OAAOA,MAAM,cAAc;IAE7B,OAAOA;AACT;AAUA,SAASC,8BAA8BC,GAAW,EAAEF,KAAU;IAE5D,IAAIE,AAAQ,iBAARA,OAAwB,AAAiB,YAAjB,OAAOF,SAAsBA,AAAU,SAAVA,OACvD,OAAOA;IAGLG,eAAe,YAAY,CAACH;IAIhC,OAAOA;AACT;AAKO,MAAMI;IAoBX,UAAUC,OAAgB,EAAU;QAClC,OAAOC,KAAK,SAAS,CAAC,IAAI,CAAC,MAAM,IAAIR,8BAA8BO;IACrE;IAKA,SAAyB;QACvB,OAAO;YACL,IAAI,IAAI,CAAC,EAAE;YACX,SAAS,IAAI,CAAC,OAAO;YACrB,MAAM,IAAI,CAAC,IAAI;YACf,aAAa,IAAI,CAAC,WAAW;YAC7B,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAACE,OAAU;oBAC/B,GAAGA,IAAI;oBACP,UAAUA,KAAK,QAAQ,IAAI,EAAE;gBAC/B;YACA,cAAc,IAAI,CAAC,YAAY;QACjC;IACF;IAKA,OAAO,qBAAqBC,UAAkB,EAAiB;QAC7D,MAAMC,SAASH,KAAK,KAAK,CACvBE,YACAP;QAEF,OAAO,IAAIG,cAAcK;IAC3B;IAKA,OAAO,SAASC,IAAoB,EAAiB;QACnD,OAAO,IAAIN,cAAcM;IAC3B;IAQA,qBAAuC;QACrC,MAAMC,cAAgC,EAAE;QAExC,KAAK,MAAMJ,QAAQ,IAAI,CAAC,KAAK,CAAE;YAE7B,IAAIA,KAAK,SAAS,EAAE,sBAAsBJ,gBACxCQ,YAAY,IAAI,CAACJ,KAAK,SAAS,CAAC,UAAU;YAI5C,IAAIA,KAAK,QAAQ,EACf;gBAAA,KAAK,MAAMK,UAAUL,KAAK,QAAQ,CAChC,IAAIK,OAAO,UAAU,YAAYT,gBAC/BQ,YAAY,IAAI,CAACC,OAAO,UAAU;YAEtC;QAEJ;QAEA,OAAOD;IACT;IA7EA,YAAYD,IAAoB,CAAE;QAPlC;QACA;QACA;QACA;QACA;QACA;QAGE,IAAI,CAAC,EAAE,GAAGA,KAAK,EAAE;QACjB,IAAI,CAAC,OAAO,GAAGA,KAAK,OAAO;QAC3B,IAAI,CAAC,IAAI,GAAGA,KAAK,IAAI;QACrB,IAAI,CAAC,WAAW,GAAGA,KAAK,WAAW;QACnC,IAAI,CAAC,KAAK,GAAGA,KAAK,KAAK;QACvB,IAAI,CAAC,YAAY,GAAGA,KAAK,YAAY;IACvC;AAuEF;AA2KO,MAAMG;IAuBX,UAAUR,OAAgB,EAAU;QAClC,OAAOC,KAAK,SAAS,CAAC,IAAI,CAAC,MAAM,IAAIR,8BAA8BO;IACrE;IAMA,+BAA+BA,OAAgB,EAAU;QACvD,MAAMS,eAAe,CAACC;YACpB,IAAIA,eAAeZ,gBACjB,OAAO;gBAAE,QAAQY,IAAI,MAAM;gBAAE,YAAYA,IAAI,UAAU;YAAC;YAE1D,IAAIC,MAAM,OAAO,CAACD,MAChB,OAAOA,IAAI,GAAG,CAACD;YAEjB,IAAIC,OAAO,AAAe,YAAf,OAAOA,KAAkB;gBAClC,MAAME,UAAUC,OAAO,OAAO,CAACH,KAAK,GAAG,CAAC,CAAC,CAACb,KAAKF,MAAM,GAAK;wBACxDE;wBACAY,aAAad;qBACd;gBACD,OAAOkB,OAAO,WAAW,CAACD;YAC5B;YACA,OAAOF;QACT;QAEA,MAAML,OAAOI,aAAa,IAAI,CAAC,MAAM;QACrC,OAAOR,KAAK,SAAS,CAACI,MAAM,MAAML;IACpC;IAKA,SAA4B;QAC1B,OAAO;YACL,YAAY,IAAI,CAAC,UAAU;YAC3B,WAAW,IAAI,CAAC,SAAS;YACzB,kBAAkB,IAAI,CAAC,gBAAgB;YACvC,aAAa,IAAI,CAAC,WAAW;YAC7B,YAAY,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAACc,OAASA,KAAK,MAAM;YACrD,YAAY,IAAI,CAAC,UAAU;QAC7B;IACF;IAKA,OAAO,qBAAqBX,UAAkB,EAAoB;QAChE,MAAMC,SAASH,KAAK,KAAK,CACvBE,YACAP;QAEF,OAAO,IAAIY,iBAAiBJ;IAC9B;IAKA,OAAO,SAASC,IAAuB,EAAoB;QACzD,OAAO,IAAIG,iBAAiBH;IAC9B;IAOA,wBAA0C;QACxC,MAAMC,cAAgC,EAAE;QACxC,KAAK,MAAMS,aAAa,IAAI,CAAC,UAAU,CACrCT,YAAY,IAAI,IAAIS,UAAU,kBAAkB;QAElD,OAAOT;IACT;IAUA,iBAAiBU,QAAgB,EAAQ;QACvC,MAAMC,iBAAiB,GAAGD,SAAS,YAAY,CAAC;QAChD,IAAI,CAACE,WAAWD,iBACdE,UAAUF,gBAAgB;YAAE,WAAW;QAAK;QAG9C,MAAMX,cAAc,IAAI,CAAC,qBAAqB;QAE9C,KAAK,MAAMc,cAAcd,YAAa;YACpC,MAAMe,YAAYC,KAChBL,gBACA,GAAGG,WAAW,EAAE,CAAC,CAAC,EAAEA,WAAW,SAAS,EAAE;YAE5C,IAAIF,WAAWG,YACb;YAGF,MAAME,YAAYH,WAAW,SAAS;YACtCI,cAAcH,WAAWI,OAAO,IAAI,CAACF,WAAW;QAClD;QAGAC,cAAcR,UAAU,IAAI,CAAC,SAAS,IAAI;IAC5C;IASA,OAAO,sBAAsBA,QAAgB,EAAU;QACrD,MAAMU,aAAaC,aAAaX,UAAU;QAC1C,MAAMC,iBAAiB,GAAGD,SAAS,YAAY,CAAC;QAEhD,MAAMY,iCAAiC,CAACC,IAAYC;YAClD,MAAMC,MAAMD,AAAa,iBAAbA,WAA4B,SAAS;YACjD,MAAME,WAAWV,KAAKL,gBAAgB,GAAGY,GAAG,CAAC,EAAEE,KAAK;YACpD,IAAI,CAACb,WAAWc,WACd,OAAO;YAET,MAAM3B,OAAOsB,aAAaK;YAC1B,OAAO,CAAC,WAAW,EAAED,IAAI,QAAQ,EAAE1B,KAAK,QAAQ,CAAC,WAAW;QAC9D;QAGA,MAAM4B,WAAWhC,KAAK,KAAK,CAACyB;QAC5B,MAAMQ,QAAQ,IAAIC,gBAAgB;YAChC,MAAM;YACN,YAAYnB;QACd;QACA,MAAMoB,gBAAgBC,uBAAuBJ,UAAU,CAACK;YACtD,MAAMC,qBAAqBX,+BACzBU,IAAI,EAAE,EACNA,IAAI,QAAQ;YAEd,IAAIC,oBACF,OAAOA;YAGT,IAAID,AAAgB,aAAhBA,IAAI,OAAO,EACb,OAAO;YAET,OAAOJ,MAAM,UAAU,CAACI;QAC1B;QACA,OAAOrC,KAAK,SAAS,CAACmC;IACxB;IAOA,OAAO,aAAapB,QAAgB,EAAQ;QAC1C,MAAMwB,eAAe;YAACxB;YAAU,GAAGA,SAAS,YAAY,CAAC;SAAC;QAE1D,KAAK,MAAMgB,YAAYQ,aACrB,IAAI;YACFC,OAAOT,UAAU;gBAAE,OAAO;gBAAM,WAAW;YAAK;QAClD,EAAE,OAAM,CAER;IAEJ;IAQA,OAAO,aAAahB,QAAgB,EAAY;QAC9C,OAAO;YAACA;YAAU,GAAGA,SAAS,YAAY,CAAC;SAAC;IAC9C;IAhMA,YAAYX,IAAuB,CAAE;QAPrC;QACA;QACA;QACA;QACA;QACA;QAGE,IAAI,CAAC,UAAU,GAAGA,KAAK,UAAU;QACjC,IAAI,CAAC,SAAS,GAAGA,KAAK,SAAS;QAC/B,IAAI,CAAC,gBAAgB,GAAGA,KAAK,gBAAgB;QAC7C,IAAI,CAAC,WAAW,GAAGA,KAAK,WAAW;QACnC,IAAI,CAAC,UAAU,GAAGA,KAAK,UAAU,CAAC,GAAG,CAAC,CAACS,OACrCA,gBAAgBf,gBAAgBe,OAAOf,cAAc,QAAQ,CAACe;QAEhE,IAAI,CAAC,UAAU,GAAGT,KAAK,UAAU;IACnC;AAwLF;AAIO,MAAMqC,oBAAoBlC"}
+{"version":3,"file":"types.mjs","sources":["../../src/types.ts"],"sourcesContent":["/* eslint-disable @typescript-eslint/no-explicit-any */\n\nimport {\n  existsSync,\n  mkdirSync,\n  readFileSync,\n  rmSync,\n  writeFileSync,\n} from 'node:fs';\nimport { join } from 'node:path';\nimport type { NodeType } from '@aiscene/shared/constants';\nimport type { CreateOpenAIClientFn, TModelConfig } from '@aiscene/shared/env';\nimport type {\n  BaseElement,\n  LocateResultElement,\n  Rect,\n  Size,\n} from '@aiscene/shared/types';\nimport type { z } from 'zod';\nimport type { TUserPrompt } from './common';\nimport { restoreImageReferences } from './dump/screenshot-restoration';\nimport { ScreenshotStore } from './dump/screenshot-store';\nimport { ScreenshotItem } from './screenshot-item';\nimport type {\n  DetailedLocateParam,\n  MidsceneYamlFlowItem,\n  ServiceExtractOption,\n} from './yaml';\n\nexport type {\n  ElementTreeNode,\n  BaseElement,\n  Rect,\n  Size,\n  Point,\n} from '@aiscene/shared/types';\nexport * from './yaml';\n\nexport type AIUsageInfo = Record<string, any> & {\n  prompt_tokens: number | undefined;\n  completion_tokens: number | undefined;\n  total_tokens: number | undefined;\n  cached_input: number | undefined;\n  time_cost: number | undefined;\n  model_name: string | undefined;\n  model_description: string | undefined;\n  intent: string | undefined;\n  request_id: string | undefined;\n};\n\nexport type { LocateResultElement };\n\nexport type AISingleElementResponseByPosition = {\n  position?: {\n    x: number;\n    y: number;\n  };\n  bbox?: [number, number, number, number];\n  reason: string;\n  text: string;\n};\n\nexport interface AIElementCoordinatesResponse {\n  bbox: [number, number, number, number];\n  errors?: string[];\n}\n\nexport type AIElementResponse = AIElementCoordinatesResponse;\n\nexport interface AIDataExtractionResponse<DataDemand> {\n  data: DataDemand;\n  errors?: string[];\n  thought?: string;\n}\n\nexport interface AISectionLocatorResponse {\n  bbox: [number, number, number, number];\n  references_bbox?: [number, number, number, number][];\n  error?: string;\n}\n\nexport interface AIAssertionResponse {\n  pass: boolean;\n  thought: string;\n}\n\nexport interface AIDescribeElementResponse {\n  description: string;\n  error?: string;\n}\n\nexport interface LocatorValidatorOption {\n  centerDistanceThreshold?: number;\n}\n\nexport interface LocateValidatorResult {\n  pass: boolean;\n  rect: Rect;\n  center: [number, number];\n  centerDistance?: number;\n}\n\nexport interface AgentDescribeElementAtPointResult {\n  prompt: string;\n  deepLocate: boolean;\n  verifyResult?: LocateValidatorResult;\n}\n\n/**\n * context\n */\n\nexport abstract class UIContext {\n  /**\n   * screenshot of the current UI state. which size is shotSize(be shrunk by screenshotShrinkFactor),\n   */\n  abstract screenshot: ScreenshotItem;\n\n  /**\n   * screenshot size after shrinking\n   */\n  abstract shotSize: Size;\n\n  /**\n   * The ratio for converting shrunk screenshot coordinates to logical coordinates.\n   *\n   * Example:\n   * - Physical screen width: 3000px, dpr=6\n   * - Logical width: 500px\n   * - User-defined screenshotShrinkFactor: 2\n   * - Actual shrunk screenshot width: 3000 / 2 = 1500px\n   * - shrunkShotToLogicalRatio: dpr / screenshotShrinkFactor = 6 / 2 = 3\n   * - To map back to logical coordinates: 1500 / shrunkShotToLogicalRatio = 500px\n   */\n  abstract shrunkShotToLogicalRatio: number;\n\n  abstract _isFrozen?: boolean;\n\n  // @deprecated - backward compatibility for aiLocate\n  abstract deprecatedDpr?: number;\n}\n\nexport type EnsureObject<T> = { [K in keyof T]: any };\n\nexport type ServiceAction = 'locate' | 'extract' | 'assert' | 'describe';\n\nexport type ServiceExtractParam = string | Record<string, string>;\n\nexport type ElementCacheFeature = Record<string, unknown>;\n\nexport interface LocateResult {\n  element: LocateResultElement | null;\n  rect?: Rect;\n}\n\nexport type ThinkingLevel = 'off' | 'medium' | 'high';\n\nexport type DeepThinkOption = 'unset' | true | false;\n\nexport interface ServiceTaskInfo {\n  durationMs: number;\n  formatResponse?: string;\n  rawResponse?: string;\n  usage?: AIUsageInfo;\n  searchArea?: Rect;\n  searchAreaRawResponse?: string;\n  searchAreaUsage?: AIUsageInfo;\n  reasoning_content?: string;\n}\n\nexport interface DumpMeta {\n  logTime: number;\n}\n\nexport type ReportAttributes = Record<\n  string,\n  string | number | boolean | null | undefined\n>;\n\nexport interface ReportDumpWithAttributes {\n  dumpString: string;\n  attributes?: ReportAttributes;\n}\n\nexport interface ServiceDump extends DumpMeta {\n  type: 'locate' | 'extract' | 'assert';\n  logId: string;\n  userQuery: {\n    element?: TUserPrompt;\n    dataDemand?: ServiceExtractParam;\n    assertion?: TUserPrompt;\n  };\n  matchedElement: LocateResultElement[];\n  matchedRect?: Rect;\n  deepLocate?: boolean;\n  data: any;\n  assertionPass?: boolean;\n  assertionThought?: string;\n  taskInfo: ServiceTaskInfo;\n  error?: string;\n  output?: any;\n}\n\nexport type PartialServiceDumpFromSDK = Omit<\n  ServiceDump,\n  'logTime' | 'logId' | 'model_name'\n>;\n\nexport interface ServiceResultBase {\n  dump: ServiceDump;\n}\n\nexport type LocateResultWithDump = LocateResult & ServiceResultBase;\n\nexport interface ServiceExtractResult<T> extends ServiceResultBase {\n  data: T;\n  thought?: string;\n  usage?: AIUsageInfo;\n  reasoning_content?: string;\n}\n\nexport class ServiceError extends Error {\n  dump: ServiceDump;\n\n  constructor(message: string, dump: ServiceDump) {\n    super(message);\n    this.name = 'ServiceError';\n    this.dump = dump;\n  }\n}\n\n// intermediate variables to optimize the return value by AI\nexport interface LiteUISection {\n  name: string;\n  description: string;\n  sectionCharacteristics: string;\n  textIds: string[];\n}\n\nexport type ElementById = (id: string) => BaseElement | null;\n\nexport type ServiceAssertionResponse = AIAssertionResponse & {\n  usage?: AIUsageInfo;\n};\n\n/**\n * agent\n */\n\nexport type OnTaskStartTip = (tip: string) => Promise<void> | void;\n\nexport interface AgentWaitForOpt extends ServiceExtractOption {\n  checkIntervalMs?: number;\n  timeoutMs?: number;\n}\n\nexport interface AgentAssertOpt {\n  keepRawResponse?: boolean;\n}\n\n/**\n * planning\n *\n */\n\nexport interface PlanningLocateParam extends DetailedLocateParam {\n  bbox?: [number, number, number, number];\n}\n\nexport interface PlanningAction<ParamType = any> {\n  thought?: string;\n  log?: string; // a brief preamble to the user explaining what you’re about to do\n  type: string;\n  param: ParamType;\n}\n\nexport type SubGoalStatus = 'pending' | 'running' | 'finished';\n\nexport interface SubGoal {\n  index: number;\n  status: SubGoalStatus;\n  description: string;\n  logs?: string[];\n}\n\nexport interface RawResponsePlanningAIResponse {\n  action: PlanningAction;\n  thought?: string;\n  log: string;\n  memory?: string;\n  error?: string;\n  finalizeMessage?: string;\n  finalizeSuccess?: boolean;\n  updateSubGoals?: SubGoal[];\n  markFinishedIndexes?: number[];\n}\n\nexport interface PlanningAIResponse\n  extends Omit<RawResponsePlanningAIResponse, 'action'> {\n  actions?: PlanningAction[];\n  usage?: AIUsageInfo;\n  rawResponse?: string;\n  yamlFlow?: MidsceneYamlFlowItem[];\n  yamlString?: string;\n  error?: string;\n  reasoning_content?: string;\n  shouldContinuePlanning: boolean;\n  output?: string; // Output message from <complete> tag (same as finalizeMessage)\n}\n\nexport interface PlanningActionParamSleep {\n  timeMs: number;\n}\n\nexport interface PlanningActionParamError {\n  thought: string;\n}\n\nexport type PlanningActionParamWaitFor = AgentWaitForOpt & {};\n\nexport interface LongPressParam {\n  duration?: number;\n}\n\nexport interface PullParam {\n  direction: 'up' | 'down';\n  distance?: number;\n  duration?: number;\n}\n/**\n * misc\n */\n\nexport interface Color {\n  name: string;\n  hex: string;\n}\n\nexport interface BaseAgentParserOpt {\n  selector?: string;\n}\n// eslint-disable-next-line @typescript-eslint/no-empty-interface\nexport interface PuppeteerParserOpt extends BaseAgentParserOpt {}\n\n// eslint-disable-next-line @typescript-eslint/no-empty-interface\nexport interface PlaywrightParserOpt extends BaseAgentParserOpt {}\n\n/*\naction\n*/\nexport interface ExecutionTaskProgressOptions {\n  onTaskStart?: (task: ExecutionTask) => Promise<void> | void;\n}\n\nexport interface ExecutionRecorderItem {\n  type: 'screenshot';\n  ts: number;\n  screenshot?: ScreenshotItem;\n  timing?: string;\n}\n\nexport type ExecutionTaskType = 'Planning' | 'Insight' | 'Action Space' | 'Log';\n\nexport interface ExecutorContext {\n  task: ExecutionTask;\n  element?: LocateResultElement | null;\n  uiContext?: UIContext;\n}\n\nexport interface ExecutionTaskApply<\n  Type extends ExecutionTaskType = any,\n  TaskParam = any,\n  TaskOutput = any,\n  TaskLog = any,\n> {\n  type: Type;\n  subType?: string;\n  param?: TaskParam;\n  thought?: string;\n  uiContext?: UIContext;\n  executor: (\n    param: TaskParam,\n    context: ExecutorContext,\n  ) => // biome-ignore lint/suspicious/noConfusingVoidType: void is intentionally allowed as some executors may not return a value\n    | Promise<ExecutionTaskReturn<TaskOutput, TaskLog> | undefined | void>\n    | undefined\n    | void;\n}\n\nexport interface ExecutionTaskHitBy {\n  from: string;\n  context: Record<string, any>;\n}\n\nexport interface ExecutionTaskReturn<TaskOutput = unknown, TaskLog = unknown> {\n  output?: TaskOutput;\n  log?: TaskLog;\n  recorder?: ExecutionRecorderItem[];\n  hitBy?: ExecutionTaskHitBy;\n}\n\nexport type ExecutionTask<\n  E extends ExecutionTaskApply<any, any, any> = ExecutionTaskApply<\n    any,\n    any,\n    any\n  >,\n> = E &\n  ExecutionTaskReturn<\n    E extends ExecutionTaskApply<any, any, infer TaskOutput, any>\n      ? TaskOutput\n      : unknown,\n    E extends ExecutionTaskApply<any, any, any, infer TaskLog>\n      ? TaskLog\n      : unknown\n  > & {\n    taskId: string;\n    status: 'pending' | 'running' | 'finished' | 'failed' | 'cancelled';\n    error?: Error;\n    errorMessage?: string;\n    errorStack?: string;\n    timing?: {\n      start: number;\n      getUiContextStart?: number;\n      getUiContextEnd?: number;\n      callAiStart?: number;\n      callAiEnd?: number;\n      beforeInvokeActionHookStart?: number;\n      beforeInvokeActionHookEnd?: number;\n      callActionStart?: number;\n      callActionEnd?: number;\n      afterInvokeActionHookStart?: number;\n      afterInvokeActionHookEnd?: number;\n      captureAfterCallingSnapshotStart?: number;\n      captureAfterCallingSnapshotEnd?: number;\n      end?: number;\n      cost?: number;\n    };\n    usage?: AIUsageInfo;\n    searchAreaUsage?: AIUsageInfo;\n    reasoning_content?: string;\n  };\n\nexport interface IExecutionDump extends DumpMeta {\n  /** Stable unique identifier for this execution run */\n  id?: string;\n  name: string;\n  description?: string;\n  tasks: ExecutionTask[];\n  aiActContext?: string;\n}\n\n/**\n * Replacer function for JSON serialization that handles Page, Browser objects and ScreenshotItem\n */\nfunction replacerForDumpSerialization(_key: string, value: any): any {\n  if (value && value.constructor?.name === 'Page') {\n    return '[Page object]';\n  }\n  if (value && value.constructor?.name === 'Browser') {\n    return '[Browser object]';\n  }\n  // Handle ScreenshotItem serialization\n  if (value && typeof value.toSerializable === 'function') {\n    return value.toSerializable();\n  }\n  return value;\n}\n\n/**\n * Reviver function for JSON deserialization that keeps screenshot references\n * as plain objects. Resolution is handled lazily by restoreImageReferences.\n *\n * @param key - JSON key being processed\n * @param value - JSON value being processed\n * @returns Restored value\n */\nfunction reviverForDumpDeserialization(key: string, value: any): any {\n  // Only process screenshot fields\n  if (key !== 'screenshot' || typeof value !== 'object' || value === null) {\n    return value;\n  }\n\n  if (ScreenshotItem.isSerialized(value)) {\n    return value;\n  }\n\n  return value;\n}\n\n/**\n * ExecutionDump class for serializing and deserializing execution dumps\n */\nexport class ExecutionDump implements IExecutionDump {\n  id?: string;\n  logTime: number;\n  name: string;\n  description?: string;\n  tasks: ExecutionTask[];\n  aiActContext?: string;\n\n  constructor(data: IExecutionDump) {\n    this.id = data.id;\n    this.logTime = data.logTime;\n    this.name = data.name;\n    this.description = data.description;\n    this.tasks = data.tasks;\n    this.aiActContext = data.aiActContext;\n  }\n\n  /**\n   * Serialize the ExecutionDump to a JSON string\n   */\n  serialize(indents?: number): string {\n    return JSON.stringify(this.toJSON(), replacerForDumpSerialization, indents);\n  }\n\n  /**\n   * Convert to a plain object for JSON serialization\n   */\n  toJSON(): IExecutionDump {\n    return {\n      id: this.id,\n      logTime: this.logTime,\n      name: this.name,\n      description: this.description,\n      tasks: this.tasks.map((task) => ({\n        ...task,\n        recorder: task.recorder || [],\n      })),\n      aiActContext: this.aiActContext,\n    };\n  }\n\n  /**\n   * Create an ExecutionDump instance from a serialized JSON string\n   */\n  static fromSerializedString(serialized: string): ExecutionDump {\n    const parsed = JSON.parse(\n      serialized,\n      reviverForDumpDeserialization,\n    ) as IExecutionDump;\n    return new ExecutionDump(parsed);\n  }\n\n  /**\n   * Create an ExecutionDump instance from a plain object\n   */\n  static fromJSON(data: IExecutionDump): ExecutionDump {\n    return new ExecutionDump(data);\n  }\n\n  /**\n   * Collect all ScreenshotItem instances from tasks.\n   * Scans through uiContext and recorder items to find screenshots.\n   *\n   * @returns Array of ScreenshotItem instances\n   */\n  collectScreenshots(): ScreenshotItem[] {\n    const screenshots: ScreenshotItem[] = [];\n\n    for (const task of this.tasks) {\n      // Collect uiContext.screenshot if present\n      if (task.uiContext?.screenshot instanceof ScreenshotItem) {\n        screenshots.push(task.uiContext.screenshot);\n      }\n\n      // Collect recorder screenshots\n      if (task.recorder) {\n        for (const record of task.recorder) {\n          if (record.screenshot instanceof ScreenshotItem) {\n            screenshots.push(record.screenshot);\n          }\n        }\n      }\n    }\n\n    return screenshots;\n  }\n}\n\n/*\ntask - service-locate\n*/\nexport type ExecutionTaskInsightLocateParam = PlanningLocateParam;\n\nexport interface ExecutionTaskInsightLocateOutput {\n  element: LocateResultElement | null;\n}\n\nexport type ExecutionTaskInsightDump = ServiceDump;\n\nexport type ExecutionTaskInsightLocateApply = ExecutionTaskApply<\n  'Insight',\n  ExecutionTaskInsightLocateParam,\n  ExecutionTaskInsightLocateOutput,\n  ExecutionTaskInsightDump\n>;\n\nexport type ExecutionTaskInsightLocate =\n  ExecutionTask<ExecutionTaskInsightLocateApply>;\n\n/*\ntask - service-query\n*/\nexport interface ExecutionTaskInsightQueryParam {\n  dataDemand: ServiceExtractParam;\n  domIncluded?: boolean | 'visible-only';\n}\n\nexport interface ExecutionTaskInsightQueryOutput {\n  data: any;\n}\n\nexport type ExecutionTaskInsightQueryApply = ExecutionTaskApply<\n  'Insight',\n  ExecutionTaskInsightQueryParam,\n  any,\n  ExecutionTaskInsightDump\n>;\n\nexport type ExecutionTaskInsightQuery =\n  ExecutionTask<ExecutionTaskInsightQueryApply>;\n\n/*\ntask - assertion\n*/\nexport interface ExecutionTaskInsightAssertionParam {\n  assertion: string;\n}\n\nexport type ExecutionTaskInsightAssertionApply = ExecutionTaskApply<\n  'Insight',\n  ExecutionTaskInsightAssertionParam,\n  ServiceAssertionResponse,\n  ExecutionTaskInsightDump\n>;\n\nexport type ExecutionTaskInsightAssertion =\n  ExecutionTask<ExecutionTaskInsightAssertionApply>;\n\n/*\ntask - action (i.e. interact) \n*/\nexport type ExecutionTaskActionApply<ActionParam = any> = ExecutionTaskApply<\n  'Action Space',\n  ActionParam,\n  void,\n  void\n>;\n\nexport type ExecutionTaskAction = ExecutionTask<ExecutionTaskActionApply>;\n\n/*\ntask - Log\n*/\n\nexport type ExecutionTaskLogApply<\n  LogParam = {\n    content: string;\n  },\n> = ExecutionTaskApply<'Log', LogParam, void, void>;\n\nexport type ExecutionTaskLog = ExecutionTask<ExecutionTaskLogApply>;\n\n/*\ntask - planning\n*/\n\nexport type ExecutionTaskPlanningApply = ExecutionTaskApply<\n  'Planning',\n  {\n    userInstruction: string;\n    aiActContext?: string;\n  },\n  PlanningAIResponse\n>;\n\nexport type ExecutionTaskPlanning = ExecutionTask<ExecutionTaskPlanningApply>;\n\n/*\ntask - planning-locate\n*/\nexport type ExecutionTaskPlanningLocateParam = PlanningLocateParam;\n\nexport interface ExecutionTaskPlanningLocateOutput {\n  element: LocateResultElement | null;\n}\n\nexport type ExecutionTaskPlanningDump = ServiceDump;\n\nexport type ExecutionTaskPlanningLocateApply = ExecutionTaskApply<\n  'Planning',\n  ExecutionTaskPlanningLocateParam,\n  ExecutionTaskPlanningLocateOutput,\n  ExecutionTaskPlanningDump\n>;\n\nexport type ExecutionTaskPlanningLocate =\n  ExecutionTask<ExecutionTaskPlanningLocateApply>;\n\n/*\nReport metadata - extracted from ReportActionDump for per-execution writes\n*/\nexport interface ReportMeta {\n  groupName: string;\n  groupDescription?: string;\n  sdkVersion: string;\n  modelBriefs: ModelBrief[];\n  deviceType?: string;\n}\n\n// Backward-compatible aliases for existing external consumers.\nexport type GroupMeta = ReportMeta;\n\n/*\nReport dump\n*/\nexport interface IReportActionDump {\n  sdkVersion: string;\n  groupName: string;\n  groupDescription?: string;\n  modelBriefs: ModelBrief[];\n  executions: IExecutionDump[];\n  deviceType?: string;\n}\n\n// Backward-compatible aliases for existing external consumers.\nexport type IGroupedActionDump = IReportActionDump;\n\nexport interface ModelBrief {\n  /**\n   * The intent/category of the model call, for example \"planning\" or \"insight\".\n   */\n  intent?: string;\n\n  /**\n   * The model name returned by usage metadata, for example \"gpt-4o\".\n   */\n  name?: string;\n\n  /**\n   * Optional human-readable model description, for example \"qwen2.5-vl mode\".\n   */\n  modelDescription?: string;\n}\n\n/**\n * ReportActionDump class for serializing and deserializing report action dumps\n */\nexport class ReportActionDump implements IReportActionDump {\n  sdkVersion: string;\n  groupName: string;\n  groupDescription?: string;\n  modelBriefs: ModelBrief[];\n  executions: ExecutionDump[];\n  deviceType?: string;\n\n  constructor(data: IReportActionDump) {\n    this.sdkVersion = data.sdkVersion;\n    this.groupName = data.groupName;\n    this.groupDescription = data.groupDescription;\n    this.modelBriefs = data.modelBriefs;\n    this.executions = data.executions.map((exec) =>\n      exec instanceof ExecutionDump ? exec : ExecutionDump.fromJSON(exec),\n    );\n    this.deviceType = data.deviceType;\n  }\n\n  /**\n   * Serialize the ReportActionDump to a JSON string\n   * Uses compact { $screenshot: id } format\n   */\n  serialize(indents?: number): string {\n    return JSON.stringify(this.toJSON(), replacerForDumpSerialization, indents);\n  }\n\n  /**\n   * Serialize the ReportActionDump with inline screenshots to a JSON string.\n   * Each ScreenshotItem is replaced with { base64: \"...\", capturedAt }.\n   */\n  serializeWithInlineScreenshots(indents?: number): string {\n    const processValue = (obj: unknown): unknown => {\n      if (obj instanceof ScreenshotItem) {\n        return { base64: obj.base64, capturedAt: obj.capturedAt };\n      }\n      if (Array.isArray(obj)) {\n        return obj.map(processValue);\n      }\n      if (obj && typeof obj === 'object') {\n        const entries = Object.entries(obj).map(([key, value]) => [\n          key,\n          processValue(value),\n        ]);\n        return Object.fromEntries(entries);\n      }\n      return obj;\n    };\n\n    const data = processValue(this.toJSON());\n    return JSON.stringify(data, null, indents);\n  }\n\n  /**\n   * Convert to a plain object for JSON serialization\n   */\n  toJSON(): IReportActionDump {\n    return {\n      sdkVersion: this.sdkVersion,\n      groupName: this.groupName,\n      groupDescription: this.groupDescription,\n      modelBriefs: this.modelBriefs,\n      executions: this.executions.map((exec) => exec.toJSON()),\n      deviceType: this.deviceType,\n    };\n  }\n\n  /**\n   * Create a ReportActionDump instance from a serialized JSON string\n   */\n  static fromSerializedString(serialized: string): ReportActionDump {\n    const parsed = JSON.parse(\n      serialized,\n      reviverForDumpDeserialization,\n    ) as IReportActionDump;\n    return new ReportActionDump(parsed);\n  }\n\n  /**\n   * Create a ReportActionDump instance from a plain object\n   */\n  static fromJSON(data: IReportActionDump): ReportActionDump {\n    return new ReportActionDump(data);\n  }\n\n  /**\n   * Collect all ScreenshotItem instances from all executions.\n   *\n   * @returns Array of all ScreenshotItem instances across all executions\n   */\n  collectAllScreenshots(): ScreenshotItem[] {\n    const screenshots: ScreenshotItem[] = [];\n    for (const execution of this.executions) {\n      screenshots.push(...execution.collectScreenshots());\n    }\n    return screenshots;\n  }\n\n  /**\n   * Serialize the dump to files with screenshots as separate PNG files.\n   * Creates:\n   * - {basePath} - dump JSON with { $screenshot: id } references\n   * - {basePath}.screenshots/ - PNG files\n   *\n   * @param basePath - Base path for the dump file\n   */\n  serializeToFiles(basePath: string): void {\n    const screenshotsDir = `${basePath}.screenshots`;\n    if (!existsSync(screenshotsDir)) {\n      mkdirSync(screenshotsDir, { recursive: true });\n    }\n\n    const screenshots = this.collectAllScreenshots();\n\n    for (const screenshot of screenshots) {\n      const imagePath = join(\n        screenshotsDir,\n        `${screenshot.id}.${screenshot.extension}`,\n      );\n      if (existsSync(imagePath)) {\n        continue;\n      }\n\n      const rawBase64 = screenshot.rawBase64;\n      writeFileSync(imagePath, Buffer.from(rawBase64, 'base64'));\n    }\n\n    // Write dump JSON with references\n    writeFileSync(basePath, this.serialize(), 'utf-8');\n  }\n\n  /**\n   * Read dump from files and return JSON string with inline screenshots.\n   * Reads the dump JSON and screenshot files, then inlines the base64 data.\n   *\n   * @param basePath - Base path for the dump file\n   * @returns JSON string with inline screenshots ({ base64: \"...\" } format)\n   */\n  static fromFilesAsInlineJson(basePath: string): string {\n    const dumpString = readFileSync(basePath, 'utf-8');\n    const screenshotsDir = `${basePath}.screenshots`;\n\n    const loadFromExecutionScreenshotDir = (id: string, mimeType: string) => {\n      const ext = mimeType === 'image/jpeg' ? 'jpeg' : 'png';\n      const filePath = join(screenshotsDir, `${id}.${ext}`);\n      if (!existsSync(filePath)) {\n        return '';\n      }\n      const data = readFileSync(filePath);\n      return `data:image/${ext};base64,${data.toString('base64')}`;\n    };\n\n    // Restore image references\n    const dumpData = JSON.parse(dumpString);\n    const store = new ScreenshotStore({\n      mode: 'directory',\n      reportPath: basePath,\n    });\n    const processedData = restoreImageReferences(dumpData, (ref) => {\n      const executionFileImage = loadFromExecutionScreenshotDir(\n        ref.id,\n        ref.mimeType,\n      );\n      if (executionFileImage) {\n        return executionFileImage;\n      }\n\n      if (ref.storage === 'inline') {\n        return '';\n      }\n      return store.loadBase64(ref);\n    });\n    return JSON.stringify(processedData);\n  }\n\n  /**\n   * Clean up all files associated with a serialized dump.\n   *\n   * @param basePath - Base path for the dump file\n   */\n  static cleanupFiles(basePath: string): void {\n    const filesToClean = [basePath, `${basePath}.screenshots`];\n\n    for (const filePath of filesToClean) {\n      try {\n        rmSync(filePath, { force: true, recursive: true });\n      } catch {\n        // Ignore errors - file may already be deleted\n      }\n    }\n  }\n\n  /**\n   * Get all file paths associated with a serialized dump.\n   *\n   * @param basePath - Base path for the dump file\n   * @returns Array of all associated file paths\n   */\n  static getFilePaths(basePath: string): string[] {\n    return [basePath, `${basePath}.screenshots`];\n  }\n}\n\n// Backward-compatible aliases for existing external consumers.\nexport type GroupedActionDump = ReportActionDump;\nexport const GroupedActionDump = ReportActionDump;\n\nexport type InterfaceType =\n  | 'puppeteer'\n  | 'playwright'\n  | 'static'\n  | 'chrome-extension-proxy'\n  | 'android'\n  | string;\n\nexport interface StreamingCodeGenerationOptions {\n  /** Whether to enable streaming output */\n  stream?: boolean;\n  /** Callback function to handle streaming chunks */\n  onChunk?: StreamingCallback;\n  /** Callback function to handle streaming completion */\n  onComplete?: (finalCode: string) => void;\n  /** Callback function to handle streaming errors */\n  onError?: (error: Error) => void;\n}\n\nexport type StreamingCallback = (chunk: CodeGenerationChunk) => void;\n\nexport interface CodeGenerationChunk {\n  /** The incremental content chunk */\n  content: string;\n  /** The reasoning content */\n  reasoning_content: string;\n  /** The accumulated content so far */\n  accumulated: string;\n  /** Whether this is the final chunk */\n  isComplete: boolean;\n  /** Token usage information if available */\n  usage?: AIUsageInfo;\n}\n\nexport interface StreamingAIResponse {\n  /** The final accumulated content */\n  content: string;\n  /** Token usage information */\n  usage?: AIUsageInfo;\n  /** Whether the response was streamed */\n  isStreamed: boolean;\n}\n\nexport interface DeviceAction<TParam = any, TReturn = any> {\n  name: string;\n  description?: string;\n  interfaceAlias?: string;\n  paramSchema?: z.ZodType<TParam>;\n  call: (param: TParam, context: ExecutorContext) => Promise<TReturn> | TReturn;\n  delayAfterRunner?: number;\n  /**\n   * An example param object for this action.\n   * Locate fields with { prompt } will automatically get bbox injected when needed.\n   */\n  sample?: { [K in keyof TParam]?: any };\n}\n\n/**\n * Type utilities for extracting types from DeviceAction definitions\n */\n\n/**\n * Extract parameter type from a DeviceAction\n */\nexport type ActionParam<Action extends DeviceAction<any, any>> =\n  Action extends DeviceAction<infer P, any> ? P : never;\n\n/**\n * Extract return type from a DeviceAction\n */\nexport type ActionReturn<Action extends DeviceAction<any, any>> =\n  Action extends DeviceAction<any, infer R> ? R : never;\n\n/**\n * Web-specific types\n */\nexport interface WebElementInfo extends BaseElement {\n  id: string;\n  attributes: {\n    nodeType: NodeType;\n    [key: string]: string;\n  };\n}\n\n/**\n * Agent\n */\n\nexport type CacheConfig = {\n  strategy?: 'read-only' | 'read-write' | 'write-only';\n  id: string;\n};\n\nexport type Cache =\n  | false // No read, no write\n  | true // Will throw error at runtime - deprecated\n  | CacheConfig; // Object configuration (requires explicit id)\n\nexport interface AgentOpt {\n  // @deprecated Use `reportFileName` and `cache.id` instead.\n  testId?: string;\n  // @deprecated\n  cacheId?: string; // Keep backward compatibility, but marked as deprecated\n  groupName?: string;\n  groupDescription?: string;\n  /* if auto generate report, default true */\n  generateReport?: boolean;\n  /* if persist per-execution dump files next to the report, default false */\n  persistExecutionDump?: boolean;\n  /* if auto print report msg, default true */\n  autoPrintReportMsg?: boolean;\n\n  /**\n   * Use directory-based report format with separate image files.\n   *\n   * When enabled:\n   * - Screenshots are saved as PNG files in a `screenshots/` subdirectory\n   * - Report is generated as `index.html` with relative image paths\n   * - Reduces memory usage and report file size\n   *\n   * IMPORTANT: 'html-and-external-assets' reports must be served via HTTP server\n   * (e.g., `npx serve ./report-dir`). The file:// protocol will not\n   * work due to browser CORS restrictions.\n   *\n   * @default 'single-html'\n   */\n  outputFormat?: 'single-html' | 'html-and-external-assets';\n\n  onTaskStartTip?: OnTaskStartTip;\n  aiActContext?: string;\n  aiActionContext?: string;\n  /* custom report file name */\n  reportFileName?: string;\n  modelConfig?: TModelConfig;\n  cache?: Cache;\n  /**\n   * Maximum number of replanning cycles for aiAct.\n   * Defaults to 20 (40 for `vlm-ui-tars`) when not provided.\n   * If omitted, the agent will also read `MIDSCENE_REPLANNING_CYCLE_LIMIT` for backward compatibility.\n   */\n  replanningCycleLimit?: number;\n\n  /**\n   * Wait time in milliseconds after each action execution.\n   * This allows the UI to settle and stabilize before the next action.\n   * Defaults to 300ms when not provided.\n   */\n  waitAfterAction?: number;\n\n  /**\n   * When set to true, Midscene will use the target device's time (Android/iOS)\n   * instead of the system time. Useful when the device time differs from the\n   * host machine. Default: false\n   */\n  useDeviceTimestamp?: boolean;\n\n  /**\n   * Custom screenshot shrink factor to reduce AI token usage.\n   * When set, the screenshot will be scaled down by this factor from the physical resolution.\n   *\n   * Example:\n   * - Physical screen width: 3000px, dpr=6\n   * - Logical width: 500px\n   * - screenshotShrinkFactor: 2\n   * - Actual shrunk screenshot width: 3000 / 2 = 1500px\n   * - AI analyzes the 1500px screenshot\n   * - Coordinates are transformed back to logical (500px) before actions execute\n   *\n   * Benefits:\n   * - Reduces token usage for high-resolution screenshots\n   * - Maintains accuracy by scaling coordinates appropriately\n   *\n   * Must be >= 1 (shrinking only, enlarging is not supported).\n   *\n   * @default 1 (no shrinking, uses original physical screenshot)\n   */\n  screenshotShrinkFactor?: number;\n\n  /**\n   * Custom OpenAI client factory function\n   *\n   * If provided, this function will be called to create OpenAI client instances\n   * for each AI call, allowing you to:\n   * - Wrap clients with observability tools (langsmith, langfuse)\n   * - Use custom OpenAI-compatible clients\n   * - Apply different configurations based on intent\n   *\n   * @param config - Resolved model configuration\n   * @returns OpenAI client instance (original or wrapped)\n   *\n   * @example\n   * ```typescript\n   * createOpenAIClient: async (openai, opts) => {\n   *   // Wrap with langsmith for planning tasks\n   *   if (opts.baseURL?.includes('planning')) {\n   *     return wrapOpenAI(openai, { metadata: { task: 'planning' } });\n   *   }\n   *\n   *   return openai;\n   * }\n   * ```\n   */\n  createOpenAIClient?: CreateOpenAIClientFn;\n}\n\nexport type TestStatus =\n  | 'passed'\n  | 'failed'\n  | 'timedOut'\n  | 'skipped'\n  | 'interrupted';\n\nexport interface ReportFileAttributes {\n  testDuration: number;\n  testStatus: TestStatus;\n  testTitle: string;\n  testId: string;\n  testDescription: string;\n}\n\nexport type ReportFileWithAttributes =\n  | {\n      reportFilePath: string;\n      reportAttributes: ReportFileAttributes;\n    }\n  | {\n      reportFilePath?: string;\n      reportAttributes: ReportFileAttributes & { testStatus: 'skipped' };\n    };\n"],"names":["UIContext","ServiceError","Error","message","dump","replacerForDumpSerialization","_key","value","reviverForDumpDeserialization","key","ScreenshotItem","ExecutionDump","indents","JSON","task","serialized","parsed","data","screenshots","record","ReportActionDump","processValue","obj","Array","entries","Object","exec","execution","basePath","screenshotsDir","existsSync","mkdirSync","screenshot","imagePath","join","rawBase64","writeFileSync","Buffer","dumpString","readFileSync","loadFromExecutionScreenshotDir","id","mimeType","ext","filePath","dumpData","store","ScreenshotStore","processedData","restoreImageReferences","ref","executionFileImage","filesToClean","rmSync","GroupedActionDump"],"mappings":";;;;;;AAAqD;;;;;;;;;;AAgH9C,MAAeA;AA4BtB;AAiFO,MAAMC,qBAAqBC;IAGhC,YAAYC,OAAe,EAAEC,IAAiB,CAAE;QAC9C,KAAK,CAACD,UAHR;QAIE,IAAI,CAAC,IAAI,GAAG;QACZ,IAAI,CAAC,IAAI,GAAGC;IACd;AACF;AAkOA,SAASC,6BAA6BC,IAAY,EAAEC,KAAU;IAC5D,IAAIA,SAASA,MAAM,WAAW,EAAE,SAAS,QACvC,OAAO;IAET,IAAIA,SAASA,MAAM,WAAW,EAAE,SAAS,WACvC,OAAO;IAGT,IAAIA,SAAS,AAAgC,cAAhC,OAAOA,MAAM,cAAc,EACtC,OAAOA,MAAM,cAAc;IAE7B,OAAOA;AACT;AAUA,SAASC,8BAA8BC,GAAW,EAAEF,KAAU;IAE5D,IAAIE,AAAQ,iBAARA,OAAwB,AAAiB,YAAjB,OAAOF,SAAsBA,AAAU,SAAVA,OACvD,OAAOA;IAGLG,eAAe,YAAY,CAACH;IAIhC,OAAOA;AACT;AAKO,MAAMI;IAoBX,UAAUC,OAAgB,EAAU;QAClC,OAAOC,KAAK,SAAS,CAAC,IAAI,CAAC,MAAM,IAAIR,8BAA8BO;IACrE;IAKA,SAAyB;QACvB,OAAO;YACL,IAAI,IAAI,CAAC,EAAE;YACX,SAAS,IAAI,CAAC,OAAO;YACrB,MAAM,IAAI,CAAC,IAAI;YACf,aAAa,IAAI,CAAC,WAAW;YAC7B,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAACE,OAAU;oBAC/B,GAAGA,IAAI;oBACP,UAAUA,KAAK,QAAQ,IAAI,EAAE;gBAC/B;YACA,cAAc,IAAI,CAAC,YAAY;QACjC;IACF;IAKA,OAAO,qBAAqBC,UAAkB,EAAiB;QAC7D,MAAMC,SAASH,KAAK,KAAK,CACvBE,YACAP;QAEF,OAAO,IAAIG,cAAcK;IAC3B;IAKA,OAAO,SAASC,IAAoB,EAAiB;QACnD,OAAO,IAAIN,cAAcM;IAC3B;IAQA,qBAAuC;QACrC,MAAMC,cAAgC,EAAE;QAExC,KAAK,MAAMJ,QAAQ,IAAI,CAAC,KAAK,CAAE;YAE7B,IAAIA,KAAK,SAAS,EAAE,sBAAsBJ,gBACxCQ,YAAY,IAAI,CAACJ,KAAK,SAAS,CAAC,UAAU;YAI5C,IAAIA,KAAK,QAAQ,EACf;gBAAA,KAAK,MAAMK,UAAUL,KAAK,QAAQ,CAChC,IAAIK,OAAO,UAAU,YAAYT,gBAC/BQ,YAAY,IAAI,CAACC,OAAO,UAAU;YAEtC;QAEJ;QAEA,OAAOD;IACT;IA7EA,YAAYD,IAAoB,CAAE;QAPlC;QACA;QACA;QACA;QACA;QACA;QAGE,IAAI,CAAC,EAAE,GAAGA,KAAK,EAAE;QACjB,IAAI,CAAC,OAAO,GAAGA,KAAK,OAAO;QAC3B,IAAI,CAAC,IAAI,GAAGA,KAAK,IAAI;QACrB,IAAI,CAAC,WAAW,GAAGA,KAAK,WAAW;QACnC,IAAI,CAAC,KAAK,GAAGA,KAAK,KAAK;QACvB,IAAI,CAAC,YAAY,GAAGA,KAAK,YAAY;IACvC;AAuEF;AA2KO,MAAMG;IAuBX,UAAUR,OAAgB,EAAU;QAClC,OAAOC,KAAK,SAAS,CAAC,IAAI,CAAC,MAAM,IAAIR,8BAA8BO;IACrE;IAMA,+BAA+BA,OAAgB,EAAU;QACvD,MAAMS,eAAe,CAACC;YACpB,IAAIA,eAAeZ,gBACjB,OAAO;gBAAE,QAAQY,IAAI,MAAM;gBAAE,YAAYA,IAAI,UAAU;YAAC;YAE1D,IAAIC,MAAM,OAAO,CAACD,MAChB,OAAOA,IAAI,GAAG,CAACD;YAEjB,IAAIC,OAAO,AAAe,YAAf,OAAOA,KAAkB;gBAClC,MAAME,UAAUC,OAAO,OAAO,CAACH,KAAK,GAAG,CAAC,CAAC,CAACb,KAAKF,MAAM,GAAK;wBACxDE;wBACAY,aAAad;qBACd;gBACD,OAAOkB,OAAO,WAAW,CAACD;YAC5B;YACA,OAAOF;QACT;QAEA,MAAML,OAAOI,aAAa,IAAI,CAAC,MAAM;QACrC,OAAOR,KAAK,SAAS,CAACI,MAAM,MAAML;IACpC;IAKA,SAA4B;QAC1B,OAAO;YACL,YAAY,IAAI,CAAC,UAAU;YAC3B,WAAW,IAAI,CAAC,SAAS;YACzB,kBAAkB,IAAI,CAAC,gBAAgB;YACvC,aAAa,IAAI,CAAC,WAAW;YAC7B,YAAY,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAACc,OAASA,KAAK,MAAM;YACrD,YAAY,IAAI,CAAC,UAAU;QAC7B;IACF;IAKA,OAAO,qBAAqBX,UAAkB,EAAoB;QAChE,MAAMC,SAASH,KAAK,KAAK,CACvBE,YACAP;QAEF,OAAO,IAAIY,iBAAiBJ;IAC9B;IAKA,OAAO,SAASC,IAAuB,EAAoB;QACzD,OAAO,IAAIG,iBAAiBH;IAC9B;IAOA,wBAA0C;QACxC,MAAMC,cAAgC,EAAE;QACxC,KAAK,MAAMS,aAAa,IAAI,CAAC,UAAU,CACrCT,YAAY,IAAI,IAAIS,UAAU,kBAAkB;QAElD,OAAOT;IACT;IAUA,iBAAiBU,QAAgB,EAAQ;QACvC,MAAMC,iBAAiB,GAAGD,SAAS,YAAY,CAAC;QAChD,IAAI,CAACE,WAAWD,iBACdE,UAAUF,gBAAgB;YAAE,WAAW;QAAK;QAG9C,MAAMX,cAAc,IAAI,CAAC,qBAAqB;QAE9C,KAAK,MAAMc,cAAcd,YAAa;YACpC,MAAMe,YAAYC,KAChBL,gBACA,GAAGG,WAAW,EAAE,CAAC,CAAC,EAAEA,WAAW,SAAS,EAAE;YAE5C,IAAIF,WAAWG,YACb;YAGF,MAAME,YAAYH,WAAW,SAAS;YACtCI,cAAcH,WAAWI,OAAO,IAAI,CAACF,WAAW;QAClD;QAGAC,cAAcR,UAAU,IAAI,CAAC,SAAS,IAAI;IAC5C;IASA,OAAO,sBAAsBA,QAAgB,EAAU;QACrD,MAAMU,aAAaC,aAAaX,UAAU;QAC1C,MAAMC,iBAAiB,GAAGD,SAAS,YAAY,CAAC;QAEhD,MAAMY,iCAAiC,CAACC,IAAYC;YAClD,MAAMC,MAAMD,AAAa,iBAAbA,WAA4B,SAAS;YACjD,MAAME,WAAWV,KAAKL,gBAAgB,GAAGY,GAAG,CAAC,EAAEE,KAAK;YACpD,IAAI,CAACb,WAAWc,WACd,OAAO;YAET,MAAM3B,OAAOsB,aAAaK;YAC1B,OAAO,CAAC,WAAW,EAAED,IAAI,QAAQ,EAAE1B,KAAK,QAAQ,CAAC,WAAW;QAC9D;QAGA,MAAM4B,WAAWhC,KAAK,KAAK,CAACyB;QAC5B,MAAMQ,QAAQ,IAAIC,gBAAgB;YAChC,MAAM;YACN,YAAYnB;QACd;QACA,MAAMoB,gBAAgBC,uBAAuBJ,UAAU,CAACK;YACtD,MAAMC,qBAAqBX,+BACzBU,IAAI,EAAE,EACNA,IAAI,QAAQ;YAEd,IAAIC,oBACF,OAAOA;YAGT,IAAID,AAAgB,aAAhBA,IAAI,OAAO,EACb,OAAO;YAET,OAAOJ,MAAM,UAAU,CAACI;QAC1B;QACA,OAAOrC,KAAK,SAAS,CAACmC;IACxB;IAOA,OAAO,aAAapB,QAAgB,EAAQ;QAC1C,MAAMwB,eAAe;YAACxB;YAAU,GAAGA,SAAS,YAAY,CAAC;SAAC;QAE1D,KAAK,MAAMgB,YAAYQ,aACrB,IAAI;YACFC,OAAOT,UAAU;gBAAE,OAAO;gBAAM,WAAW;YAAK;QAClD,EAAE,OAAM,CAER;IAEJ;IAQA,OAAO,aAAahB,QAAgB,EAAY;QAC9C,OAAO;YAACA;YAAU,GAAGA,SAAS,YAAY,CAAC;SAAC;IAC9C;IAhMA,YAAYX,IAAuB,CAAE;QAPrC;QACA;QACA;QACA;QACA;QACA;QAGE,IAAI,CAAC,UAAU,GAAGA,KAAK,UAAU;QACjC,IAAI,CAAC,SAAS,GAAGA,KAAK,SAAS;QAC/B,IAAI,CAAC,gBAAgB,GAAGA,KAAK,gBAAgB;QAC7C,IAAI,CAAC,WAAW,GAAGA,KAAK,WAAW;QACnC,IAAI,CAAC,UAAU,GAAGA,KAAK,UAAU,CAAC,GAAG,CAAC,CAACS,OACrCA,gBAAgBf,gBAAgBe,OAAOf,cAAc,QAAQ,CAACe;QAEhE,IAAI,CAAC,UAAU,GAAGT,KAAK,UAAU;IACnC;AAwLF;AAIO,MAAMqC,oBAAoBlC"}

package/dist/es/utils.mjs

@@ -3,10 +3,10 @@ import { appendFileSync, closeSync, existsSync, mkdirSync, openSync, readFileSyn
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { promisify } from "node:util";
-import { defaultRunDirName, getMidsceneRunSubDir } from "@midscene/shared/common";
-import { MIDSCENE_CACHE, MIDSCENE_DEBUG_MODE, globalConfigManager } from "@midscene/shared/env";
-import { getRunningPkgInfo } from "@midscene/shared/node";
-import { assert, escapeScriptTag, ifInBrowser, ifInWorker, logMsg, uuid } from "@midscene/shared/utils";
+import { defaultRunDirName, getMidsceneRunSubDir } from "@aiscene/shared/common";
+import { MIDSCENE_CACHE, MIDSCENE_DEBUG_MODE, globalConfigManager } from "@aiscene/shared/env";
+import { getRunningPkgInfo } from "@aiscene/shared/node";
+import { assert, escapeScriptTag, ifInBrowser, ifInWorker, logMsg, uuid } from "@aiscene/shared/utils";
 let logEnvReady = false;
 const groupedActionDumpFileExt = 'web-dump.json';
 function processCacheConfig(cache, cacheId) {
@@ -175,7 +175,7 @@ function stringifyDumpData(data, indents) {
     return JSON.stringify(data, replacerForPageObject, indents);
 }
 function getVersion() {
-    return "8.0.1";
+    return "8.0.3";
 }
 function debugLog(...message) {
     const debugMode = process.env[MIDSCENE_DEBUG_MODE];