@midscene/core 1.16.5 → 1.17.0

package/dist/es/ai-model/shared/model-locate-result/types.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/shared/model-locate-result/types.mjs","sources":["../../../../../src/ai-model/shared/model-locate-result/types.ts"],"sourcesContent":["import type { Rect } from '@midscene/shared/types';\nimport type { Bbox } from '../../../types';\n\nexport type { Bbox };\nexport type LocateResultBbox = Bbox;\nexport type PixelBbox = Bbox;\nexport type NonEmptyArray<T> = [T, ...T[]];\nexport type RawLocateValue = unknown;\n\nexport type LocateResultPoint = [number, number];\nexport type PointLocateResultCoordinates = ResolvedLocateResultCoordinates & {\n  shape: 'point';\n};\nexport type BboxLocateResultCoordinates = ResolvedLocateResultCoordinates & {\n  shape: 'bbox';\n};\n\nexport type LocateResultValue =\n  | {\n      coordinates: LocateResultPoint;\n      coordinatesMeta: PointLocateResultCoordinates;\n    }\n  | {\n      coordinates: LocateResultBbox;\n      coordinatesMeta: BboxLocateResultCoordinates;\n    };\n\nexport type PointLocateResultValue = Extract<\n  LocateResultValue,\n  { coordinatesMeta: { shape: 'point' } }\n>;\nexport type BboxLocateResultValue = Extract<\n  LocateResultValue,\n  { coordinatesMeta: { shape: 'bbox' } }\n>;\n\nexport function isBboxLocateResultValue(\n  result: LocateResultValue,\n): result is BboxLocateResultValue {\n  return result.coordinatesMeta.shape === 'bbox';\n}\n\nexport function isPointLocateResultValue(\n  result: LocateResultValue,\n): result is PointLocateResultValue {\n  return result.coordinatesMeta.shape === 'point';\n}\n\nexport type LocateResultShape = 'bbox' | 'point';\n\nexport interface LocateResultContext {\n  preparedSize: {\n    width: number;\n    height: number;\n  };\n  contentSize?: {\n    width: number;\n    height: number;\n  };\n}\n\nexport interface LocateResultPromptSpec {\n  resultKey: string;\n  /** Parse-only alternatives, tried in order after resultKey. */\n  resultKeyAliases?: string[];\n  resultValueSchema: string;\n  resultValueDescription: string;\n  resultNoun: string;\n  resultNounPlural: string;\n  exampleValues: NonEmptyArray<unknown>;\n}\n\nexport interface PixelLocateResult {\n  /** Raw bbox midpoint (or raw point), mapped to the locate image using the configured rounding. */\n  center: LocateResultPoint;\n  /**\n   * Original bbox mapped to locate-image pixels and clipped to content bounds.\n   * Undefined for point results; never derive center or click coordinates from it.\n   */\n  rect?: Rect;\n}\n\nexport interface LocateResultCodec {\n  promptSpec: LocateResultPromptSpec;\n  /** Parse once and independently map the center and optional original bbox. */\n  toPixelResult(\n    input: RawLocateValue,\n    ctx: LocateResultContext,\n  ): PixelLocateResult;\n}\n\nexport type CoordinateRounding = 'round' | 'trunc' | 'none';\n\nexport interface LocateResultCoordinates {\n  shape: LocateResultShape;\n  /** Axis order in the raw coordinates; defaults to xy. */\n  order?: 'xy' | 'yx';\n  /** Normalization range, e.g. 1 or 1000. Omit for model-image pixel coordinates. */\n  normalizedBy?: number;\n  /** Rounding after normalized-to-pixel mapping; defaults to round. Pixel inputs retain their precision. */\n  rounding?: CoordinateRounding;\n}\n\n/** Coordinate metadata with defaults resolved and an explicit rounding policy. */\nexport type ResolvedLocateResultCoordinates =\n  | {\n      shape: 'point';\n      order: 'xy' | 'yx';\n      normalizedBy?: number;\n      rounding: CoordinateRounding;\n    }\n  | {\n      shape: 'bbox';\n      order: 'xy' | 'yx';\n      normalizedBy?: number;\n      rounding: CoordinateRounding;\n    };\n\nexport type RawLocateValueParser = (input: RawLocateValue) => LocateResultValue;\n\n/**\n * Format of one model coordinate value, shared by element and search-area locate.\n * The operation protocol extracts target/reference values from the response;\n * the codec parses each value once and maps it to `{ center, rect }` using the\n * parsed result's coordinatesMeta. Rect is undefined for actual point results.\n * Raw-value parsing and normalized-coordinate rounding are configurable;\n * coordinate mapping is shared.\n */\nexport type LocateResultFormatDefinition = {\n  /** Response field used in prompts and parsing; defaults to bbox or point. */\n  resultKey?: string;\n  /** Parse-only alternatives, tried in order when the primary field is undefined. */\n  resultKeyAliases?: string[];\n  /**\n   * Expected shape, axis order, and normalization used by the prompt and default\n   * parser. A custom parser may return different metadata for an actual response.\n   */\n  coordinates: LocateResultCoordinates;\n  /**\n   * Optional model-specific parser for nonstandard formats or point/bbox fallback.\n   * Return `{ coordinates, coordinatesMeta }` describing the actual shape, axis\n   * order, and normalization. Preserve coordinate precision; do not round or map\n   * to pixels here. Include an explicit rounding policy in `coordinatesMeta`.\n   * Omit the parser to parse numeric values according to `coordinates`.\n   */\n  parseRawLocateValue?: RawLocateValueParser;\n};\n"],"names":["isBboxLocateResultValue","result","isPointLocateResultValue"],"mappings":"AAoCO,SAASA,wBACdC,MAAyB;IAEzB,OAAOA,AAAiC,WAAjCA,OAAO,eAAe,CAAC,KAAK;AACrC;AAEO,SAASC,yBACdD,MAAyB;IAEzB,OAAOA,AAAiC,YAAjCA,OAAO,eAAe,CAAC,KAAK;AACrC"}
+{"version":3,"file":"ai-model/shared/model-locate-result/types.mjs","sources":["../../../../../src/ai-model/shared/model-locate-result/types.ts"],"sourcesContent":["import type { Rect } from '@midscene/shared/types';\nimport type { Bbox } from '../../../types';\n\nexport type { Bbox };\nexport type LocateResultBbox = Bbox;\nexport type PixelBbox = Bbox;\nexport type NonEmptyArray<T> = [T, ...T[]];\nexport type RawLocateValue = unknown;\n\nexport type LocateResultPoint = [number, number];\nexport type PointLocateResultCoordinates = ResolvedLocateResultCoordinates & {\n  shape: 'point';\n};\nexport type BboxLocateResultCoordinates = ResolvedLocateResultCoordinates & {\n  shape: 'bbox';\n};\n\nexport type LocateResultValue =\n  | {\n      coordinates: LocateResultPoint;\n      coordinatesMeta: PointLocateResultCoordinates;\n    }\n  | {\n      coordinates: LocateResultBbox;\n      coordinatesMeta: BboxLocateResultCoordinates;\n    };\n\nexport type PointLocateResultValue = Extract<\n  LocateResultValue,\n  { coordinatesMeta: { shape: 'point' } }\n>;\nexport type BboxLocateResultValue = Extract<\n  LocateResultValue,\n  { coordinatesMeta: { shape: 'bbox' } }\n>;\n\nexport function isBboxLocateResultValue(\n  result: LocateResultValue,\n): result is BboxLocateResultValue {\n  return result.coordinatesMeta.shape === 'bbox';\n}\n\nexport function isPointLocateResultValue(\n  result: LocateResultValue,\n): result is PointLocateResultValue {\n  return result.coordinatesMeta.shape === 'point';\n}\n\nexport type LocateResultShape = 'bbox' | 'point';\n\nexport interface LocateResultContext {\n  preparedSize: {\n    width: number;\n    height: number;\n  };\n  contentSize?: {\n    width: number;\n    height: number;\n  };\n}\n\nexport interface LocateResultPromptSpec {\n  resultKey: string;\n  /** Parse-only alternatives, tried in order after resultKey. */\n  resultKeyAliases?: string[];\n  resultValueSchema: string;\n  resultValueDescription: string;\n  resultNoun: string;\n  resultNounPlural: string;\n  exampleValues: NonEmptyArray<unknown>;\n}\n\nexport interface PixelLocateResult {\n  /** Raw bbox midpoint (or raw point), mapped to the locate image using the configured rounding. */\n  center: LocateResultPoint;\n  /**\n   * Original bbox mapped to locate-image pixels and clipped to content bounds.\n   * Undefined for point results; never derive center or click coordinates from it.\n   */\n  rect?: Rect;\n}\n\nexport interface LocateResultCodec {\n  promptSpec: LocateResultPromptSpec;\n  /** Parse once and independently map the center and optional original bbox. */\n  toPixelResult(\n    input: RawLocateValue,\n    ctx: LocateResultContext,\n  ): PixelLocateResult;\n}\n\nexport type CoordinateRounding = 'round' | 'trunc' | 'none';\n\nexport type BboxBoundsType = 'inclusive' | 'edges';\n\nexport interface LocateResultCoordinates {\n  /** Bbox boundary semantics; defaults to inclusive pixel endpoints. */\n  boundsType?: BboxBoundsType;\n  shape: LocateResultShape;\n  /** Axis order in the raw coordinates; defaults to xy. */\n  order?: 'xy' | 'yx';\n  /** Normalization range, e.g. 1 or 1000. Omit for model-image pixel coordinates. */\n  normalizedBy?: number;\n  /** Rounding after normalized-to-pixel mapping; defaults to round. Pixel inputs retain their precision. */\n  rounding?: CoordinateRounding;\n}\n\n/** Coordinate metadata with defaults resolved and an explicit rounding policy. */\nexport type ResolvedLocateResultCoordinates =\n  | {\n      shape: 'point';\n      order: 'xy' | 'yx';\n      normalizedBy?: number;\n      rounding: CoordinateRounding;\n    }\n  | {\n      shape: 'bbox';\n      boundsType?: BboxBoundsType;\n      order: 'xy' | 'yx';\n      normalizedBy?: number;\n      rounding: CoordinateRounding;\n    };\n\nexport type RawLocateValueParser = (input: RawLocateValue) => LocateResultValue;\n\n/**\n * Format of one model coordinate value, shared by element and search-area locate.\n * The operation protocol extracts target/reference values from the response;\n * the codec parses each value once and maps it to `{ center, rect }` using the\n * parsed result's coordinatesMeta. Rect is undefined for actual point results.\n * Raw-value parsing and normalized-coordinate rounding are configurable;\n * coordinate mapping is shared.\n */\nexport type LocateResultFormatDefinition = {\n  /** Response field used in prompts and parsing; defaults to bbox or point. */\n  resultKey?: string;\n  /** Parse-only alternatives, tried in order when the primary field is undefined. */\n  resultKeyAliases?: string[];\n  /**\n   * Expected shape, axis order, and normalization used by the prompt and default\n   * parser. A custom parser may return different metadata for an actual response.\n   */\n  coordinates: LocateResultCoordinates;\n  /**\n   * Optional model-specific parser for nonstandard formats or point/bbox fallback.\n   * Return `{ coordinates, coordinatesMeta }` describing the actual shape, axis\n   * order, and normalization. Preserve coordinate precision; do not round or map\n   * to pixels here. Include an explicit rounding policy in `coordinatesMeta`.\n   * Omit the parser to parse numeric values according to `coordinates`.\n   */\n  parseRawLocateValue?: RawLocateValueParser;\n};\n"],"names":["isBboxLocateResultValue","result","isPointLocateResultValue"],"mappings":"AAoCO,SAASA,wBACdC,MAAyB;IAEzB,OAAOA,AAAiC,WAAjCA,OAAO,eAAe,CAAC,KAAK;AACrC;AAEO,SAASC,yBACdD,MAAyB;IAEzB,OAAOA,AAAiC,YAAjCA,OAAO,eAAe,CAAC,KAAK;AACrC"}

package/dist/es/ai-model/shared/model-locate-result/validation.mjs

@@ -6,6 +6,10 @@ function assertLocateResultStructure(result) {
     if (!expectedLength) throw new Error(`invalid parsed locate result: unsupported coordinatesMeta.shape ${JSON.stringify(coordinatesMeta?.shape)}`);
     const coordinates = result.coordinates;
     if (!Array.isArray(coordinates) || coordinates.length !== expectedLength || !coordinates.every((value)=>'number' == typeof value && Number.isFinite(value))) throw new Error(`invalid parsed locate result: ${coordinatesMeta.shape} coordinates must be ${expectedLength} finite numbers, got ${JSON.stringify(coordinates)}`);
+    if (isBboxLocateResultValue(result) && void 0 !== result.coordinatesMeta.boundsType && ![
+        'inclusive',
+        'edges'
+    ].includes(result.coordinatesMeta.boundsType)) throw new Error(`invalid locate bbox boundsType: ${result.coordinatesMeta.boundsType}`);
     const { order, normalizedBy, rounding } = coordinatesMeta;
     if (![
         'round',

package/dist/es/ai-model/shared/model-locate-result/validation.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/shared/model-locate-result/validation.mjs","sources":["../../../../../src/ai-model/shared/model-locate-result/validation.ts"],"sourcesContent":["import { isBboxLocateResultValue } from './types';\nimport type { LocateResultValue } from './types';\n\nexport function assertLocateResultStructure(result: LocateResultValue): void {\n  if (!result || typeof result !== 'object') {\n    throw new Error(\n      `invalid parsed locate result: expected object, got ${JSON.stringify(\n        result,\n      )}`,\n    );\n  }\n\n  const coordinatesMeta = result.coordinatesMeta;\n  const expectedLength =\n    coordinatesMeta?.shape === 'bbox'\n      ? 4\n      : coordinatesMeta?.shape === 'point'\n        ? 2\n        : 0;\n  if (!expectedLength) {\n    throw new Error(\n      `invalid parsed locate result: unsupported coordinatesMeta.shape ${JSON.stringify(\n        coordinatesMeta?.shape,\n      )}`,\n    );\n  }\n\n  const coordinates = result.coordinates;\n  if (\n    !Array.isArray(coordinates) ||\n    coordinates.length !== expectedLength ||\n    !coordinates.every(\n      (value) => typeof value === 'number' && Number.isFinite(value),\n    )\n  ) {\n    throw new Error(\n      `invalid parsed locate result: ${coordinatesMeta.shape} coordinates must be ${expectedLength} finite numbers, got ${JSON.stringify(\n        coordinates,\n      )}`,\n    );\n  }\n\n  const { order, normalizedBy, rounding } = coordinatesMeta;\n  if (!['round', 'trunc', 'none'].includes(rounding)) {\n    throw new Error(`invalid locate coordinate rounding: ${rounding}`);\n  }\n  if (order !== 'xy' && order !== 'yx') {\n    throw new Error(\n      `invalid locate coordinate order: ${JSON.stringify(order)}`,\n    );\n  }\n  if (\n    normalizedBy !== undefined &&\n    (!Number.isFinite(normalizedBy) || normalizedBy <= 0)\n  ) {\n    throw new Error(\n      `locate result normalizedBy must be positive and finite: ${normalizedBy}`,\n    );\n  }\n}\n\nfunction resolveCoordinateLimits(\n  result: LocateResultValue,\n  maxX: number,\n  maxY: number,\n): number[] {\n  const resolvedCoordinates = result.coordinatesMeta;\n  const normalizedBy = resolvedCoordinates.normalizedBy;\n  if (normalizedBy !== undefined) {\n    return result.coordinates.map(() => normalizedBy);\n  }\n\n  if (resolvedCoordinates.shape === 'bbox') {\n    return resolvedCoordinates.order === 'yx'\n      ? [maxY, maxX, maxY, maxX]\n      : [maxX, maxY, maxX, maxY];\n  }\n\n  return resolvedCoordinates.order === 'yx' ? [maxY, maxX] : [maxX, maxY];\n}\n\nexport function assertLocateResultCoordinateRangeAndOrder(\n  result: LocateResultValue,\n  maxX: number,\n  maxY: number,\n) {\n  const coordinatesMeta = result.coordinatesMeta;\n  const { normalizedBy } = coordinatesMeta;\n  const limits = resolveCoordinateLimits(result, maxX, maxY);\n  const outOfRange = result.coordinates.some(\n    (value, index) => value < 0 || value > limits[index],\n  );\n\n  if (outOfRange) {\n    const source =\n      normalizedBy !== undefined\n        ? `normalized range [0, ${normalizedBy}]`\n        : `image size [0, ${maxX}]x[0, ${maxY}]`;\n    const normalizedInfo =\n      normalizedBy !== undefined ? ` normalizedBy=${normalizedBy}` : '';\n    throw new Error(\n      `locate result coordinates ${JSON.stringify(\n        result.coordinates,\n      )} exceed ${source}. shape=${\n        coordinatesMeta.shape\n      } order=${coordinatesMeta.order}${normalizedInfo} limits=${JSON.stringify(\n        limits,\n      )}`,\n    );\n  }\n\n  if (\n    isBboxLocateResultValue(result) &&\n    (result.coordinates[2] < result.coordinates[0] ||\n      result.coordinates[3] < result.coordinates[1])\n  ) {\n    throw new Error(\n      `invalid locate bbox coordinate order: ${JSON.stringify(result.coordinates)}`,\n    );\n  }\n}\n"],"names":["assertLocateResultStructure","result","Error","JSON","coordinatesMeta","expectedLength","coordinates","Array","value","Number","order","normalizedBy","rounding","undefined","resolveCoordinateLimits","maxX","maxY","resolvedCoordinates","assertLocateResultCoordinateRangeAndOrder","limits","outOfRange","index","source","normalizedInfo","isBboxLocateResultValue"],"mappings":";AAGO,SAASA,4BAA4BC,MAAyB;IACnE,IAAI,CAACA,UAAU,AAAkB,YAAlB,OAAOA,QACpB,MAAM,IAAIC,MACR,CAAC,mDAAmD,EAAEC,KAAK,SAAS,CAClEF,SACC;IAIP,MAAMG,kBAAkBH,OAAO,eAAe;IAC9C,MAAMI,iBACJD,iBAAiB,UAAU,SACvB,IACAA,iBAAiB,UAAU,UACzB,IACA;IACR,IAAI,CAACC,gBACH,MAAM,IAAIH,MACR,CAAC,gEAAgE,EAAEC,KAAK,SAAS,CAC/EC,iBAAiB,QAChB;IAIP,MAAME,cAAcL,OAAO,WAAW;IACtC,IACE,CAACM,MAAM,OAAO,CAACD,gBACfA,YAAY,MAAM,KAAKD,kBACvB,CAACC,YAAY,KAAK,CAChB,CAACE,QAAU,AAAiB,YAAjB,OAAOA,SAAsBC,OAAO,QAAQ,CAACD,SAG1D,MAAM,IAAIN,MACR,CAAC,8BAA8B,EAAEE,gBAAgB,KAAK,CAAC,qBAAqB,EAAEC,eAAe,qBAAqB,EAAEF,KAAK,SAAS,CAChIG,cACC;IAIP,MAAM,EAAEI,KAAK,EAAEC,YAAY,EAAEC,QAAQ,EAAE,GAAGR;IAC1C,IAAI,CAAC;QAAC;QAAS;QAAS;KAAO,CAAC,QAAQ,CAACQ,WACvC,MAAM,IAAIV,MAAM,CAAC,oCAAoC,EAAEU,UAAU;IAEnE,IAAIF,AAAU,SAAVA,SAAkBA,AAAU,SAAVA,OACpB,MAAM,IAAIR,MACR,CAAC,iCAAiC,EAAEC,KAAK,SAAS,CAACO,QAAQ;IAG/D,IACEC,AAAiBE,WAAjBF,gBACC,EAACF,OAAO,QAAQ,CAACE,iBAAiBA,gBAAgB,IAEnD,MAAM,IAAIT,MACR,CAAC,wDAAwD,EAAES,cAAc;AAG/E;AAEA,SAASG,wBACPb,MAAyB,EACzBc,IAAY,EACZC,IAAY;IAEZ,MAAMC,sBAAsBhB,OAAO,eAAe;IAClD,MAAMU,eAAeM,oBAAoB,YAAY;IACrD,IAAIN,AAAiBE,WAAjBF,cACF,OAAOV,OAAO,WAAW,CAAC,GAAG,CAAC,IAAMU;IAGtC,IAAIM,AAA8B,WAA9BA,oBAAoB,KAAK,EAC3B,OAAOA,AAA8B,SAA9BA,oBAAoB,KAAK,GAC5B;QAACD;QAAMD;QAAMC;QAAMD;KAAK,GACxB;QAACA;QAAMC;QAAMD;QAAMC;KAAK;IAG9B,OAAOC,AAA8B,SAA9BA,oBAAoB,KAAK,GAAY;QAACD;QAAMD;KAAK,GAAG;QAACA;QAAMC;KAAK;AACzE;AAEO,SAASE,0CACdjB,MAAyB,EACzBc,IAAY,EACZC,IAAY;IAEZ,MAAMZ,kBAAkBH,OAAO,eAAe;IAC9C,MAAM,EAAEU,YAAY,EAAE,GAAGP;IACzB,MAAMe,SAASL,wBAAwBb,QAAQc,MAAMC;IACrD,MAAMI,aAAanB,OAAO,WAAW,CAAC,IAAI,CACxC,CAACO,OAAOa,QAAUb,QAAQ,KAAKA,QAAQW,MAAM,CAACE,MAAM;IAGtD,IAAID,YAAY;QACd,MAAME,SACJX,AAAiBE,WAAjBF,eACI,CAAC,qBAAqB,EAAEA,aAAa,CAAC,CAAC,GACvC,CAAC,eAAe,EAAEI,KAAK,MAAM,EAAEC,KAAK,CAAC,CAAC;QAC5C,MAAMO,iBACJZ,AAAiBE,WAAjBF,eAA6B,CAAC,cAAc,EAAEA,cAAc,GAAG;QACjE,MAAM,IAAIT,MACR,CAAC,0BAA0B,EAAEC,KAAK,SAAS,CACzCF,OAAO,WAAW,EAClB,QAAQ,EAAEqB,OAAO,QAAQ,EACzBlB,gBAAgB,KAAK,CACtB,OAAO,EAAEA,gBAAgB,KAAK,GAAGmB,eAAe,QAAQ,EAAEpB,KAAK,SAAS,CACvEgB,SACC;IAEP;IAEA,IACEK,wBAAwBvB,WACvBA,CAAAA,OAAO,WAAW,CAAC,EAAE,GAAGA,OAAO,WAAW,CAAC,EAAE,IAC5CA,OAAO,WAAW,CAAC,EAAE,GAAGA,OAAO,WAAW,CAAC,EAAC,GAE9C,MAAM,IAAIC,MACR,CAAC,sCAAsC,EAAEC,KAAK,SAAS,CAACF,OAAO,WAAW,GAAG;AAGnF"}
+{"version":3,"file":"ai-model/shared/model-locate-result/validation.mjs","sources":["../../../../../src/ai-model/shared/model-locate-result/validation.ts"],"sourcesContent":["import { isBboxLocateResultValue } from './types';\nimport type { LocateResultValue } from './types';\n\nexport function assertLocateResultStructure(result: LocateResultValue): void {\n  if (!result || typeof result !== 'object') {\n    throw new Error(\n      `invalid parsed locate result: expected object, got ${JSON.stringify(\n        result,\n      )}`,\n    );\n  }\n\n  const coordinatesMeta = result.coordinatesMeta;\n  const expectedLength =\n    coordinatesMeta?.shape === 'bbox'\n      ? 4\n      : coordinatesMeta?.shape === 'point'\n        ? 2\n        : 0;\n  if (!expectedLength) {\n    throw new Error(\n      `invalid parsed locate result: unsupported coordinatesMeta.shape ${JSON.stringify(\n        coordinatesMeta?.shape,\n      )}`,\n    );\n  }\n\n  const coordinates = result.coordinates;\n  if (\n    !Array.isArray(coordinates) ||\n    coordinates.length !== expectedLength ||\n    !coordinates.every(\n      (value) => typeof value === 'number' && Number.isFinite(value),\n    )\n  ) {\n    throw new Error(\n      `invalid parsed locate result: ${coordinatesMeta.shape} coordinates must be ${expectedLength} finite numbers, got ${JSON.stringify(\n        coordinates,\n      )}`,\n    );\n  }\n\n  if (\n    isBboxLocateResultValue(result) &&\n    result.coordinatesMeta.boundsType !== undefined &&\n    !['inclusive', 'edges'].includes(result.coordinatesMeta.boundsType)\n  ) {\n    throw new Error(\n      `invalid locate bbox boundsType: ${result.coordinatesMeta.boundsType}`,\n    );\n  }\n  const { order, normalizedBy, rounding } = coordinatesMeta;\n  if (!['round', 'trunc', 'none'].includes(rounding)) {\n    throw new Error(`invalid locate coordinate rounding: ${rounding}`);\n  }\n  if (order !== 'xy' && order !== 'yx') {\n    throw new Error(\n      `invalid locate coordinate order: ${JSON.stringify(order)}`,\n    );\n  }\n  if (\n    normalizedBy !== undefined &&\n    (!Number.isFinite(normalizedBy) || normalizedBy <= 0)\n  ) {\n    throw new Error(\n      `locate result normalizedBy must be positive and finite: ${normalizedBy}`,\n    );\n  }\n}\n\nfunction resolveCoordinateLimits(\n  result: LocateResultValue,\n  maxX: number,\n  maxY: number,\n): number[] {\n  const resolvedCoordinates = result.coordinatesMeta;\n  const normalizedBy = resolvedCoordinates.normalizedBy;\n  if (normalizedBy !== undefined) {\n    return result.coordinates.map(() => normalizedBy);\n  }\n\n  if (resolvedCoordinates.shape === 'bbox') {\n    return resolvedCoordinates.order === 'yx'\n      ? [maxY, maxX, maxY, maxX]\n      : [maxX, maxY, maxX, maxY];\n  }\n\n  return resolvedCoordinates.order === 'yx' ? [maxY, maxX] : [maxX, maxY];\n}\n\nexport function assertLocateResultCoordinateRangeAndOrder(\n  result: LocateResultValue,\n  maxX: number,\n  maxY: number,\n) {\n  const coordinatesMeta = result.coordinatesMeta;\n  const { normalizedBy } = coordinatesMeta;\n  const limits = resolveCoordinateLimits(result, maxX, maxY);\n  const outOfRange = result.coordinates.some(\n    (value, index) => value < 0 || value > limits[index],\n  );\n\n  if (outOfRange) {\n    const source =\n      normalizedBy !== undefined\n        ? `normalized range [0, ${normalizedBy}]`\n        : `image size [0, ${maxX}]x[0, ${maxY}]`;\n    const normalizedInfo =\n      normalizedBy !== undefined ? ` normalizedBy=${normalizedBy}` : '';\n    throw new Error(\n      `locate result coordinates ${JSON.stringify(\n        result.coordinates,\n      )} exceed ${source}. shape=${\n        coordinatesMeta.shape\n      } order=${coordinatesMeta.order}${normalizedInfo} limits=${JSON.stringify(\n        limits,\n      )}`,\n    );\n  }\n\n  if (\n    isBboxLocateResultValue(result) &&\n    (result.coordinates[2] < result.coordinates[0] ||\n      result.coordinates[3] < result.coordinates[1])\n  ) {\n    throw new Error(\n      `invalid locate bbox coordinate order: ${JSON.stringify(result.coordinates)}`,\n    );\n  }\n}\n"],"names":["assertLocateResultStructure","result","Error","JSON","coordinatesMeta","expectedLength","coordinates","Array","value","Number","isBboxLocateResultValue","undefined","order","normalizedBy","rounding","resolveCoordinateLimits","maxX","maxY","resolvedCoordinates","assertLocateResultCoordinateRangeAndOrder","limits","outOfRange","index","source","normalizedInfo"],"mappings":";AAGO,SAASA,4BAA4BC,MAAyB;IACnE,IAAI,CAACA,UAAU,AAAkB,YAAlB,OAAOA,QACpB,MAAM,IAAIC,MACR,CAAC,mDAAmD,EAAEC,KAAK,SAAS,CAClEF,SACC;IAIP,MAAMG,kBAAkBH,OAAO,eAAe;IAC9C,MAAMI,iBACJD,iBAAiB,UAAU,SACvB,IACAA,iBAAiB,UAAU,UACzB,IACA;IACR,IAAI,CAACC,gBACH,MAAM,IAAIH,MACR,CAAC,gEAAgE,EAAEC,KAAK,SAAS,CAC/EC,iBAAiB,QAChB;IAIP,MAAME,cAAcL,OAAO,WAAW;IACtC,IACE,CAACM,MAAM,OAAO,CAACD,gBACfA,YAAY,MAAM,KAAKD,kBACvB,CAACC,YAAY,KAAK,CAChB,CAACE,QAAU,AAAiB,YAAjB,OAAOA,SAAsBC,OAAO,QAAQ,CAACD,SAG1D,MAAM,IAAIN,MACR,CAAC,8BAA8B,EAAEE,gBAAgB,KAAK,CAAC,qBAAqB,EAAEC,eAAe,qBAAqB,EAAEF,KAAK,SAAS,CAChIG,cACC;IAIP,IACEI,wBAAwBT,WACxBA,AAAsCU,WAAtCV,OAAO,eAAe,CAAC,UAAU,IACjC,CAAC;QAAC;QAAa;KAAQ,CAAC,QAAQ,CAACA,OAAO,eAAe,CAAC,UAAU,GAElE,MAAM,IAAIC,MACR,CAAC,gCAAgC,EAAED,OAAO,eAAe,CAAC,UAAU,EAAE;IAG1E,MAAM,EAAEW,KAAK,EAAEC,YAAY,EAAEC,QAAQ,EAAE,GAAGV;IAC1C,IAAI,CAAC;QAAC;QAAS;QAAS;KAAO,CAAC,QAAQ,CAACU,WACvC,MAAM,IAAIZ,MAAM,CAAC,oCAAoC,EAAEY,UAAU;IAEnE,IAAIF,AAAU,SAAVA,SAAkBA,AAAU,SAAVA,OACpB,MAAM,IAAIV,MACR,CAAC,iCAAiC,EAAEC,KAAK,SAAS,CAACS,QAAQ;IAG/D,IACEC,AAAiBF,WAAjBE,gBACC,EAACJ,OAAO,QAAQ,CAACI,iBAAiBA,gBAAgB,IAEnD,MAAM,IAAIX,MACR,CAAC,wDAAwD,EAAEW,cAAc;AAG/E;AAEA,SAASE,wBACPd,MAAyB,EACzBe,IAAY,EACZC,IAAY;IAEZ,MAAMC,sBAAsBjB,OAAO,eAAe;IAClD,MAAMY,eAAeK,oBAAoB,YAAY;IACrD,IAAIL,AAAiBF,WAAjBE,cACF,OAAOZ,OAAO,WAAW,CAAC,GAAG,CAAC,IAAMY;IAGtC,IAAIK,AAA8B,WAA9BA,oBAAoB,KAAK,EAC3B,OAAOA,AAA8B,SAA9BA,oBAAoB,KAAK,GAC5B;QAACD;QAAMD;QAAMC;QAAMD;KAAK,GACxB;QAACA;QAAMC;QAAMD;QAAMC;KAAK;IAG9B,OAAOC,AAA8B,SAA9BA,oBAAoB,KAAK,GAAY;QAACD;QAAMD;KAAK,GAAG;QAACA;QAAMC;KAAK;AACzE;AAEO,SAASE,0CACdlB,MAAyB,EACzBe,IAAY,EACZC,IAAY;IAEZ,MAAMb,kBAAkBH,OAAO,eAAe;IAC9C,MAAM,EAAEY,YAAY,EAAE,GAAGT;IACzB,MAAMgB,SAASL,wBAAwBd,QAAQe,MAAMC;IACrD,MAAMI,aAAapB,OAAO,WAAW,CAAC,IAAI,CACxC,CAACO,OAAOc,QAAUd,QAAQ,KAAKA,QAAQY,MAAM,CAACE,MAAM;IAGtD,IAAID,YAAY;QACd,MAAME,SACJV,AAAiBF,WAAjBE,eACI,CAAC,qBAAqB,EAAEA,aAAa,CAAC,CAAC,GACvC,CAAC,eAAe,EAAEG,KAAK,MAAM,EAAEC,KAAK,CAAC,CAAC;QAC5C,MAAMO,iBACJX,AAAiBF,WAAjBE,eAA6B,CAAC,cAAc,EAAEA,cAAc,GAAG;QACjE,MAAM,IAAIX,MACR,CAAC,0BAA0B,EAAEC,KAAK,SAAS,CACzCF,OAAO,WAAW,EAClB,QAAQ,EAAEsB,OAAO,QAAQ,EACzBnB,gBAAgB,KAAK,CACtB,OAAO,EAAEA,gBAAgB,KAAK,GAAGoB,eAAe,QAAQ,EAAErB,KAAK,SAAS,CACvEiB,SACC;IAEP;IAEA,IACEV,wBAAwBT,WACvBA,CAAAA,OAAO,WAAW,CAAC,EAAE,GAAGA,OAAO,WAAW,CAAC,EAAE,IAC5CA,OAAO,WAAW,CAAC,EAAE,GAAGA,OAAO,WAAW,CAAC,EAAC,GAE9C,MAAM,IAAIC,MACR,CAAC,sCAAsC,EAAEC,KAAK,SAAS,CAACF,OAAO,WAAW,GAAG;AAGnF"}

package/dist/es/ai-model/workflows/grounding/locate.mjs

@@ -2,7 +2,7 @@ import { getDebug } from "@midscene/shared/logger";
 import { assert } from "@midscene/shared/utils";
 import { userPromptToString } from "../../../common.mjs";
 import { buildElementLocateSystemPrompt } from "../../prompt/locate/index.mjs";
-import { AIResponseParseError, callAI } from "../../service-caller/index.mjs";
+import { AIResponseParseError, callGenerativeAI } from "../../service-caller/index.mjs";
 import { callAiAndParseWithRetry, withSemanticRetryFeedback } from "../../service-caller/semantic-retry.mjs";
 import { mapSearchAreaResultToOriginalResult } from "./search-area-mapping.mjs";
 import { formatLocateModelContext, prepareLocateModelInput } from "./utils.mjs";
@@ -88,7 +88,7 @@ async function genericLocate(locateRequest) {
     });
     try {
         return await callAiAndParseWithRetry({
-            callAi: (retryAttempt, previousParseError)=>callAI(withSemanticRetryFeedback(messages, previousParseError), modelRuntime, {
+            callAi: (retryAttempt, previousParseError)=>callGenerativeAI(withSemanticRetryFeedback(messages, previousParseError), modelRuntime, {
                     abortSignal: options.abortSignal,
                     expectedJsonObjectResponse: protocol.expectedJsonObjectResponse,
                     semanticRetryAttempt: retryAttempt

package/dist/es/ai-model/workflows/grounding/locate.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/workflows/grounding/locate.mjs","sources":["../../../../../src/ai-model/workflows/grounding/locate.ts"],"sourcesContent":["import { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TUserPrompt } from '../../../common';\nimport { userPromptToString } from '../../../common';\nimport { buildElementLocateSystemPrompt } from '../../prompt/locate';\nimport { AIResponseParseError, callAI } from '../../service-caller/index';\nimport {\n  callAiAndParseWithRetry,\n  withSemanticRetryFeedback,\n} from '../../service-caller/semantic-retry';\nimport { mapSearchAreaResultToOriginalResult } from './search-area-mapping';\nimport type {\n  LocateModelResponse,\n  LocateOptions,\n  LocateRequest,\n  LocateResult,\n} from './types';\nimport { formatLocateModelContext, prepareLocateModelInput } from './utils';\n\nconst debugGrounding = getDebug('ai:grounding');\n\nexport { userPromptToString as extraTextFromUserPrompt } from '../../../common';\n\nexport async function AiLocateElement(\n  options: LocateOptions & { targetElementDescription: TUserPrompt },\n): Promise<LocateResult> {\n  const { targetElementDescription, ...locateOptions } = options;\n  assert(\n    targetElementDescription,\n    'cannot find the target element description',\n  );\n\n  const { context } = locateOptions;\n  const locateImage = locateOptions.searchConfig?.image ?? {\n    imageBase64: context.screenshot.base64,\n    width: context.shotSize.width,\n    height: context.shotSize.height,\n  };\n  const locateRequest: LocateRequest = {\n    targetElementDescription,\n    locateImage,\n    options: locateOptions,\n  };\n\n  const locateAdapter = options.modelRuntime.adapter.locate;\n  const locateFn =\n    locateAdapter.kind === 'custom' ? locateAdapter.locateFn : genericLocate;\n  const locateResponse = await locateFn(locateRequest);\n  const {\n    locatedPixelResult,\n    rawResponse,\n    rawAssistantOutput,\n    usage,\n    reasoningContent,\n    errors = [],\n  } = locateResponse;\n  const baseLocateResult = {\n    rawResponse,\n    rawAssistantOutput,\n    usage,\n    reasoning_content: reasoningContent,\n  };\n\n  if (!locatedPixelResult) {\n    return {\n      parseResult: {\n        element: undefined,\n        errors,\n      },\n      ...baseLocateResult,\n    };\n  }\n\n  try {\n    const mappedResult = mapSearchAreaResultToOriginalResult(\n      locatedPixelResult,\n      locateOptions.searchConfig?.mapping,\n    );\n    debugGrounding('center', mappedResult.center);\n\n    return {\n      parseResult: {\n        element: {\n          ...mappedResult,\n          description: userPromptToString(targetElementDescription),\n        },\n        errors: [],\n      },\n      ...baseLocateResult,\n    };\n  } catch (error) {\n    const msg =\n      error instanceof Error\n        ? `Failed to parse locate result: ${error.message}`\n        : 'unknown error in locate';\n    return {\n      parseResult: {\n        element: undefined,\n        errors: errors.length > 0 ? [...errors, `(${msg})`] : [msg],\n      },\n      ...baseLocateResult,\n    };\n  }\n}\n\nexport async function genericLocate(\n  locateRequest: LocateRequest,\n): Promise<LocateModelResponse> {\n  const { options, targetElementDescription } = locateRequest;\n  const modelRuntime = options.modelRuntime;\n  const { adapter } = modelRuntime;\n  assert(\n    adapter.locate.kind === 'standard',\n    'generic locate requires a standard locate adapter',\n  );\n  const { protocol, resultCodec } = adapter.locate.element;\n\n  const systemPrompt = buildElementLocateSystemPrompt({\n    systemPromptIntroduction: protocol.systemPromptIntroduction,\n    responseInstructions: protocol.buildResponseInstructions(\n      resultCodec.promptSpec,\n    ),\n  });\n\n  const userInstructionPrompt = protocol.buildUserPrompt(\n    userPromptToString(targetElementDescription),\n  );\n\n  const { messages, preparedImage } = await prepareLocateModelInput({\n    systemPrompt,\n    userPrompt: userInstructionPrompt,\n    locateImage: locateRequest.locateImage,\n    imagePreprocess: adapter.imagePreprocess,\n    targetDescription: targetElementDescription,\n    userMessageContentOrder: adapter.locate.userMessageContentOrder,\n  });\n\n  try {\n    return await callAiAndParseWithRetry({\n      callAi: (retryAttempt, previousParseError) =>\n        callAI(\n          withSemanticRetryFeedback(messages, previousParseError),\n          modelRuntime,\n          {\n            abortSignal: options.abortSignal,\n            expectedJsonObjectResponse: protocol.expectedJsonObjectResponse,\n            semanticRetryAttempt: retryAttempt,\n          },\n        ),\n      parseResponse: (response): LocateModelResponse => {\n        const parsedLocateResult = protocol.parseRawResponse(\n          response.content,\n          resultCodec.promptSpec,\n        );\n        const rawResponse = response.content;\n        const locateError = parsedLocateResult.error;\n        if (parsedLocateResult.kind === 'not-found') {\n          return {\n            rawResponse,\n            rawAssistantOutput: response.rawAssistantOutput,\n            usage: response.usage,\n            reasoningContent: response.reasoning_content,\n            errors: locateError ? [locateError] : [],\n          };\n        }\n\n        try {\n          const result = resultCodec.toPixelResult(parsedLocateResult.target, {\n            preparedSize: preparedImage.preparedSize,\n            contentSize: preparedImage.contentSize,\n          });\n          return {\n            locatedPixelResult: result,\n            rawResponse,\n            rawAssistantOutput: response.rawAssistantOutput,\n            usage: response.usage,\n            reasoningContent: response.reasoning_content,\n            errors: [],\n          };\n        } catch (error) {\n          const errorMessage =\n            error instanceof Error ? error.message : String(error);\n          const message = [\n            locateError && `error in locate result: ${locateError}`,\n            `coordinate parsing error: ${errorMessage}`,\n            formatLocateModelContext(modelRuntime),\n          ]\n            .filter(Boolean)\n            .join('\\n');\n          throw new Error(message, { cause: error });\n        }\n      },\n      toParseError: (error, response) => {\n        const parseErrorMessage =\n          error instanceof Error\n            ? `Failed to parse locate result: ${error.message}`\n            : 'unknown error in locate result';\n        return new AIResponseParseError(\n          parseErrorMessage,\n          response.content,\n          response.usage,\n          response.rawAssistantOutput,\n          response.reasoning_content,\n        );\n      },\n      parseRetryTimes: modelRuntime.config.retryCount,\n      parseRetryInterval: modelRuntime.config.retryInterval,\n      abortSignal: options.abortSignal,\n      onParseRetry: (error) => {\n        debugGrounding(\n          'retrying locate after coordinate parsing failed: %s',\n          error instanceof Error ? error.message : String(error),\n        );\n      },\n    });\n  } catch (callError) {\n    if (callError instanceof AIResponseParseError) {\n      return {\n        rawResponse: callError.rawResponse,\n        rawAssistantOutput: callError.rawAssistantOutput,\n        usage: callError.usage,\n        reasoningContent: callError.reasoningContent,\n        errors: [callError.message],\n      };\n    }\n\n    const errorMessage =\n      callError instanceof Error ? callError.message : String(callError);\n    return {\n      rawResponse: errorMessage,\n      errors: [`AI call error: ${errorMessage}`],\n    };\n  }\n}\n"],"names":["debugGrounding","getDebug","AiLocateElement","options","targetElementDescription","locateOptions","assert","context","locateImage","locateRequest","locateAdapter","locateFn","genericLocate","locateResponse","locatedPixelResult","rawResponse","rawAssistantOutput","usage","reasoningContent","errors","baseLocateResult","undefined","mappedResult","mapSearchAreaResultToOriginalResult","userPromptToString","error","msg","Error","modelRuntime","adapter","protocol","resultCodec","systemPrompt","buildElementLocateSystemPrompt","userInstructionPrompt","messages","preparedImage","prepareLocateModelInput","callAiAndParseWithRetry","retryAttempt","previousParseError","callAI","withSemanticRetryFeedback","response","parsedLocateResult","locateError","result","errorMessage","String","message","formatLocateModelContext","Boolean","parseErrorMessage","AIResponseParseError","callError"],"mappings":";;;;;;;;AAmBA,MAAMA,iBAAiBC,SAAS;AAIzB,eAAeC,gBACpBC,OAAkE;IAElE,MAAM,EAAEC,wBAAwB,EAAE,GAAGC,eAAe,GAAGF;IACvDG,OACEF,0BACA;IAGF,MAAM,EAAEG,OAAO,EAAE,GAAGF;IACpB,MAAMG,cAAcH,cAAc,YAAY,EAAE,SAAS;QACvD,aAAaE,QAAQ,UAAU,CAAC,MAAM;QACtC,OAAOA,QAAQ,QAAQ,CAAC,KAAK;QAC7B,QAAQA,QAAQ,QAAQ,CAAC,MAAM;IACjC;IACA,MAAME,gBAA+B;QACnCL;QACAI;QACA,SAASH;IACX;IAEA,MAAMK,gBAAgBP,QAAQ,YAAY,CAAC,OAAO,CAAC,MAAM;IACzD,MAAMQ,WACJD,AAAuB,aAAvBA,cAAc,IAAI,GAAgBA,cAAc,QAAQ,GAAGE;IAC7D,MAAMC,iBAAiB,MAAMF,SAASF;IACtC,MAAM,EACJK,kBAAkB,EAClBC,WAAW,EACXC,kBAAkB,EAClBC,KAAK,EACLC,gBAAgB,EAChBC,SAAS,EAAE,EACZ,GAAGN;IACJ,MAAMO,mBAAmB;QACvBL;QACAC;QACAC;QACA,mBAAmBC;IACrB;IAEA,IAAI,CAACJ,oBACH,OAAO;QACL,aAAa;YACX,SAASO;YACTF;QACF;QACA,GAAGC,gBAAgB;IACrB;IAGF,IAAI;QACF,MAAME,eAAeC,oCACnBT,oBACAT,cAAc,YAAY,EAAE;QAE9BL,eAAe,UAAUsB,aAAa,MAAM;QAE5C,OAAO;YACL,aAAa;gBACX,SAAS;oBACP,GAAGA,YAAY;oBACf,aAAaE,mBAAmBpB;gBAClC;gBACA,QAAQ,EAAE;YACZ;YACA,GAAGgB,gBAAgB;QACrB;IACF,EAAE,OAAOK,OAAO;QACd,MAAMC,MACJD,iBAAiBE,QACb,CAAC,+BAA+B,EAAEF,MAAM,OAAO,EAAE,GACjD;QACN,OAAO;YACL,aAAa;gBACX,SAASJ;gBACT,QAAQF,OAAO,MAAM,GAAG,IAAI;uBAAIA;oBAAQ,CAAC,CAAC,EAAEO,IAAI,CAAC,CAAC;iBAAC,GAAG;oBAACA;iBAAI;YAC7D;YACA,GAAGN,gBAAgB;QACrB;IACF;AACF;AAEO,eAAeR,cACpBH,aAA4B;IAE5B,MAAM,EAAEN,OAAO,EAAEC,wBAAwB,EAAE,GAAGK;IAC9C,MAAMmB,eAAezB,QAAQ,YAAY;IACzC,MAAM,EAAE0B,OAAO,EAAE,GAAGD;IACpBtB,OACEuB,AAAwB,eAAxBA,QAAQ,MAAM,CAAC,IAAI,EACnB;IAEF,MAAM,EAAEC,QAAQ,EAAEC,WAAW,EAAE,GAAGF,QAAQ,MAAM,CAAC,OAAO;IAExD,MAAMG,eAAeC,+BAA+B;QAClD,0BAA0BH,SAAS,wBAAwB;QAC3D,sBAAsBA,SAAS,yBAAyB,CACtDC,YAAY,UAAU;IAE1B;IAEA,MAAMG,wBAAwBJ,SAAS,eAAe,CACpDN,mBAAmBpB;IAGrB,MAAM,EAAE+B,QAAQ,EAAEC,aAAa,EAAE,GAAG,MAAMC,wBAAwB;QAChEL;QACA,YAAYE;QACZ,aAAazB,cAAc,WAAW;QACtC,iBAAiBoB,QAAQ,eAAe;QACxC,mBAAmBzB;QACnB,yBAAyByB,QAAQ,MAAM,CAAC,uBAAuB;IACjE;IAEA,IAAI;QACF,OAAO,MAAMS,wBAAwB;YACnC,QAAQ,CAACC,cAAcC,qBACrBC,OACEC,0BAA0BP,UAAUK,qBACpCZ,cACA;oBACE,aAAazB,QAAQ,WAAW;oBAChC,4BAA4B2B,SAAS,0BAA0B;oBAC/D,sBAAsBS;gBACxB;YAEJ,eAAe,CAACI;gBACd,MAAMC,qBAAqBd,SAAS,gBAAgB,CAClDa,SAAS,OAAO,EAChBZ,YAAY,UAAU;gBAExB,MAAMhB,cAAc4B,SAAS,OAAO;gBACpC,MAAME,cAAcD,mBAAmB,KAAK;gBAC5C,IAAIA,AAA4B,gBAA5BA,mBAAmB,IAAI,EACzB,OAAO;oBACL7B;oBACA,oBAAoB4B,SAAS,kBAAkB;oBAC/C,OAAOA,SAAS,KAAK;oBACrB,kBAAkBA,SAAS,iBAAiB;oBAC5C,QAAQE,cAAc;wBAACA;qBAAY,GAAG,EAAE;gBAC1C;gBAGF,IAAI;oBACF,MAAMC,SAASf,YAAY,aAAa,CAACa,mBAAmB,MAAM,EAAE;wBAClE,cAAcR,cAAc,YAAY;wBACxC,aAAaA,cAAc,WAAW;oBACxC;oBACA,OAAO;wBACL,oBAAoBU;wBACpB/B;wBACA,oBAAoB4B,SAAS,kBAAkB;wBAC/C,OAAOA,SAAS,KAAK;wBACrB,kBAAkBA,SAAS,iBAAiB;wBAC5C,QAAQ,EAAE;oBACZ;gBACF,EAAE,OAAOlB,OAAO;oBACd,MAAMsB,eACJtB,iBAAiBE,QAAQF,MAAM,OAAO,GAAGuB,OAAOvB;oBAClD,MAAMwB,UAAU;wBACdJ,eAAe,CAAC,wBAAwB,EAAEA,aAAa;wBACvD,CAAC,0BAA0B,EAAEE,cAAc;wBAC3CG,yBAAyBtB;qBAC1B,CACE,MAAM,CAACuB,SACP,IAAI,CAAC;oBACR,MAAM,IAAIxB,MAAMsB,SAAS;wBAAE,OAAOxB;oBAAM;gBAC1C;YACF;YACA,cAAc,CAACA,OAAOkB;gBACpB,MAAMS,oBACJ3B,iBAAiBE,QACb,CAAC,+BAA+B,EAAEF,MAAM,OAAO,EAAE,GACjD;gBACN,OAAO,IAAI4B,qBACTD,mBACAT,SAAS,OAAO,EAChBA,SAAS,KAAK,EACdA,SAAS,kBAAkB,EAC3BA,SAAS,iBAAiB;YAE9B;YACA,iBAAiBf,aAAa,MAAM,CAAC,UAAU;YAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;YACrD,aAAazB,QAAQ,WAAW;YAChC,cAAc,CAACsB;gBACbzB,eACE,uDACAyB,iBAAiBE,QAAQF,MAAM,OAAO,GAAGuB,OAAOvB;YAEpD;QACF;IACF,EAAE,OAAO6B,WAAW;QAClB,IAAIA,qBAAqBD,sBACvB,OAAO;YACL,aAAaC,UAAU,WAAW;YAClC,oBAAoBA,UAAU,kBAAkB;YAChD,OAAOA,UAAU,KAAK;YACtB,kBAAkBA,UAAU,gBAAgB;YAC5C,QAAQ;gBAACA,UAAU,OAAO;aAAC;QAC7B;QAGF,MAAMP,eACJO,qBAAqB3B,QAAQ2B,UAAU,OAAO,GAAGN,OAAOM;QAC1D,OAAO;YACL,aAAaP;YACb,QAAQ;gBAAC,CAAC,eAAe,EAAEA,cAAc;aAAC;QAC5C;IACF;AACF"}
+{"version":3,"file":"ai-model/workflows/grounding/locate.mjs","sources":["../../../../../src/ai-model/workflows/grounding/locate.ts"],"sourcesContent":["import { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TUserPrompt } from '../../../common';\nimport { userPromptToString } from '../../../common';\nimport { buildElementLocateSystemPrompt } from '../../prompt/locate';\nimport {\n  AIResponseParseError,\n  callGenerativeAI,\n} from '../../service-caller/index';\nimport {\n  callAiAndParseWithRetry,\n  withSemanticRetryFeedback,\n} from '../../service-caller/semantic-retry';\nimport { mapSearchAreaResultToOriginalResult } from './search-area-mapping';\nimport type {\n  LocateModelResponse,\n  LocateOptions,\n  LocateRequest,\n  LocateResult,\n} from './types';\nimport { formatLocateModelContext, prepareLocateModelInput } from './utils';\n\nconst debugGrounding = getDebug('ai:grounding');\n\nexport { userPromptToString as extraTextFromUserPrompt } from '../../../common';\n\nexport async function AiLocateElement(\n  options: LocateOptions & { targetElementDescription: TUserPrompt },\n): Promise<LocateResult> {\n  const { targetElementDescription, ...locateOptions } = options;\n  assert(\n    targetElementDescription,\n    'cannot find the target element description',\n  );\n\n  const { context } = locateOptions;\n  const locateImage = locateOptions.searchConfig?.image ?? {\n    imageBase64: context.screenshot.base64,\n    width: context.shotSize.width,\n    height: context.shotSize.height,\n  };\n  const locateRequest: LocateRequest = {\n    targetElementDescription,\n    locateImage,\n    options: locateOptions,\n  };\n\n  const locateAdapter = options.modelRuntime.adapter.locate;\n  const locateFn =\n    locateAdapter.kind === 'custom' ? locateAdapter.locateFn : genericLocate;\n  const locateResponse = await locateFn(locateRequest);\n  const {\n    locatedPixelResult,\n    rawResponse,\n    rawAssistantOutput,\n    usage,\n    reasoningContent,\n    errors = [],\n  } = locateResponse;\n  const baseLocateResult = {\n    rawResponse,\n    rawAssistantOutput,\n    usage,\n    reasoning_content: reasoningContent,\n  };\n\n  if (!locatedPixelResult) {\n    return {\n      parseResult: {\n        element: undefined,\n        errors,\n      },\n      ...baseLocateResult,\n    };\n  }\n\n  try {\n    const mappedResult = mapSearchAreaResultToOriginalResult(\n      locatedPixelResult,\n      locateOptions.searchConfig?.mapping,\n    );\n    debugGrounding('center', mappedResult.center);\n\n    return {\n      parseResult: {\n        element: {\n          ...mappedResult,\n          description: userPromptToString(targetElementDescription),\n        },\n        errors: [],\n      },\n      ...baseLocateResult,\n    };\n  } catch (error) {\n    const msg =\n      error instanceof Error\n        ? `Failed to parse locate result: ${error.message}`\n        : 'unknown error in locate';\n    return {\n      parseResult: {\n        element: undefined,\n        errors: errors.length > 0 ? [...errors, `(${msg})`] : [msg],\n      },\n      ...baseLocateResult,\n    };\n  }\n}\n\nexport async function genericLocate(\n  locateRequest: LocateRequest,\n): Promise<LocateModelResponse> {\n  const { options, targetElementDescription } = locateRequest;\n  const modelRuntime = options.modelRuntime;\n  const { adapter } = modelRuntime;\n  assert(\n    adapter.locate.kind === 'standard',\n    'generic locate requires a standard locate adapter',\n  );\n  const { protocol, resultCodec } = adapter.locate.element;\n\n  const systemPrompt = buildElementLocateSystemPrompt({\n    systemPromptIntroduction: protocol.systemPromptIntroduction,\n    responseInstructions: protocol.buildResponseInstructions(\n      resultCodec.promptSpec,\n    ),\n  });\n\n  const userInstructionPrompt = protocol.buildUserPrompt(\n    userPromptToString(targetElementDescription),\n  );\n\n  const { messages, preparedImage } = await prepareLocateModelInput({\n    systemPrompt,\n    userPrompt: userInstructionPrompt,\n    locateImage: locateRequest.locateImage,\n    imagePreprocess: adapter.imagePreprocess,\n    targetDescription: targetElementDescription,\n    userMessageContentOrder: adapter.locate.userMessageContentOrder,\n  });\n\n  try {\n    return await callAiAndParseWithRetry({\n      callAi: (retryAttempt, previousParseError) =>\n        callGenerativeAI(\n          withSemanticRetryFeedback(messages, previousParseError),\n          modelRuntime,\n          {\n            abortSignal: options.abortSignal,\n            expectedJsonObjectResponse: protocol.expectedJsonObjectResponse,\n            semanticRetryAttempt: retryAttempt,\n          },\n        ),\n      parseResponse: (response): LocateModelResponse => {\n        const parsedLocateResult = protocol.parseRawResponse(\n          response.content,\n          resultCodec.promptSpec,\n        );\n        const rawResponse = response.content;\n        const locateError = parsedLocateResult.error;\n        if (parsedLocateResult.kind === 'not-found') {\n          return {\n            rawResponse,\n            rawAssistantOutput: response.rawAssistantOutput,\n            usage: response.usage,\n            reasoningContent: response.reasoning_content,\n            errors: locateError ? [locateError] : [],\n          };\n        }\n\n        try {\n          const result = resultCodec.toPixelResult(parsedLocateResult.target, {\n            preparedSize: preparedImage.preparedSize,\n            contentSize: preparedImage.contentSize,\n          });\n          return {\n            locatedPixelResult: result,\n            rawResponse,\n            rawAssistantOutput: response.rawAssistantOutput,\n            usage: response.usage,\n            reasoningContent: response.reasoning_content,\n            errors: [],\n          };\n        } catch (error) {\n          const errorMessage =\n            error instanceof Error ? error.message : String(error);\n          const message = [\n            locateError && `error in locate result: ${locateError}`,\n            `coordinate parsing error: ${errorMessage}`,\n            formatLocateModelContext(modelRuntime),\n          ]\n            .filter(Boolean)\n            .join('\\n');\n          throw new Error(message, { cause: error });\n        }\n      },\n      toParseError: (error, response) => {\n        const parseErrorMessage =\n          error instanceof Error\n            ? `Failed to parse locate result: ${error.message}`\n            : 'unknown error in locate result';\n        return new AIResponseParseError(\n          parseErrorMessage,\n          response.content,\n          response.usage,\n          response.rawAssistantOutput,\n          response.reasoning_content,\n        );\n      },\n      parseRetryTimes: modelRuntime.config.retryCount,\n      parseRetryInterval: modelRuntime.config.retryInterval,\n      abortSignal: options.abortSignal,\n      onParseRetry: (error) => {\n        debugGrounding(\n          'retrying locate after coordinate parsing failed: %s',\n          error instanceof Error ? error.message : String(error),\n        );\n      },\n    });\n  } catch (callError) {\n    if (callError instanceof AIResponseParseError) {\n      return {\n        rawResponse: callError.rawResponse,\n        rawAssistantOutput: callError.rawAssistantOutput,\n        usage: callError.usage,\n        reasoningContent: callError.reasoningContent,\n        errors: [callError.message],\n      };\n    }\n\n    const errorMessage =\n      callError instanceof Error ? callError.message : String(callError);\n    return {\n      rawResponse: errorMessage,\n      errors: [`AI call error: ${errorMessage}`],\n    };\n  }\n}\n"],"names":["debugGrounding","getDebug","AiLocateElement","options","targetElementDescription","locateOptions","assert","context","locateImage","locateRequest","locateAdapter","locateFn","genericLocate","locateResponse","locatedPixelResult","rawResponse","rawAssistantOutput","usage","reasoningContent","errors","baseLocateResult","undefined","mappedResult","mapSearchAreaResultToOriginalResult","userPromptToString","error","msg","Error","modelRuntime","adapter","protocol","resultCodec","systemPrompt","buildElementLocateSystemPrompt","userInstructionPrompt","messages","preparedImage","prepareLocateModelInput","callAiAndParseWithRetry","retryAttempt","previousParseError","callGenerativeAI","withSemanticRetryFeedback","response","parsedLocateResult","locateError","result","errorMessage","String","message","formatLocateModelContext","Boolean","parseErrorMessage","AIResponseParseError","callError"],"mappings":";;;;;;;;AAsBA,MAAMA,iBAAiBC,SAAS;AAIzB,eAAeC,gBACpBC,OAAkE;IAElE,MAAM,EAAEC,wBAAwB,EAAE,GAAGC,eAAe,GAAGF;IACvDG,OACEF,0BACA;IAGF,MAAM,EAAEG,OAAO,EAAE,GAAGF;IACpB,MAAMG,cAAcH,cAAc,YAAY,EAAE,SAAS;QACvD,aAAaE,QAAQ,UAAU,CAAC,MAAM;QACtC,OAAOA,QAAQ,QAAQ,CAAC,KAAK;QAC7B,QAAQA,QAAQ,QAAQ,CAAC,MAAM;IACjC;IACA,MAAME,gBAA+B;QACnCL;QACAI;QACA,SAASH;IACX;IAEA,MAAMK,gBAAgBP,QAAQ,YAAY,CAAC,OAAO,CAAC,MAAM;IACzD,MAAMQ,WACJD,AAAuB,aAAvBA,cAAc,IAAI,GAAgBA,cAAc,QAAQ,GAAGE;IAC7D,MAAMC,iBAAiB,MAAMF,SAASF;IACtC,MAAM,EACJK,kBAAkB,EAClBC,WAAW,EACXC,kBAAkB,EAClBC,KAAK,EACLC,gBAAgB,EAChBC,SAAS,EAAE,EACZ,GAAGN;IACJ,MAAMO,mBAAmB;QACvBL;QACAC;QACAC;QACA,mBAAmBC;IACrB;IAEA,IAAI,CAACJ,oBACH,OAAO;QACL,aAAa;YACX,SAASO;YACTF;QACF;QACA,GAAGC,gBAAgB;IACrB;IAGF,IAAI;QACF,MAAME,eAAeC,oCACnBT,oBACAT,cAAc,YAAY,EAAE;QAE9BL,eAAe,UAAUsB,aAAa,MAAM;QAE5C,OAAO;YACL,aAAa;gBACX,SAAS;oBACP,GAAGA,YAAY;oBACf,aAAaE,mBAAmBpB;gBAClC;gBACA,QAAQ,EAAE;YACZ;YACA,GAAGgB,gBAAgB;QACrB;IACF,EAAE,OAAOK,OAAO;QACd,MAAMC,MACJD,iBAAiBE,QACb,CAAC,+BAA+B,EAAEF,MAAM,OAAO,EAAE,GACjD;QACN,OAAO;YACL,aAAa;gBACX,SAASJ;gBACT,QAAQF,OAAO,MAAM,GAAG,IAAI;uBAAIA;oBAAQ,CAAC,CAAC,EAAEO,IAAI,CAAC,CAAC;iBAAC,GAAG;oBAACA;iBAAI;YAC7D;YACA,GAAGN,gBAAgB;QACrB;IACF;AACF;AAEO,eAAeR,cACpBH,aAA4B;IAE5B,MAAM,EAAEN,OAAO,EAAEC,wBAAwB,EAAE,GAAGK;IAC9C,MAAMmB,eAAezB,QAAQ,YAAY;IACzC,MAAM,EAAE0B,OAAO,EAAE,GAAGD;IACpBtB,OACEuB,AAAwB,eAAxBA,QAAQ,MAAM,CAAC,IAAI,EACnB;IAEF,MAAM,EAAEC,QAAQ,EAAEC,WAAW,EAAE,GAAGF,QAAQ,MAAM,CAAC,OAAO;IAExD,MAAMG,eAAeC,+BAA+B;QAClD,0BAA0BH,SAAS,wBAAwB;QAC3D,sBAAsBA,SAAS,yBAAyB,CACtDC,YAAY,UAAU;IAE1B;IAEA,MAAMG,wBAAwBJ,SAAS,eAAe,CACpDN,mBAAmBpB;IAGrB,MAAM,EAAE+B,QAAQ,EAAEC,aAAa,EAAE,GAAG,MAAMC,wBAAwB;QAChEL;QACA,YAAYE;QACZ,aAAazB,cAAc,WAAW;QACtC,iBAAiBoB,QAAQ,eAAe;QACxC,mBAAmBzB;QACnB,yBAAyByB,QAAQ,MAAM,CAAC,uBAAuB;IACjE;IAEA,IAAI;QACF,OAAO,MAAMS,wBAAwB;YACnC,QAAQ,CAACC,cAAcC,qBACrBC,iBACEC,0BAA0BP,UAAUK,qBACpCZ,cACA;oBACE,aAAazB,QAAQ,WAAW;oBAChC,4BAA4B2B,SAAS,0BAA0B;oBAC/D,sBAAsBS;gBACxB;YAEJ,eAAe,CAACI;gBACd,MAAMC,qBAAqBd,SAAS,gBAAgB,CAClDa,SAAS,OAAO,EAChBZ,YAAY,UAAU;gBAExB,MAAMhB,cAAc4B,SAAS,OAAO;gBACpC,MAAME,cAAcD,mBAAmB,KAAK;gBAC5C,IAAIA,AAA4B,gBAA5BA,mBAAmB,IAAI,EACzB,OAAO;oBACL7B;oBACA,oBAAoB4B,SAAS,kBAAkB;oBAC/C,OAAOA,SAAS,KAAK;oBACrB,kBAAkBA,SAAS,iBAAiB;oBAC5C,QAAQE,cAAc;wBAACA;qBAAY,GAAG,EAAE;gBAC1C;gBAGF,IAAI;oBACF,MAAMC,SAASf,YAAY,aAAa,CAACa,mBAAmB,MAAM,EAAE;wBAClE,cAAcR,cAAc,YAAY;wBACxC,aAAaA,cAAc,WAAW;oBACxC;oBACA,OAAO;wBACL,oBAAoBU;wBACpB/B;wBACA,oBAAoB4B,SAAS,kBAAkB;wBAC/C,OAAOA,SAAS,KAAK;wBACrB,kBAAkBA,SAAS,iBAAiB;wBAC5C,QAAQ,EAAE;oBACZ;gBACF,EAAE,OAAOlB,OAAO;oBACd,MAAMsB,eACJtB,iBAAiBE,QAAQF,MAAM,OAAO,GAAGuB,OAAOvB;oBAClD,MAAMwB,UAAU;wBACdJ,eAAe,CAAC,wBAAwB,EAAEA,aAAa;wBACvD,CAAC,0BAA0B,EAAEE,cAAc;wBAC3CG,yBAAyBtB;qBAC1B,CACE,MAAM,CAACuB,SACP,IAAI,CAAC;oBACR,MAAM,IAAIxB,MAAMsB,SAAS;wBAAE,OAAOxB;oBAAM;gBAC1C;YACF;YACA,cAAc,CAACA,OAAOkB;gBACpB,MAAMS,oBACJ3B,iBAAiBE,QACb,CAAC,+BAA+B,EAAEF,MAAM,OAAO,EAAE,GACjD;gBACN,OAAO,IAAI4B,qBACTD,mBACAT,SAAS,OAAO,EAChBA,SAAS,KAAK,EACdA,SAAS,kBAAkB,EAC3BA,SAAS,iBAAiB;YAE9B;YACA,iBAAiBf,aAAa,MAAM,CAAC,UAAU;YAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;YACrD,aAAazB,QAAQ,WAAW;YAChC,cAAc,CAACsB;gBACbzB,eACE,uDACAyB,iBAAiBE,QAAQF,MAAM,OAAO,GAAGuB,OAAOvB;YAEpD;QACF;IACF,EAAE,OAAO6B,WAAW;QAClB,IAAIA,qBAAqBD,sBACvB,OAAO;YACL,aAAaC,UAAU,WAAW;YAClC,oBAAoBA,UAAU,kBAAkB;YAChD,OAAOA,UAAU,KAAK;YACtB,kBAAkBA,UAAU,gBAAgB;YAC5C,QAAQ;gBAACA,UAAU,OAAO;aAAC;QAC7B;QAGF,MAAMP,eACJO,qBAAqB3B,QAAQ2B,UAAU,OAAO,GAAGN,OAAOM;QAC1D,OAAO;YACL,aAAaP;YACb,QAAQ;gBAAC,CAAC,eAAe,EAAEA,cAAc;aAAC;QAC5C;IACF;AACF"}

package/dist/es/ai-model/workflows/grounding/section-locate.mjs

@@ -2,7 +2,7 @@ import { getDebug } from "@midscene/shared/logger";
 import { assert } from "@midscene/shared/utils";
 import { userPromptToString } from "../../../common.mjs";
 import { buildSearchAreaLocateSystemPrompt } from "../../prompt/locate/index.mjs";
-import { AIResponseParseError, callAI } from "../../service-caller/index.mjs";
+import { AIResponseParseError, callGenerativeAI } from "../../service-caller/index.mjs";
 import { callAiAndParseWithRetry, withSemanticRetryFeedback } from "../../service-caller/semantic-retry.mjs";
 import { buildSearchAreaConfig, expandSearchArea, mergeSearchAreaResults } from "./search-area.mjs";
 import { formatLocateModelContext, prepareLocateModelInput } from "./utils.mjs";
@@ -36,7 +36,7 @@ async function AiLocateSection(options) {
     let parsedResult;
     try {
         parsedResult = await callAiAndParseWithRetry({
-            callAi: (retryAttempt, previousParseError)=>callAI(withSemanticRetryFeedback(messages, previousParseError), modelRuntime, {
+            callAi: (retryAttempt, previousParseError)=>callGenerativeAI(withSemanticRetryFeedback(messages, previousParseError), modelRuntime, {
                     abortSignal: options.abortSignal,
                     expectedJsonObjectResponse: searchAreaProtocol.expectedJsonObjectResponse,
                     semanticRetryAttempt: retryAttempt

package/dist/es/ai-model/workflows/grounding/section-locate.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/workflows/grounding/section-locate.mjs","sources":["../../../../../src/ai-model/workflows/grounding/section-locate.ts"],"sourcesContent":["import type { RawAssistantOutput } from '@/types';\nimport type { AIUsageInfo, Rect, UIContext } from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TUserPrompt } from '../../../common';\nimport { userPromptToString } from '../../../common';\nimport type { ModelRuntime } from '../../models';\nimport { buildSearchAreaLocateSystemPrompt } from '../../prompt/locate';\nimport { AIResponseParseError, callAI } from '../../service-caller/index';\nimport {\n  callAiAndParseWithRetry,\n  withSemanticRetryFeedback,\n} from '../../service-caller/semantic-retry';\nimport {\n  buildSearchAreaConfig,\n  expandSearchArea,\n  mergeSearchAreaResults,\n} from './search-area';\nimport type { SearchAreaConfig } from './types';\nimport { formatLocateModelContext, prepareLocateModelInput } from './utils';\n\nconst debugSection = getDebug('ai:grounding:section');\n\ntype SectionLocateRawResponse = Awaited<ReturnType<typeof callAI>>;\n\nexport async function AiLocateSection(options: {\n  context: UIContext;\n  sectionDescription: TUserPrompt;\n  modelRuntime: ModelRuntime;\n  abortSignal?: AbortSignal;\n}): Promise<{\n  searchAreaConfig?: SearchAreaConfig;\n  error?: string;\n  rawResponse: string;\n  rawAssistantOutput?: RawAssistantOutput;\n  usage?: AIUsageInfo;\n}> {\n  const { context, sectionDescription } = options;\n  const modelRuntime = options.modelRuntime;\n  const { adapter } = modelRuntime;\n  assert(\n    adapter.locate.kind === 'standard',\n    'section locate requires a standard locate adapter',\n  );\n  const searchArea = adapter.locate.searchArea;\n  assert(searchArea, 'section locate requires a search area operation');\n  const { protocol: searchAreaProtocol, resultCodec } = searchArea;\n  const screenshotBase64 = context.screenshot.base64;\n\n  const systemPrompt = buildSearchAreaLocateSystemPrompt({\n    systemPromptIntroduction: searchAreaProtocol.systemPromptIntroduction,\n    responseInstructions: searchAreaProtocol.buildResponseInstructions(\n      resultCodec.promptSpec,\n    ),\n  });\n\n  const userInstructionPrompt = searchAreaProtocol.buildUserPrompt(\n    userPromptToString(sectionDescription),\n  );\n\n  const { messages, preparedImage } = await prepareLocateModelInput({\n    systemPrompt,\n    userPrompt: userInstructionPrompt,\n    locateImage: {\n      imageBase64: screenshotBase64,\n      width: context.shotSize.width,\n      height: context.shotSize.height,\n    },\n    imagePreprocess: adapter.imagePreprocess,\n    targetDescription: sectionDescription,\n    userMessageContentOrder: adapter.locate.userMessageContentOrder,\n  });\n\n  let parsedResult:\n    | {\n        result: SectionLocateRawResponse;\n        sectionError?: string;\n        mergedRect?: undefined;\n      }\n    | {\n        result: SectionLocateRawResponse;\n        sectionError?: string;\n        mergedRect: Rect;\n      };\n\n  try {\n    parsedResult = await callAiAndParseWithRetry({\n      callAi: (retryAttempt, previousParseError) =>\n        callAI(\n          withSemanticRetryFeedback(messages, previousParseError),\n          modelRuntime,\n          {\n            abortSignal: options.abortSignal,\n            expectedJsonObjectResponse:\n              searchAreaProtocol.expectedJsonObjectResponse,\n            semanticRetryAttempt: retryAttempt,\n          },\n        ),\n      parseResponse: (response) => {\n        const parsedLocateResult = searchAreaProtocol.parseRawResponse(\n          response.content,\n          resultCodec.promptSpec,\n        );\n        const sectionError = parsedLocateResult.error;\n        if (parsedLocateResult.kind === 'not-found') {\n          return { result: response, sectionError };\n        }\n\n        try {\n          const locateResultContext = {\n            preparedSize: preparedImage.preparedSize,\n            contentSize: preparedImage.contentSize,\n          };\n          const target = resultCodec.toPixelResult(\n            parsedLocateResult.target,\n            locateResultContext,\n          );\n          const references = parsedLocateResult.references?.map((reference) =>\n            resultCodec.toPixelResult(reference, locateResultContext),\n          );\n          const mergedRect = mergeSearchAreaResults(target, references);\n          debugSection('mergedRect %j', mergedRect);\n          return {\n            result: response,\n            sectionError,\n            mergedRect,\n          };\n        } catch (error) {\n          const errorMessage =\n            error instanceof Error ? error.message : String(error);\n          const message = [\n            sectionError && `error in section locate result: ${sectionError}`,\n            `coordinate parsing error: ${errorMessage}`,\n            formatLocateModelContext(modelRuntime),\n          ]\n            .filter(Boolean)\n            .join('\\n');\n          throw new Error(message, { cause: error });\n        }\n      },\n      toParseError: (error, response) => {\n        const parseErrorMessage =\n          error instanceof Error\n            ? `Failed to parse section locate result: ${error.message}`\n            : 'unknown error in section locate';\n        return new AIResponseParseError(\n          parseErrorMessage,\n          response.content,\n          response.usage,\n          response.rawAssistantOutput,\n          response.reasoning_content,\n        );\n      },\n      parseRetryTimes: modelRuntime.config.retryCount,\n      parseRetryInterval: modelRuntime.config.retryInterval,\n      abortSignal: options.abortSignal,\n      onParseRetry: (error) => {\n        debugSection(\n          'retrying section locate after coordinate parsing failed: %s',\n          error instanceof Error ? error.message : String(error),\n        );\n      },\n    });\n  } catch (callError) {\n    if (callError instanceof AIResponseParseError) {\n      return {\n        searchAreaConfig: undefined,\n        error: callError.message,\n        rawResponse: callError.rawResponse,\n        rawAssistantOutput: callError.rawAssistantOutput,\n        usage: callError.usage,\n      };\n    }\n\n    const errorMessage =\n      callError instanceof Error ? callError.message : String(callError);\n    return {\n      searchAreaConfig: undefined,\n      error: `AI call error: ${errorMessage}`,\n      rawResponse: errorMessage,\n    };\n  }\n\n  const { result, sectionError, mergedRect } = parsedResult;\n  if (!mergedRect) {\n    return {\n      searchAreaConfig: undefined,\n      error: sectionError,\n      rawResponse: result.content,\n      rawAssistantOutput: result.rawAssistantOutput,\n      usage: result.usage,\n    };\n  }\n\n  try {\n    const expandedRect = expandSearchArea(mergedRect, context.shotSize);\n    const originalWidth = expandedRect.width;\n    const originalHeight = expandedRect.height;\n    debugSection('expanded sectionRect %j', expandedRect);\n\n    const searchAreaConfig = await buildSearchAreaConfig({\n      context,\n      baseRect: mergedRect,\n    });\n\n    debugSection(\n      'scaled section image from %dx%d to %dx%d (scale=%d)',\n      originalWidth,\n      originalHeight,\n      searchAreaConfig.image.width,\n      searchAreaConfig.image.height,\n      searchAreaConfig.mapping.scale,\n    );\n    return {\n      searchAreaConfig,\n      error: sectionError,\n      rawResponse: result.content,\n      rawAssistantOutput: result.rawAssistantOutput,\n      usage: result.usage,\n    };\n  } catch (error) {\n    const parseErrorMessage =\n      error instanceof Error\n        ? `Failed to parse section locate result: ${error.message}`\n        : 'unknown error in section locate';\n    const errorMessage = sectionError\n      ? `${sectionError} (${parseErrorMessage})`\n      : parseErrorMessage;\n    return {\n      searchAreaConfig: undefined,\n      error: errorMessage,\n      rawResponse: result.content,\n      rawAssistantOutput: result.rawAssistantOutput,\n      usage: result.usage,\n    };\n  }\n}\n"],"names":["debugSection","getDebug","AiLocateSection","options","context","sectionDescription","modelRuntime","adapter","assert","searchArea","searchAreaProtocol","resultCodec","screenshotBase64","systemPrompt","buildSearchAreaLocateSystemPrompt","userInstructionPrompt","userPromptToString","messages","preparedImage","prepareLocateModelInput","parsedResult","callAiAndParseWithRetry","retryAttempt","previousParseError","callAI","withSemanticRetryFeedback","response","parsedLocateResult","sectionError","locateResultContext","target","references","reference","mergedRect","mergeSearchAreaResults","error","errorMessage","Error","String","message","formatLocateModelContext","Boolean","parseErrorMessage","AIResponseParseError","callError","undefined","result","expandedRect","expandSearchArea","originalWidth","originalHeight","searchAreaConfig","buildSearchAreaConfig"],"mappings":";;;;;;;;AAqBA,MAAMA,eAAeC,SAAS;AAIvB,eAAeC,gBAAgBC,OAKrC;IAOC,MAAM,EAAEC,OAAO,EAAEC,kBAAkB,EAAE,GAAGF;IACxC,MAAMG,eAAeH,QAAQ,YAAY;IACzC,MAAM,EAAEI,OAAO,EAAE,GAAGD;IACpBE,OACED,AAAwB,eAAxBA,QAAQ,MAAM,CAAC,IAAI,EACnB;IAEF,MAAME,aAAaF,QAAQ,MAAM,CAAC,UAAU;IAC5CC,OAAOC,YAAY;IACnB,MAAM,EAAE,UAAUC,kBAAkB,EAAEC,WAAW,EAAE,GAAGF;IACtD,MAAMG,mBAAmBR,QAAQ,UAAU,CAAC,MAAM;IAElD,MAAMS,eAAeC,kCAAkC;QACrD,0BAA0BJ,mBAAmB,wBAAwB;QACrE,sBAAsBA,mBAAmB,yBAAyB,CAChEC,YAAY,UAAU;IAE1B;IAEA,MAAMI,wBAAwBL,mBAAmB,eAAe,CAC9DM,mBAAmBX;IAGrB,MAAM,EAAEY,QAAQ,EAAEC,aAAa,EAAE,GAAG,MAAMC,wBAAwB;QAChEN;QACA,YAAYE;QACZ,aAAa;YACX,aAAaH;YACb,OAAOR,QAAQ,QAAQ,CAAC,KAAK;YAC7B,QAAQA,QAAQ,QAAQ,CAAC,MAAM;QACjC;QACA,iBAAiBG,QAAQ,eAAe;QACxC,mBAAmBF;QACnB,yBAAyBE,QAAQ,MAAM,CAAC,uBAAuB;IACjE;IAEA,IAAIa;IAYJ,IAAI;QACFA,eAAe,MAAMC,wBAAwB;YAC3C,QAAQ,CAACC,cAAcC,qBACrBC,OACEC,0BAA0BR,UAAUM,qBACpCjB,cACA;oBACE,aAAaH,QAAQ,WAAW;oBAChC,4BACEO,mBAAmB,0BAA0B;oBAC/C,sBAAsBY;gBACxB;YAEJ,eAAe,CAACI;gBACd,MAAMC,qBAAqBjB,mBAAmB,gBAAgB,CAC5DgB,SAAS,OAAO,EAChBf,YAAY,UAAU;gBAExB,MAAMiB,eAAeD,mBAAmB,KAAK;gBAC7C,IAAIA,AAA4B,gBAA5BA,mBAAmB,IAAI,EACzB,OAAO;oBAAE,QAAQD;oBAAUE;gBAAa;gBAG1C,IAAI;oBACF,MAAMC,sBAAsB;wBAC1B,cAAcX,cAAc,YAAY;wBACxC,aAAaA,cAAc,WAAW;oBACxC;oBACA,MAAMY,SAASnB,YAAY,aAAa,CACtCgB,mBAAmB,MAAM,EACzBE;oBAEF,MAAME,aAAaJ,mBAAmB,UAAU,EAAE,IAAI,CAACK,YACrDrB,YAAY,aAAa,CAACqB,WAAWH;oBAEvC,MAAMI,aAAaC,uBAAuBJ,QAAQC;oBAClD/B,aAAa,iBAAiBiC;oBAC9B,OAAO;wBACL,QAAQP;wBACRE;wBACAK;oBACF;gBACF,EAAE,OAAOE,OAAO;oBACd,MAAMC,eACJD,iBAAiBE,QAAQF,MAAM,OAAO,GAAGG,OAAOH;oBAClD,MAAMI,UAAU;wBACdX,gBAAgB,CAAC,gCAAgC,EAAEA,cAAc;wBACjE,CAAC,0BAA0B,EAAEQ,cAAc;wBAC3CI,yBAAyBlC;qBAC1B,CACE,MAAM,CAACmC,SACP,IAAI,CAAC;oBACR,MAAM,IAAIJ,MAAME,SAAS;wBAAE,OAAOJ;oBAAM;gBAC1C;YACF;YACA,cAAc,CAACA,OAAOT;gBACpB,MAAMgB,oBACJP,iBAAiBE,QACb,CAAC,uCAAuC,EAAEF,MAAM,OAAO,EAAE,GACzD;gBACN,OAAO,IAAIQ,qBACTD,mBACAhB,SAAS,OAAO,EAChBA,SAAS,KAAK,EACdA,SAAS,kBAAkB,EAC3BA,SAAS,iBAAiB;YAE9B;YACA,iBAAiBpB,aAAa,MAAM,CAAC,UAAU;YAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;YACrD,aAAaH,QAAQ,WAAW;YAChC,cAAc,CAACgC;gBACbnC,aACE,+DACAmC,iBAAiBE,QAAQF,MAAM,OAAO,GAAGG,OAAOH;YAEpD;QACF;IACF,EAAE,OAAOS,WAAW;QAClB,IAAIA,qBAAqBD,sBACvB,OAAO;YACL,kBAAkBE;YAClB,OAAOD,UAAU,OAAO;YACxB,aAAaA,UAAU,WAAW;YAClC,oBAAoBA,UAAU,kBAAkB;YAChD,OAAOA,UAAU,KAAK;QACxB;QAGF,MAAMR,eACJQ,qBAAqBP,QAAQO,UAAU,OAAO,GAAGN,OAAOM;QAC1D,OAAO;YACL,kBAAkBC;YAClB,OAAO,CAAC,eAAe,EAAET,cAAc;YACvC,aAAaA;QACf;IACF;IAEA,MAAM,EAAEU,MAAM,EAAElB,YAAY,EAAEK,UAAU,EAAE,GAAGb;IAC7C,IAAI,CAACa,YACH,OAAO;QACL,kBAAkBY;QAClB,OAAOjB;QACP,aAAakB,OAAO,OAAO;QAC3B,oBAAoBA,OAAO,kBAAkB;QAC7C,OAAOA,OAAO,KAAK;IACrB;IAGF,IAAI;QACF,MAAMC,eAAeC,iBAAiBf,YAAY7B,QAAQ,QAAQ;QAClE,MAAM6C,gBAAgBF,aAAa,KAAK;QACxC,MAAMG,iBAAiBH,aAAa,MAAM;QAC1C/C,aAAa,2BAA2B+C;QAExC,MAAMI,mBAAmB,MAAMC,sBAAsB;YACnDhD;YACA,UAAU6B;QACZ;QAEAjC,aACE,uDACAiD,eACAC,gBACAC,iBAAiB,KAAK,CAAC,KAAK,EAC5BA,iBAAiB,KAAK,CAAC,MAAM,EAC7BA,iBAAiB,OAAO,CAAC,KAAK;QAEhC,OAAO;YACLA;YACA,OAAOvB;YACP,aAAakB,OAAO,OAAO;YAC3B,oBAAoBA,OAAO,kBAAkB;YAC7C,OAAOA,OAAO,KAAK;QACrB;IACF,EAAE,OAAOX,OAAO;QACd,MAAMO,oBACJP,iBAAiBE,QACb,CAAC,uCAAuC,EAAEF,MAAM,OAAO,EAAE,GACzD;QACN,MAAMC,eAAeR,eACjB,GAAGA,aAAa,EAAE,EAAEc,kBAAkB,CAAC,CAAC,GACxCA;QACJ,OAAO;YACL,kBAAkBG;YAClB,OAAOT;YACP,aAAaU,OAAO,OAAO;YAC3B,oBAAoBA,OAAO,kBAAkB;YAC7C,OAAOA,OAAO,KAAK;QACrB;IACF;AACF"}
+{"version":3,"file":"ai-model/workflows/grounding/section-locate.mjs","sources":["../../../../../src/ai-model/workflows/grounding/section-locate.ts"],"sourcesContent":["import type { RawAssistantOutput } from '@/types';\nimport type { AIUsageInfo, Rect, UIContext } from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TUserPrompt } from '../../../common';\nimport { userPromptToString } from '../../../common';\nimport type { ModelRuntime } from '../../models';\nimport { buildSearchAreaLocateSystemPrompt } from '../../prompt/locate';\nimport {\n  AIResponseParseError,\n  callGenerativeAI,\n} from '../../service-caller/index';\nimport {\n  callAiAndParseWithRetry,\n  withSemanticRetryFeedback,\n} from '../../service-caller/semantic-retry';\nimport {\n  buildSearchAreaConfig,\n  expandSearchArea,\n  mergeSearchAreaResults,\n} from './search-area';\nimport type { SearchAreaConfig } from './types';\nimport { formatLocateModelContext, prepareLocateModelInput } from './utils';\n\nconst debugSection = getDebug('ai:grounding:section');\n\ntype SectionLocateRawResponse = Awaited<ReturnType<typeof callGenerativeAI>>;\n\nexport async function AiLocateSection(options: {\n  context: UIContext;\n  sectionDescription: TUserPrompt;\n  modelRuntime: ModelRuntime;\n  abortSignal?: AbortSignal;\n}): Promise<{\n  searchAreaConfig?: SearchAreaConfig;\n  error?: string;\n  rawResponse: string;\n  rawAssistantOutput?: RawAssistantOutput;\n  usage?: AIUsageInfo;\n}> {\n  const { context, sectionDescription } = options;\n  const modelRuntime = options.modelRuntime;\n  const { adapter } = modelRuntime;\n  assert(\n    adapter.locate.kind === 'standard',\n    'section locate requires a standard locate adapter',\n  );\n  const searchArea = adapter.locate.searchArea;\n  assert(searchArea, 'section locate requires a search area operation');\n  const { protocol: searchAreaProtocol, resultCodec } = searchArea;\n  const screenshotBase64 = context.screenshot.base64;\n\n  const systemPrompt = buildSearchAreaLocateSystemPrompt({\n    systemPromptIntroduction: searchAreaProtocol.systemPromptIntroduction,\n    responseInstructions: searchAreaProtocol.buildResponseInstructions(\n      resultCodec.promptSpec,\n    ),\n  });\n\n  const userInstructionPrompt = searchAreaProtocol.buildUserPrompt(\n    userPromptToString(sectionDescription),\n  );\n\n  const { messages, preparedImage } = await prepareLocateModelInput({\n    systemPrompt,\n    userPrompt: userInstructionPrompt,\n    locateImage: {\n      imageBase64: screenshotBase64,\n      width: context.shotSize.width,\n      height: context.shotSize.height,\n    },\n    imagePreprocess: adapter.imagePreprocess,\n    targetDescription: sectionDescription,\n    userMessageContentOrder: adapter.locate.userMessageContentOrder,\n  });\n\n  let parsedResult:\n    | {\n        result: SectionLocateRawResponse;\n        sectionError?: string;\n        mergedRect?: undefined;\n      }\n    | {\n        result: SectionLocateRawResponse;\n        sectionError?: string;\n        mergedRect: Rect;\n      };\n\n  try {\n    parsedResult = await callAiAndParseWithRetry({\n      callAi: (retryAttempt, previousParseError) =>\n        callGenerativeAI(\n          withSemanticRetryFeedback(messages, previousParseError),\n          modelRuntime,\n          {\n            abortSignal: options.abortSignal,\n            expectedJsonObjectResponse:\n              searchAreaProtocol.expectedJsonObjectResponse,\n            semanticRetryAttempt: retryAttempt,\n          },\n        ),\n      parseResponse: (response) => {\n        const parsedLocateResult = searchAreaProtocol.parseRawResponse(\n          response.content,\n          resultCodec.promptSpec,\n        );\n        const sectionError = parsedLocateResult.error;\n        if (parsedLocateResult.kind === 'not-found') {\n          return { result: response, sectionError };\n        }\n\n        try {\n          const locateResultContext = {\n            preparedSize: preparedImage.preparedSize,\n            contentSize: preparedImage.contentSize,\n          };\n          const target = resultCodec.toPixelResult(\n            parsedLocateResult.target,\n            locateResultContext,\n          );\n          const references = parsedLocateResult.references?.map((reference) =>\n            resultCodec.toPixelResult(reference, locateResultContext),\n          );\n          const mergedRect = mergeSearchAreaResults(target, references);\n          debugSection('mergedRect %j', mergedRect);\n          return {\n            result: response,\n            sectionError,\n            mergedRect,\n          };\n        } catch (error) {\n          const errorMessage =\n            error instanceof Error ? error.message : String(error);\n          const message = [\n            sectionError && `error in section locate result: ${sectionError}`,\n            `coordinate parsing error: ${errorMessage}`,\n            formatLocateModelContext(modelRuntime),\n          ]\n            .filter(Boolean)\n            .join('\\n');\n          throw new Error(message, { cause: error });\n        }\n      },\n      toParseError: (error, response) => {\n        const parseErrorMessage =\n          error instanceof Error\n            ? `Failed to parse section locate result: ${error.message}`\n            : 'unknown error in section locate';\n        return new AIResponseParseError(\n          parseErrorMessage,\n          response.content,\n          response.usage,\n          response.rawAssistantOutput,\n          response.reasoning_content,\n        );\n      },\n      parseRetryTimes: modelRuntime.config.retryCount,\n      parseRetryInterval: modelRuntime.config.retryInterval,\n      abortSignal: options.abortSignal,\n      onParseRetry: (error) => {\n        debugSection(\n          'retrying section locate after coordinate parsing failed: %s',\n          error instanceof Error ? error.message : String(error),\n        );\n      },\n    });\n  } catch (callError) {\n    if (callError instanceof AIResponseParseError) {\n      return {\n        searchAreaConfig: undefined,\n        error: callError.message,\n        rawResponse: callError.rawResponse,\n        rawAssistantOutput: callError.rawAssistantOutput,\n        usage: callError.usage,\n      };\n    }\n\n    const errorMessage =\n      callError instanceof Error ? callError.message : String(callError);\n    return {\n      searchAreaConfig: undefined,\n      error: `AI call error: ${errorMessage}`,\n      rawResponse: errorMessage,\n    };\n  }\n\n  const { result, sectionError, mergedRect } = parsedResult;\n  if (!mergedRect) {\n    return {\n      searchAreaConfig: undefined,\n      error: sectionError,\n      rawResponse: result.content,\n      rawAssistantOutput: result.rawAssistantOutput,\n      usage: result.usage,\n    };\n  }\n\n  try {\n    const expandedRect = expandSearchArea(mergedRect, context.shotSize);\n    const originalWidth = expandedRect.width;\n    const originalHeight = expandedRect.height;\n    debugSection('expanded sectionRect %j', expandedRect);\n\n    const searchAreaConfig = await buildSearchAreaConfig({\n      context,\n      baseRect: mergedRect,\n    });\n\n    debugSection(\n      'scaled section image from %dx%d to %dx%d (scale=%d)',\n      originalWidth,\n      originalHeight,\n      searchAreaConfig.image.width,\n      searchAreaConfig.image.height,\n      searchAreaConfig.mapping.scale,\n    );\n    return {\n      searchAreaConfig,\n      error: sectionError,\n      rawResponse: result.content,\n      rawAssistantOutput: result.rawAssistantOutput,\n      usage: result.usage,\n    };\n  } catch (error) {\n    const parseErrorMessage =\n      error instanceof Error\n        ? `Failed to parse section locate result: ${error.message}`\n        : 'unknown error in section locate';\n    const errorMessage = sectionError\n      ? `${sectionError} (${parseErrorMessage})`\n      : parseErrorMessage;\n    return {\n      searchAreaConfig: undefined,\n      error: errorMessage,\n      rawResponse: result.content,\n      rawAssistantOutput: result.rawAssistantOutput,\n      usage: result.usage,\n    };\n  }\n}\n"],"names":["debugSection","getDebug","AiLocateSection","options","context","sectionDescription","modelRuntime","adapter","assert","searchArea","searchAreaProtocol","resultCodec","screenshotBase64","systemPrompt","buildSearchAreaLocateSystemPrompt","userInstructionPrompt","userPromptToString","messages","preparedImage","prepareLocateModelInput","parsedResult","callAiAndParseWithRetry","retryAttempt","previousParseError","callGenerativeAI","withSemanticRetryFeedback","response","parsedLocateResult","sectionError","locateResultContext","target","references","reference","mergedRect","mergeSearchAreaResults","error","errorMessage","Error","String","message","formatLocateModelContext","Boolean","parseErrorMessage","AIResponseParseError","callError","undefined","result","expandedRect","expandSearchArea","originalWidth","originalHeight","searchAreaConfig","buildSearchAreaConfig"],"mappings":";;;;;;;;AAwBA,MAAMA,eAAeC,SAAS;AAIvB,eAAeC,gBAAgBC,OAKrC;IAOC,MAAM,EAAEC,OAAO,EAAEC,kBAAkB,EAAE,GAAGF;IACxC,MAAMG,eAAeH,QAAQ,YAAY;IACzC,MAAM,EAAEI,OAAO,EAAE,GAAGD;IACpBE,OACED,AAAwB,eAAxBA,QAAQ,MAAM,CAAC,IAAI,EACnB;IAEF,MAAME,aAAaF,QAAQ,MAAM,CAAC,UAAU;IAC5CC,OAAOC,YAAY;IACnB,MAAM,EAAE,UAAUC,kBAAkB,EAAEC,WAAW,EAAE,GAAGF;IACtD,MAAMG,mBAAmBR,QAAQ,UAAU,CAAC,MAAM;IAElD,MAAMS,eAAeC,kCAAkC;QACrD,0BAA0BJ,mBAAmB,wBAAwB;QACrE,sBAAsBA,mBAAmB,yBAAyB,CAChEC,YAAY,UAAU;IAE1B;IAEA,MAAMI,wBAAwBL,mBAAmB,eAAe,CAC9DM,mBAAmBX;IAGrB,MAAM,EAAEY,QAAQ,EAAEC,aAAa,EAAE,GAAG,MAAMC,wBAAwB;QAChEN;QACA,YAAYE;QACZ,aAAa;YACX,aAAaH;YACb,OAAOR,QAAQ,QAAQ,CAAC,KAAK;YAC7B,QAAQA,QAAQ,QAAQ,CAAC,MAAM;QACjC;QACA,iBAAiBG,QAAQ,eAAe;QACxC,mBAAmBF;QACnB,yBAAyBE,QAAQ,MAAM,CAAC,uBAAuB;IACjE;IAEA,IAAIa;IAYJ,IAAI;QACFA,eAAe,MAAMC,wBAAwB;YAC3C,QAAQ,CAACC,cAAcC,qBACrBC,iBACEC,0BAA0BR,UAAUM,qBACpCjB,cACA;oBACE,aAAaH,QAAQ,WAAW;oBAChC,4BACEO,mBAAmB,0BAA0B;oBAC/C,sBAAsBY;gBACxB;YAEJ,eAAe,CAACI;gBACd,MAAMC,qBAAqBjB,mBAAmB,gBAAgB,CAC5DgB,SAAS,OAAO,EAChBf,YAAY,UAAU;gBAExB,MAAMiB,eAAeD,mBAAmB,KAAK;gBAC7C,IAAIA,AAA4B,gBAA5BA,mBAAmB,IAAI,EACzB,OAAO;oBAAE,QAAQD;oBAAUE;gBAAa;gBAG1C,IAAI;oBACF,MAAMC,sBAAsB;wBAC1B,cAAcX,cAAc,YAAY;wBACxC,aAAaA,cAAc,WAAW;oBACxC;oBACA,MAAMY,SAASnB,YAAY,aAAa,CACtCgB,mBAAmB,MAAM,EACzBE;oBAEF,MAAME,aAAaJ,mBAAmB,UAAU,EAAE,IAAI,CAACK,YACrDrB,YAAY,aAAa,CAACqB,WAAWH;oBAEvC,MAAMI,aAAaC,uBAAuBJ,QAAQC;oBAClD/B,aAAa,iBAAiBiC;oBAC9B,OAAO;wBACL,QAAQP;wBACRE;wBACAK;oBACF;gBACF,EAAE,OAAOE,OAAO;oBACd,MAAMC,eACJD,iBAAiBE,QAAQF,MAAM,OAAO,GAAGG,OAAOH;oBAClD,MAAMI,UAAU;wBACdX,gBAAgB,CAAC,gCAAgC,EAAEA,cAAc;wBACjE,CAAC,0BAA0B,EAAEQ,cAAc;wBAC3CI,yBAAyBlC;qBAC1B,CACE,MAAM,CAACmC,SACP,IAAI,CAAC;oBACR,MAAM,IAAIJ,MAAME,SAAS;wBAAE,OAAOJ;oBAAM;gBAC1C;YACF;YACA,cAAc,CAACA,OAAOT;gBACpB,MAAMgB,oBACJP,iBAAiBE,QACb,CAAC,uCAAuC,EAAEF,MAAM,OAAO,EAAE,GACzD;gBACN,OAAO,IAAIQ,qBACTD,mBACAhB,SAAS,OAAO,EAChBA,SAAS,KAAK,EACdA,SAAS,kBAAkB,EAC3BA,SAAS,iBAAiB;YAE9B;YACA,iBAAiBpB,aAAa,MAAM,CAAC,UAAU;YAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;YACrD,aAAaH,QAAQ,WAAW;YAChC,cAAc,CAACgC;gBACbnC,aACE,+DACAmC,iBAAiBE,QAAQF,MAAM,OAAO,GAAGG,OAAOH;YAEpD;QACF;IACF,EAAE,OAAOS,WAAW;QAClB,IAAIA,qBAAqBD,sBACvB,OAAO;YACL,kBAAkBE;YAClB,OAAOD,UAAU,OAAO;YACxB,aAAaA,UAAU,WAAW;YAClC,oBAAoBA,UAAU,kBAAkB;YAChD,OAAOA,UAAU,KAAK;QACxB;QAGF,MAAMR,eACJQ,qBAAqBP,QAAQO,UAAU,OAAO,GAAGN,OAAOM;QAC1D,OAAO;YACL,kBAAkBC;YAClB,OAAO,CAAC,eAAe,EAAET,cAAc;YACvC,aAAaA;QACf;IACF;IAEA,MAAM,EAAEU,MAAM,EAAElB,YAAY,EAAEK,UAAU,EAAE,GAAGb;IAC7C,IAAI,CAACa,YACH,OAAO;QACL,kBAAkBY;QAClB,OAAOjB;QACP,aAAakB,OAAO,OAAO;QAC3B,oBAAoBA,OAAO,kBAAkB;QAC7C,OAAOA,OAAO,KAAK;IACrB;IAGF,IAAI;QACF,MAAMC,eAAeC,iBAAiBf,YAAY7B,QAAQ,QAAQ;QAClE,MAAM6C,gBAAgBF,aAAa,KAAK;QACxC,MAAMG,iBAAiBH,aAAa,MAAM;QAC1C/C,aAAa,2BAA2B+C;QAExC,MAAMI,mBAAmB,MAAMC,sBAAsB;YACnDhD;YACA,UAAU6B;QACZ;QAEAjC,aACE,uDACAiD,eACAC,gBACAC,iBAAiB,KAAK,CAAC,KAAK,EAC5BA,iBAAiB,KAAK,CAAC,MAAM,EAC7BA,iBAAiB,OAAO,CAAC,KAAK;QAEhC,OAAO;YACLA;YACA,OAAOvB;YACP,aAAakB,OAAO,OAAO;YAC3B,oBAAoBA,OAAO,kBAAkB;YAC7C,OAAOA,OAAO,KAAK;QACrB;IACF,EAAE,OAAOX,OAAO;QACd,MAAMO,oBACJP,iBAAiBE,QACb,CAAC,uCAAuC,EAAEF,MAAM,OAAO,EAAE,GACzD;QACN,MAAMC,eAAeR,eACjB,GAAGA,aAAa,EAAE,EAAEc,kBAAkB,CAAC,CAAC,GACxCA;QACJ,OAAO;YACL,kBAAkBG;YAClB,OAAOT;YACP,aAAaU,OAAO,OAAO;YAC3B,oBAAoBA,OAAO,kBAAkB;YAC7C,OAAOA,OAAO,KAAK;QACrB;IACF;AACF"}

package/dist/es/ai-model/workflows/insight/boolean.mjs

@@ -3,7 +3,7 @@ async function extractBoolean(options) {
     const { booleanFn } = options.modelRuntime.adapter.insight;
     assert(booleanFn, 'A boolean adapter is required');
     const keys = 'object' == typeof options.dataQuery ? Object.keys(options.dataQuery) : [];
-    assert(1 === keys.length && ('Boolean' === keys[0] || 'StatementIsTruthy' === keys[0]), 'This model only supports boolean insight tasks. Use the default model for extraction.');
+    assert(1 === keys.length, 'Boolean insight tasks require exactly one result field.');
     const result = await booleanFn(options);
     return {
         parseResult: {

package/dist/es/ai-model/workflows/insight/boolean.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/workflows/insight/boolean.mjs","sources":["../../../../../src/ai-model/workflows/insight/boolean.ts"],"sourcesContent":["import type {\n  AIDataExtractionResponse,\n  AIUsageInfo,\n  RawAssistantOutput,\n} from '@/types';\nimport { assert } from '@midscene/shared/utils';\nimport type { ExtractOptions } from './extraction';\n\nexport type BooleanFn = (options: ExtractOptions) => Promise<{\n  value: boolean;\n  rawResponse: string;\n  usage?: AIUsageInfo;\n}>;\n\nexport async function extractBoolean<T>(options: ExtractOptions): Promise<{\n  parseResult: AIDataExtractionResponse<T>;\n  rawResponse: string;\n  usage?: AIUsageInfo;\n  rawAssistantOutput?: RawAssistantOutput;\n  reasoning_content?: string;\n}> {\n  const { booleanFn } = options.modelRuntime.adapter.insight;\n  assert(booleanFn, 'A boolean adapter is required');\n  const keys =\n    typeof options.dataQuery === 'object' ? Object.keys(options.dataQuery) : [];\n  assert(\n    keys.length === 1 &&\n      (keys[0] === 'Boolean' || keys[0] === 'StatementIsTruthy'),\n    'This model only supports boolean insight tasks. Use the default model for extraction.',\n  );\n  const result = await booleanFn(options);\n  return {\n    parseResult: {\n      data: { [keys[0]]: result.value } as T,\n    },\n    rawResponse: result.rawResponse,\n    usage: result.usage,\n  };\n}\n"],"names":["extractBoolean","options","booleanFn","assert","keys","Object","result"],"mappings":";AAcO,eAAeA,eAAkBC,OAAuB;IAO7D,MAAM,EAAEC,SAAS,EAAE,GAAGD,QAAQ,YAAY,CAAC,OAAO,CAAC,OAAO;IAC1DE,OAAOD,WAAW;IAClB,MAAME,OACJ,AAA6B,YAA7B,OAAOH,QAAQ,SAAS,GAAgBI,OAAO,IAAI,CAACJ,QAAQ,SAAS,IAAI,EAAE;IAC7EE,OACEC,AAAgB,MAAhBA,KAAK,MAAM,IACRA,CAAAA,AAAY,cAAZA,IAAI,CAAC,EAAE,IAAkBA,AAAY,wBAAZA,IAAI,CAAC,EAAE,AAAuB,GAC1D;IAEF,MAAME,SAAS,MAAMJ,UAAUD;IAC/B,OAAO;QACL,aAAa;YACX,MAAM;gBAAE,CAACG,IAAI,CAAC,EAAE,CAAC,EAAEE,OAAO,KAAK;YAAC;QAClC;QACA,aAAaA,OAAO,WAAW;QAC/B,OAAOA,OAAO,KAAK;IACrB;AACF"}
+{"version":3,"file":"ai-model/workflows/insight/boolean.mjs","sources":["../../../../../src/ai-model/workflows/insight/boolean.ts"],"sourcesContent":["import type {\n  AIDataExtractionResponse,\n  AIUsageInfo,\n  RawAssistantOutput,\n} from '@/types';\nimport { assert } from '@midscene/shared/utils';\nimport type { ExtractOptions } from './extraction';\n\nexport type BooleanFn = (options: ExtractOptions) => Promise<{\n  value: boolean;\n  rawResponse: string;\n  usage?: AIUsageInfo;\n}>;\n\nexport async function extractBoolean<T>(options: ExtractOptions): Promise<{\n  parseResult: AIDataExtractionResponse<T>;\n  rawResponse: string;\n  usage?: AIUsageInfo;\n  rawAssistantOutput?: RawAssistantOutput;\n  reasoning_content?: string;\n}> {\n  const { booleanFn } = options.modelRuntime.adapter.insight;\n  assert(booleanFn, 'A boolean adapter is required');\n  const keys =\n    typeof options.dataQuery === 'object' ? Object.keys(options.dataQuery) : [];\n  assert(\n    keys.length === 1,\n    'Boolean insight tasks require exactly one result field.',\n  );\n  const result = await booleanFn(options);\n  return {\n    parseResult: {\n      data: { [keys[0]]: result.value } as T,\n    },\n    rawResponse: result.rawResponse,\n    usage: result.usage,\n  };\n}\n"],"names":["extractBoolean","options","booleanFn","assert","keys","Object","result"],"mappings":";AAcO,eAAeA,eAAkBC,OAAuB;IAO7D,MAAM,EAAEC,SAAS,EAAE,GAAGD,QAAQ,YAAY,CAAC,OAAO,CAAC,OAAO;IAC1DE,OAAOD,WAAW;IAClB,MAAME,OACJ,AAA6B,YAA7B,OAAOH,QAAQ,SAAS,GAAgBI,OAAO,IAAI,CAACJ,QAAQ,SAAS,IAAI,EAAE;IAC7EE,OACEC,AAAgB,MAAhBA,KAAK,MAAM,EACX;IAEF,MAAME,SAAS,MAAMJ,UAAUD;IAC/B,OAAO;QACL,aAAa;YACX,MAAM;gBAAE,CAACG,IAAI,CAAC,EAAE,CAAC,EAAEE,OAAO,KAAK;YAAC;QAClC;QACA,aAAaA,OAAO,WAAW;QAC/B,OAAOA,OAAO,KAAK;IACrB;AACF"}

package/dist/es/ai-model/workflows/insight/extraction.mjs

@@ -1,14 +1,14 @@
 import { getDebug } from "@midscene/shared/logger";
 import { renderAIContext } from "../../../agent/prompt-context.mjs";
 import { buildInsightSystemPrompt, extractDataQueryPrompt } from "../../prompt/insight/index.mjs";
-import { AIResponseParseError, callAI } from "../../service-caller/index.mjs";
+import { AIResponseParseError, callGenerativeAI } from "../../service-caller/index.mjs";
 import { callAiAndParseWithRetry, withSemanticRetryFeedback } from "../../service-caller/semantic-retry.mjs";
 import { multimodalPromptToMessages } from "../../shared/multimodal-prompt.mjs";
 import { extractBoolean } from "./boolean.mjs";
 import { parseInsightResponse } from "./insight-response-parser.mjs";
 const debugInsight = getDebug('ai:insight');
 async function AiExtractElementInfo(options) {
-    if (options.modelRuntime.adapter.insight.booleanFn) return extractBoolean(options);
+    if ('boolean' === options.taskType && options.modelRuntime.adapter.insight.booleanFn) return extractBoolean(options);
     const { dataQuery, context, extractOption, multimodalPrompt, modelRuntime } = options;
     const insightProtocol = modelRuntime.adapter.insight.protocol;
     const systemPrompt = buildInsightSystemPrompt({
@@ -67,7 +67,7 @@ async function AiExtractElementInfo(options) {
         msgs.push(...addOns);
     }
     return callAiAndParseWithRetry({
-        callAi: (retryAttempt, previousParseError)=>callAI(withSemanticRetryFeedback(msgs, previousParseError), modelRuntime, {
+        callAi: (retryAttempt, previousParseError)=>callGenerativeAI(withSemanticRetryFeedback(msgs, previousParseError), modelRuntime, {
                 abortSignal: options.abortSignal,
                 semanticRetryAttempt: retryAttempt
             }),

package/dist/es/ai-model/workflows/insight/extraction.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/workflows/insight/extraction.mjs","sources":["../../../../../src/ai-model/workflows/insight/extraction.ts"],"sourcesContent":["import type {\n  ConversationSystemMessage,\n  ConversationUserMessage,\n} from '@/ai-model/service-caller/types';\nimport type { ServiceExtractOption, UIContext } from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { renderAIContext } from '../../../agent/prompt-context';\nimport type { TMultimodalPrompt } from '../../../common';\nimport type { ModelRuntime } from '../../models';\nimport {\n  buildInsightSystemPrompt,\n  extractDataQueryPrompt,\n} from '../../prompt/insight';\nimport { AIResponseParseError, callAI } from '../../service-caller/index';\nimport {\n  callAiAndParseWithRetry,\n  withSemanticRetryFeedback,\n} from '../../service-caller/semantic-retry';\nimport { multimodalPromptToMessages } from '../../shared/multimodal-prompt';\nimport { extractBoolean } from './boolean';\nimport { parseInsightResponse } from './insight-response-parser';\n\ntype InsightAIArgs = [ConversationSystemMessage, ...ConversationUserMessage[]];\n\nconst debugInsight = getDebug('ai:insight');\n\nexport interface ExtractOptions {\n  dataQuery: string | Record<string, string>;\n  multimodalPrompt?: TMultimodalPrompt;\n  context: UIContext;\n  pageDescription?: string;\n  extractOption?: ServiceExtractOption;\n  modelRuntime: ModelRuntime;\n  abortSignal?: AbortSignal;\n}\n\nexport async function AiExtractElementInfo<T>(options: ExtractOptions) {\n  if (options.modelRuntime.adapter.insight.booleanFn) {\n    return extractBoolean<T>(options);\n  }\n  const { dataQuery, context, extractOption, multimodalPrompt, modelRuntime } =\n    options;\n  const insightProtocol = modelRuntime.adapter.insight.protocol;\n  const systemPrompt = buildInsightSystemPrompt({\n    screenshotIncluded: extractOption?.screenshotIncluded !== false,\n    referenceImagesIncluded: !!multimodalPrompt?.images?.length,\n    insightProtocol,\n  });\n  const screenshotBase64 = context.screenshot.base64;\n\n  const renderedContext = renderAIContext(extractOption?.context);\n  const extractDataPromptText = extractDataQueryPrompt(\n    options.pageDescription || '',\n    dataQuery,\n    renderedContext,\n  );\n\n  const userContent: ConversationUserMessage['content'] = [];\n\n  if (extractOption?.screenshotIncluded !== false) {\n    const screenshotSequence = context.screenshotSequence;\n    if (screenshotSequence && screenshotSequence.length > 1) {\n      userContent.push({\n        type: 'text',\n        text: `The following ${screenshotSequence.length} images are consecutive screenshots captured over a time window, ordered from earliest to latest (Frame 1 is first, Frame ${screenshotSequence.length} is last). They record what appeared on screen during that window. Some UI elements such as toasts, banners, or transitions may appear only in certain frames and be gone by later ones. Interpret the temporal scope from the statement or question itself: if it asks whether something appeared at any point, inspect the whole sequence; if it asks about the final or current state, use the relevant later frame; if it asks about a change or sequence, compare frames in order. Unless <DATA_DEMAND> explicitly asks for comparison or matching against reference images, base your answer on these screenshots and their contents.`,\n      });\n\n      screenshotSequence.forEach((frame, index) => {\n        userContent.push({\n          type: 'text',\n          text: `Frame ${index + 1}/${screenshotSequence.length}`,\n        });\n        userContent.push({\n          type: 'image',\n          url: frame.base64,\n        });\n      });\n    } else {\n      userContent.push({\n        type: 'text',\n        text: 'This is the current screenshot to evaluate. Unless <DATA_DEMAND> explicitly asks for comparison or matching against reference images, base your answer on this screenshot and its contents when provided.',\n      });\n\n      userContent.push({\n        type: 'image',\n        url: screenshotBase64,\n      });\n    }\n  }\n\n  userContent.push({\n    type: 'text',\n    text: extractDataPromptText,\n  });\n\n  const msgs: InsightAIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: userContent,\n    },\n  ];\n\n  if (multimodalPrompt) {\n    const addOns = await multimodalPromptToMessages(multimodalPrompt);\n    msgs.push(...addOns);\n  }\n\n  return callAiAndParseWithRetry({\n    callAi: (retryAttempt, previousParseError) =>\n      callAI(\n        withSemanticRetryFeedback(msgs, previousParseError),\n        modelRuntime,\n        {\n          abortSignal: options.abortSignal,\n          semanticRetryAttempt: retryAttempt,\n        },\n      ),\n    parseResponse: (response) => {\n      const {\n        content: rawResponse,\n        usage,\n        reasoning_content,\n        rawAssistantOutput,\n      } = response;\n      const parseResult = parseInsightResponse<T>(\n        rawResponse,\n        insightProtocol.dataOutput,\n        modelRuntime.adapter.jsonParser,\n      );\n      return {\n        parseResult,\n        rawResponse,\n        rawAssistantOutput,\n        usage,\n        reasoning_content,\n      };\n    },\n    toParseError: (parseError, response) => {\n      const errorMessage =\n        parseError instanceof Error ? parseError.message : String(parseError);\n      return new AIResponseParseError(\n        `XML parse error: ${errorMessage}`,\n        response.content,\n        response.usage,\n        response.rawAssistantOutput,\n      );\n    },\n    parseRetryTimes: modelRuntime.config.retryCount,\n    parseRetryInterval: modelRuntime.config.retryInterval,\n    abortSignal: options.abortSignal,\n    onParseRetry: (error) => {\n      debugInsight(\n        'retrying insight after XML parsing failed: %s',\n        error instanceof Error ? error.message : String(error),\n      );\n    },\n  });\n}\n"],"names":["debugInsight","getDebug","AiExtractElementInfo","options","extractBoolean","dataQuery","context","extractOption","multimodalPrompt","modelRuntime","insightProtocol","systemPrompt","buildInsightSystemPrompt","screenshotBase64","renderedContext","renderAIContext","extractDataPromptText","extractDataQueryPrompt","userContent","screenshotSequence","frame","index","msgs","addOns","multimodalPromptToMessages","callAiAndParseWithRetry","retryAttempt","previousParseError","callAI","withSemanticRetryFeedback","response","rawResponse","usage","reasoning_content","rawAssistantOutput","parseResult","parseInsightResponse","parseError","errorMessage","Error","String","AIResponseParseError","error"],"mappings":";;;;;;;;AAwBA,MAAMA,eAAeC,SAAS;AAYvB,eAAeC,qBAAwBC,OAAuB;IACnE,IAAIA,QAAQ,YAAY,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,EAChD,OAAOC,eAAkBD;IAE3B,MAAM,EAAEE,SAAS,EAAEC,OAAO,EAAEC,aAAa,EAAEC,gBAAgB,EAAEC,YAAY,EAAE,GACzEN;IACF,MAAMO,kBAAkBD,aAAa,OAAO,CAAC,OAAO,CAAC,QAAQ;IAC7D,MAAME,eAAeC,yBAAyB;QAC5C,oBAAoBL,eAAe,uBAAuB;QAC1D,yBAAyB,CAAC,CAACC,kBAAkB,QAAQ;QACrDE;IACF;IACA,MAAMG,mBAAmBP,QAAQ,UAAU,CAAC,MAAM;IAElD,MAAMQ,kBAAkBC,gBAAgBR,eAAe;IACvD,MAAMS,wBAAwBC,uBAC5Bd,QAAQ,eAAe,IAAI,IAC3BE,WACAS;IAGF,MAAMI,cAAkD,EAAE;IAE1D,IAAIX,eAAe,uBAAuB,OAAO;QAC/C,MAAMY,qBAAqBb,QAAQ,kBAAkB;QACrD,IAAIa,sBAAsBA,mBAAmB,MAAM,GAAG,GAAG;YACvDD,YAAY,IAAI,CAAC;gBACf,MAAM;gBACN,MAAM,CAAC,cAAc,EAAEC,mBAAmB,MAAM,CAAC,0HAA0H,EAAEA,mBAAmB,MAAM,CAAC,2mBAA2mB,CAAC;YACrzB;YAEAA,mBAAmB,OAAO,CAAC,CAACC,OAAOC;gBACjCH,YAAY,IAAI,CAAC;oBACf,MAAM;oBACN,MAAM,CAAC,MAAM,EAAEG,QAAQ,EAAE,CAAC,EAAEF,mBAAmB,MAAM,EAAE;gBACzD;gBACAD,YAAY,IAAI,CAAC;oBACf,MAAM;oBACN,KAAKE,MAAM,MAAM;gBACnB;YACF;QACF,OAAO;YACLF,YAAY,IAAI,CAAC;gBACf,MAAM;gBACN,MAAM;YACR;YAEAA,YAAY,IAAI,CAAC;gBACf,MAAM;gBACN,KAAKL;YACP;QACF;IACF;IAEAK,YAAY,IAAI,CAAC;QACf,MAAM;QACN,MAAMF;IACR;IAEA,MAAMM,OAAsB;QAC1B;YAAE,MAAM;YAAU,SAASX;QAAa;QACxC;YACE,MAAM;YACN,SAASO;QACX;KACD;IAED,IAAIV,kBAAkB;QACpB,MAAMe,SAAS,MAAMC,2BAA2BhB;QAChDc,KAAK,IAAI,IAAIC;IACf;IAEA,OAAOE,wBAAwB;QAC7B,QAAQ,CAACC,cAAcC,qBACrBC,OACEC,0BAA0BP,MAAMK,qBAChClB,cACA;gBACE,aAAaN,QAAQ,WAAW;gBAChC,sBAAsBuB;YACxB;QAEJ,eAAe,CAACI;YACd,MAAM,EACJ,SAASC,WAAW,EACpBC,KAAK,EACLC,iBAAiB,EACjBC,kBAAkB,EACnB,GAAGJ;YACJ,MAAMK,cAAcC,qBAClBL,aACArB,gBAAgB,UAAU,EAC1BD,aAAa,OAAO,CAAC,UAAU;YAEjC,OAAO;gBACL0B;gBACAJ;gBACAG;gBACAF;gBACAC;YACF;QACF;QACA,cAAc,CAACI,YAAYP;YACzB,MAAMQ,eACJD,sBAAsBE,QAAQF,WAAW,OAAO,GAAGG,OAAOH;YAC5D,OAAO,IAAII,qBACT,CAAC,iBAAiB,EAAEH,cAAc,EAClCR,SAAS,OAAO,EAChBA,SAAS,KAAK,EACdA,SAAS,kBAAkB;QAE/B;QACA,iBAAiBrB,aAAa,MAAM,CAAC,UAAU;QAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;QACrD,aAAaN,QAAQ,WAAW;QAChC,cAAc,CAACuC;YACb1C,aACE,iDACA0C,iBAAiBH,QAAQG,MAAM,OAAO,GAAGF,OAAOE;QAEpD;IACF;AACF"}
+{"version":3,"file":"ai-model/workflows/insight/extraction.mjs","sources":["../../../../../src/ai-model/workflows/insight/extraction.ts"],"sourcesContent":["import type {\n  ConversationSystemMessage,\n  ConversationUserMessage,\n} from '@/ai-model/service-caller/types';\nimport type { ServiceExtractOption, UIContext } from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { renderAIContext } from '../../../agent/prompt-context';\nimport type { TMultimodalPrompt } from '../../../common';\nimport type { ModelRuntime } from '../../models';\nimport {\n  buildInsightSystemPrompt,\n  extractDataQueryPrompt,\n} from '../../prompt/insight';\nimport {\n  AIResponseParseError,\n  callGenerativeAI,\n} from '../../service-caller/index';\nimport {\n  callAiAndParseWithRetry,\n  withSemanticRetryFeedback,\n} from '../../service-caller/semantic-retry';\nimport { multimodalPromptToMessages } from '../../shared/multimodal-prompt';\nimport { extractBoolean } from './boolean';\nimport { parseInsightResponse } from './insight-response-parser';\n\ntype InsightAIArgs = [ConversationSystemMessage, ...ConversationUserMessage[]];\n\nconst debugInsight = getDebug('ai:insight');\n\nexport interface ExtractOptions {\n  /** Explicit task type; never infer from query keys. */\n  taskType: 'boolean' | 'extract';\n  dataQuery: string | Record<string, string>;\n  multimodalPrompt?: TMultimodalPrompt;\n  context: UIContext;\n  pageDescription?: string;\n  extractOption?: ServiceExtractOption;\n  modelRuntime: ModelRuntime;\n  abortSignal?: AbortSignal;\n}\n\nexport async function AiExtractElementInfo<T>(options: ExtractOptions) {\n  if (\n    options.taskType === 'boolean' &&\n    options.modelRuntime.adapter.insight.booleanFn\n  ) {\n    return extractBoolean<T>(options);\n  }\n  const { dataQuery, context, extractOption, multimodalPrompt, modelRuntime } =\n    options;\n  const insightProtocol = modelRuntime.adapter.insight.protocol;\n  const systemPrompt = buildInsightSystemPrompt({\n    screenshotIncluded: extractOption?.screenshotIncluded !== false,\n    referenceImagesIncluded: !!multimodalPrompt?.images?.length,\n    insightProtocol,\n  });\n  const screenshotBase64 = context.screenshot.base64;\n\n  const renderedContext = renderAIContext(extractOption?.context);\n  const extractDataPromptText = extractDataQueryPrompt(\n    options.pageDescription || '',\n    dataQuery,\n    renderedContext,\n  );\n\n  const userContent: ConversationUserMessage['content'] = [];\n\n  if (extractOption?.screenshotIncluded !== false) {\n    const screenshotSequence = context.screenshotSequence;\n    if (screenshotSequence && screenshotSequence.length > 1) {\n      userContent.push({\n        type: 'text',\n        text: `The following ${screenshotSequence.length} images are consecutive screenshots captured over a time window, ordered from earliest to latest (Frame 1 is first, Frame ${screenshotSequence.length} is last). They record what appeared on screen during that window. Some UI elements such as toasts, banners, or transitions may appear only in certain frames and be gone by later ones. Interpret the temporal scope from the statement or question itself: if it asks whether something appeared at any point, inspect the whole sequence; if it asks about the final or current state, use the relevant later frame; if it asks about a change or sequence, compare frames in order. Unless <DATA_DEMAND> explicitly asks for comparison or matching against reference images, base your answer on these screenshots and their contents.`,\n      });\n\n      screenshotSequence.forEach((frame, index) => {\n        userContent.push({\n          type: 'text',\n          text: `Frame ${index + 1}/${screenshotSequence.length}`,\n        });\n        userContent.push({\n          type: 'image',\n          url: frame.base64,\n        });\n      });\n    } else {\n      userContent.push({\n        type: 'text',\n        text: 'This is the current screenshot to evaluate. Unless <DATA_DEMAND> explicitly asks for comparison or matching against reference images, base your answer on this screenshot and its contents when provided.',\n      });\n\n      userContent.push({\n        type: 'image',\n        url: screenshotBase64,\n      });\n    }\n  }\n\n  userContent.push({\n    type: 'text',\n    text: extractDataPromptText,\n  });\n\n  const msgs: InsightAIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: userContent,\n    },\n  ];\n\n  if (multimodalPrompt) {\n    const addOns = await multimodalPromptToMessages(multimodalPrompt);\n    msgs.push(...addOns);\n  }\n\n  return callAiAndParseWithRetry({\n    callAi: (retryAttempt, previousParseError) =>\n      callGenerativeAI(\n        withSemanticRetryFeedback(msgs, previousParseError),\n        modelRuntime,\n        {\n          abortSignal: options.abortSignal,\n          semanticRetryAttempt: retryAttempt,\n        },\n      ),\n    parseResponse: (response) => {\n      const {\n        content: rawResponse,\n        usage,\n        reasoning_content,\n        rawAssistantOutput,\n      } = response;\n      const parseResult = parseInsightResponse<T>(\n        rawResponse,\n        insightProtocol.dataOutput,\n        modelRuntime.adapter.jsonParser,\n      );\n      return {\n        parseResult,\n        rawResponse,\n        rawAssistantOutput,\n        usage,\n        reasoning_content,\n      };\n    },\n    toParseError: (parseError, response) => {\n      const errorMessage =\n        parseError instanceof Error ? parseError.message : String(parseError);\n      return new AIResponseParseError(\n        `XML parse error: ${errorMessage}`,\n        response.content,\n        response.usage,\n        response.rawAssistantOutput,\n      );\n    },\n    parseRetryTimes: modelRuntime.config.retryCount,\n    parseRetryInterval: modelRuntime.config.retryInterval,\n    abortSignal: options.abortSignal,\n    onParseRetry: (error) => {\n      debugInsight(\n        'retrying insight after XML parsing failed: %s',\n        error instanceof Error ? error.message : String(error),\n      );\n    },\n  });\n}\n"],"names":["debugInsight","getDebug","AiExtractElementInfo","options","extractBoolean","dataQuery","context","extractOption","multimodalPrompt","modelRuntime","insightProtocol","systemPrompt","buildInsightSystemPrompt","screenshotBase64","renderedContext","renderAIContext","extractDataPromptText","extractDataQueryPrompt","userContent","screenshotSequence","frame","index","msgs","addOns","multimodalPromptToMessages","callAiAndParseWithRetry","retryAttempt","previousParseError","callGenerativeAI","withSemanticRetryFeedback","response","rawResponse","usage","reasoning_content","rawAssistantOutput","parseResult","parseInsightResponse","parseError","errorMessage","Error","String","AIResponseParseError","error"],"mappings":";;;;;;;;AA2BA,MAAMA,eAAeC,SAAS;AAcvB,eAAeC,qBAAwBC,OAAuB;IACnE,IACEA,AAAqB,cAArBA,QAAQ,QAAQ,IAChBA,QAAQ,YAAY,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,EAE9C,OAAOC,eAAkBD;IAE3B,MAAM,EAAEE,SAAS,EAAEC,OAAO,EAAEC,aAAa,EAAEC,gBAAgB,EAAEC,YAAY,EAAE,GACzEN;IACF,MAAMO,kBAAkBD,aAAa,OAAO,CAAC,OAAO,CAAC,QAAQ;IAC7D,MAAME,eAAeC,yBAAyB;QAC5C,oBAAoBL,eAAe,uBAAuB;QAC1D,yBAAyB,CAAC,CAACC,kBAAkB,QAAQ;QACrDE;IACF;IACA,MAAMG,mBAAmBP,QAAQ,UAAU,CAAC,MAAM;IAElD,MAAMQ,kBAAkBC,gBAAgBR,eAAe;IACvD,MAAMS,wBAAwBC,uBAC5Bd,QAAQ,eAAe,IAAI,IAC3BE,WACAS;IAGF,MAAMI,cAAkD,EAAE;IAE1D,IAAIX,eAAe,uBAAuB,OAAO;QAC/C,MAAMY,qBAAqBb,QAAQ,kBAAkB;QACrD,IAAIa,sBAAsBA,mBAAmB,MAAM,GAAG,GAAG;YACvDD,YAAY,IAAI,CAAC;gBACf,MAAM;gBACN,MAAM,CAAC,cAAc,EAAEC,mBAAmB,MAAM,CAAC,0HAA0H,EAAEA,mBAAmB,MAAM,CAAC,2mBAA2mB,CAAC;YACrzB;YAEAA,mBAAmB,OAAO,CAAC,CAACC,OAAOC;gBACjCH,YAAY,IAAI,CAAC;oBACf,MAAM;oBACN,MAAM,CAAC,MAAM,EAAEG,QAAQ,EAAE,CAAC,EAAEF,mBAAmB,MAAM,EAAE;gBACzD;gBACAD,YAAY,IAAI,CAAC;oBACf,MAAM;oBACN,KAAKE,MAAM,MAAM;gBACnB;YACF;QACF,OAAO;YACLF,YAAY,IAAI,CAAC;gBACf,MAAM;gBACN,MAAM;YACR;YAEAA,YAAY,IAAI,CAAC;gBACf,MAAM;gBACN,KAAKL;YACP;QACF;IACF;IAEAK,YAAY,IAAI,CAAC;QACf,MAAM;QACN,MAAMF;IACR;IAEA,MAAMM,OAAsB;QAC1B;YAAE,MAAM;YAAU,SAASX;QAAa;QACxC;YACE,MAAM;YACN,SAASO;QACX;KACD;IAED,IAAIV,kBAAkB;QACpB,MAAMe,SAAS,MAAMC,2BAA2BhB;QAChDc,KAAK,IAAI,IAAIC;IACf;IAEA,OAAOE,wBAAwB;QAC7B,QAAQ,CAACC,cAAcC,qBACrBC,iBACEC,0BAA0BP,MAAMK,qBAChClB,cACA;gBACE,aAAaN,QAAQ,WAAW;gBAChC,sBAAsBuB;YACxB;QAEJ,eAAe,CAACI;YACd,MAAM,EACJ,SAASC,WAAW,EACpBC,KAAK,EACLC,iBAAiB,EACjBC,kBAAkB,EACnB,GAAGJ;YACJ,MAAMK,cAAcC,qBAClBL,aACArB,gBAAgB,UAAU,EAC1BD,aAAa,OAAO,CAAC,UAAU;YAEjC,OAAO;gBACL0B;gBACAJ;gBACAG;gBACAF;gBACAC;YACF;QACF;QACA,cAAc,CAACI,YAAYP;YACzB,MAAMQ,eACJD,sBAAsBE,QAAQF,WAAW,OAAO,GAAGG,OAAOH;YAC5D,OAAO,IAAII,qBACT,CAAC,iBAAiB,EAAEH,cAAc,EAClCR,SAAS,OAAO,EAChBA,SAAS,KAAK,EACdA,SAAS,kBAAkB;QAE/B;QACA,iBAAiBrB,aAAa,MAAM,CAAC,UAAU;QAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;QACrD,aAAaN,QAAQ,WAAW;QAChC,cAAc,CAACuC;YACb1C,aACE,iDACA0C,iBAAiBH,QAAQG,MAAM,OAAO,GAAGF,OAAOE;QAEpD;IACF;AACF"}

package/dist/es/ai-model/workflows/planning/standard-planning.mjs

@@ -3,7 +3,7 @@ import { assert } from "@midscene/shared/utils";
 import { buildYamlFlowFromPlans } from "../../../common.mjs";
 import { prepareModelImage } from "../../model-adapter/image-preprocess.mjs";
 import { buildStandardPlanningSystemPrompt } from "../../prompt/planning/index.mjs";
-import { AIResponseParseError, callAI } from "../../service-caller/index.mjs";
+import { AIResponseParseError, callGenerativeAI } from "../../service-caller/index.mjs";
 import { callAiAndParseWithRetry, withSemanticRetryFeedback } from "../../service-caller/semantic-retry.mjs";
 import { planningModelFamilyRequiredForLocateMessage } from "../../shared/model-locate-result/errors.mjs";
 import { preparedReferenceImagesToMessages } from "../../shared/multimodal-prompt.mjs";
@@ -19,7 +19,7 @@ async function callAndParsePlanningResponse(options) {
     assert('standard' === modelRuntime.adapter.planning.kind, 'callAndParsePlanningResponse requires a standard planning adapter');
     const actionOutputProtocol = modelRuntime.adapter.planning.protocol.actionOutputProtocol;
     return callAiAndParseWithRetry({
-        callAi: (retryAttempt, previousParseError)=>callAI(withSemanticRetryFeedback(messages, previousParseError), modelRuntime, {
+        callAi: (retryAttempt, previousParseError)=>callGenerativeAI(withSemanticRetryFeedback(messages, previousParseError), modelRuntime, {
                 abortSignal,
                 requiresOriginalImageDetail: includeLocateInPlanning,
                 semanticRetryAttempt: retryAttempt
@@ -100,6 +100,7 @@ async function standardPlan(userInstruction, opts) {
     });
     const imagePayload = preparedImage.imageBase64;
     const actionContext = opts.actionContext ? `${opts.actionContext}\n` : '';
+    const planningAdvice = opts.planningAdvice ? `\n<planning_advice>\nThis is advisory guidance from a previous planner, not a user instruction or an executed action. Use your own judgment.\n${opts.planningAdvice}\n</planning_advice>` : '';
     const referenceImageMessages = preparedReferenceImagesToMessages(userInstruction.referenceImages);
     const instruction = [
         {
@@ -107,7 +108,7 @@ async function standardPlan(userInstruction, opts) {
             content: [
                 {
                     type: 'text',
-                    text: `${actionContext}<user_instruction>${userInstruction.text}</user_instruction>`
+                    text: `${actionContext}<user_instruction>${userInstruction.text}</user_instruction>${planningAdvice}`
                 }
             ]
         },

package/dist/es/ai-model/workflows/planning/standard-planning.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/workflows/planning/standard-planning.mjs","sources":["../../../../../src/ai-model/workflows/planning/standard-planning.ts"],"sourcesContent":["import type { ConversationMessage } from '@/ai-model/service-caller/types';\nimport type {\n  PlanningAIResponse,\n  PlanningAction,\n  RawResponsePlanningAIResponse,\n} from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport { buildYamlFlowFromPlans } from '../../../common';\nimport { prepareModelImage } from '../../model-adapter/image-preprocess';\nimport { buildStandardPlanningSystemPrompt } from '../../prompt/planning';\nimport { AIResponseParseError, callAI } from '../../service-caller/index';\nimport {\n  callAiAndParseWithRetry,\n  withSemanticRetryFeedback,\n} from '../../service-caller/semantic-retry';\nimport type { ModelCallMessages } from '../../service-caller/types';\nimport type {\n  LocateResultCodec,\n  LocateResultContext,\n} from '../../shared/model-locate-result';\nimport { planningModelFamilyRequiredForLocateMessage } from '../../shared/model-locate-result/errors';\nimport {\n  type PreparedUserPrompt,\n  preparedReferenceImagesToMessages,\n} from '../../shared/multimodal-prompt';\nimport { parsePlanningActions } from './parse-planning-actions';\nimport { parseStandardPlanningResponse } from './standard-planning-parser';\nimport type { PlanOptions } from './types';\n\nconst debug = getDebug('planning');\nconst warnLog = getDebug('planning', { console: true });\n\nconst noPreviousActionsText =\n  'No previous actions have been executed in this aiAct execution yet. If the instruction asks for actions, choose the first action to execute.';\ntype PlanningCallResponse = Awaited<ReturnType<typeof callAI>>;\n\ntype CallAndParsePlanningResponseOptions = {\n  messages: ModelCallMessages;\n  modelRuntime: PlanOptions['modelRuntime'];\n  abortSignal?: AbortSignal;\n  includeLocateInPlanning: boolean;\n  actionSpace: PlanOptions['actionSpace'];\n  locateResultCodec?: LocateResultCodec;\n  locateResultContext: LocateResultContext;\n  includeThought: boolean;\n  includeLog: boolean;\n};\n\nasync function callAndParsePlanningResponse(\n  options: CallAndParsePlanningResponseOptions,\n): Promise<{\n  response: PlanningCallResponse;\n  planFromAI: RawResponsePlanningAIResponse;\n  actions: PlanningAction[];\n}> {\n  const {\n    messages,\n    modelRuntime,\n    abortSignal,\n    includeLocateInPlanning,\n    actionSpace,\n    locateResultCodec,\n    locateResultContext,\n    includeThought,\n    includeLog,\n  } = options;\n  assert(\n    modelRuntime.adapter.planning.kind === 'standard',\n    'callAndParsePlanningResponse requires a standard planning adapter',\n  );\n  const actionOutputProtocol =\n    modelRuntime.adapter.planning.protocol.actionOutputProtocol;\n\n  return callAiAndParseWithRetry({\n    callAi: (retryAttempt, previousParseError) =>\n      callAI(\n        withSemanticRetryFeedback(messages, previousParseError),\n        modelRuntime,\n        {\n          abortSignal,\n          requiresOriginalImageDetail: includeLocateInPlanning,\n          semanticRetryAttempt: retryAttempt,\n        },\n      ),\n    parseResponse: (response) => {\n      const planFromAI = parseStandardPlanningResponse(response.content, {\n        includeThought,\n        actionOutputProtocol,\n        actionSpace,\n        logSource: includeLog ? 'model' : 'action',\n      });\n      if (\n        !planFromAI.action &&\n        planFromAI.finalizeSuccess === undefined &&\n        !planFromAI.error\n      ) {\n        throw new Error(\n          'Incomplete planning response: provide an action, <complete>, or <error>. A <planning> explanation alone is not sufficient.',\n        );\n      }\n      if (planFromAI.action && planFromAI.finalizeSuccess !== undefined) {\n        warnLog(\n          'Planning response included both an action and <complete>; ignoring <complete> output.',\n        );\n        planFromAI.finalizeMessage = undefined;\n        planFromAI.finalizeSuccess = undefined;\n      }\n\n      const actions = planFromAI.action ? [planFromAI.action] : [];\n      parsePlanningActions(actions, {\n        parseRawLocateParameter: actionOutputProtocol.parseRawLocateParameter,\n        actionSpace,\n        includeLocateInPlanning,\n        locateResultCodec,\n        locateResultContext,\n      });\n      return { response, planFromAI, actions };\n    },\n    toParseError: (parseError, response) => {\n      const errorMessage =\n        parseError instanceof Error ? parseError.message : String(parseError);\n      return new AIResponseParseError(\n        `XML parse error: ${errorMessage}`,\n        response.content,\n        response.usage,\n        response.rawAssistantOutput,\n        response.reasoning_content,\n      );\n    },\n    parseRetryTimes: modelRuntime.config.retryCount,\n    parseRetryInterval: modelRuntime.config.retryInterval,\n    abortSignal,\n    onParseRetry: (parseError) => {\n      debug(\n        'retrying plan after response parsing failed: %s',\n        parseError instanceof Error ? parseError.message : String(parseError),\n      );\n    },\n  });\n}\n\nexport async function standardPlan(\n  userInstruction: PreparedUserPrompt,\n  opts: PlanOptions,\n): Promise<PlanningAIResponse> {\n  const { context, conversationHistory } = opts;\n  const modelRuntime = opts.modelRuntime;\n  const { adapter } = modelRuntime;\n  const { shotSize } = context;\n  const screenshotBase64 = context.screenshot.base64;\n  assert(\n    adapter.planning.kind === 'standard',\n    'standardPlan requires a standard planning adapter',\n  );\n  const planningProtocol = adapter.planning.protocol;\n\n  if (opts.includeLocateInPlanning && !modelRuntime.config.modelFamily) {\n    throw new Error(\n      planningModelFamilyRequiredForLocateMessage(modelRuntime.config.slot),\n    );\n  }\n\n  const locateResultCodec = modelRuntime.config.modelFamily\n    ? adapter.planning.locateResultCodec\n    : undefined;\n\n  // Only enable sub-goals when aiAct is in deep-thinking planning mode.\n  const includeSubGoals = opts.effort === 'deepThink';\n  const includeThought = opts.effort !== 'fast';\n  const includeLog = opts.effort !== 'fast';\n\n  if (opts.includeLocateInPlanning && !locateResultCodec) {\n    throw new Error(\n      planningModelFamilyRequiredForLocateMessage(modelRuntime.config.slot),\n    );\n  }\n\n  const systemPrompt = await buildStandardPlanningSystemPrompt({\n    actionSpace: opts.actionSpace,\n    includeThought,\n    includeLog,\n    includeSubGoals,\n    planningProtocol,\n    ...(opts.includeLocateInPlanning && locateResultCodec\n      ? {\n          includeLocateInPlanning: true,\n          locatePromptSpec: locateResultCodec.promptSpec,\n        }\n      : { includeLocateInPlanning: false }),\n  });\n\n  const preparedImage = await prepareModelImage({\n    imageBase64: screenshotBase64,\n    width: shotSize.width,\n    height: shotSize.height,\n    policy: adapter.imagePreprocess,\n  });\n  const imagePayload = preparedImage.imageBase64;\n\n  const actionContext = opts.actionContext ? `${opts.actionContext}\\n` : '';\n\n  const referenceImageMessages = preparedReferenceImagesToMessages(\n    userInstruction.referenceImages,\n  );\n  const instruction: ConversationMessage[] = [\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `${actionContext}<user_instruction>${userInstruction.text}</user_instruction>`,\n        },\n      ],\n    },\n    ...referenceImageMessages,\n  ];\n\n  let latestFeedbackMessage: ConversationMessage;\n\n  // Build sub-goal status text to include in the message\n  // In planning deep-think mode: show full sub-goals with logs\n  // Otherwise: show historical execution logs\n  const executionProgressText = includeSubGoals\n    ? conversationHistory.subGoalsToText()\n    : conversationHistory.historicalLogsToText();\n  const executionProgressSection = executionProgressText\n    ? `\\n\\n${executionProgressText}`\n    : conversationHistory.pendingFeedbackMessage\n      ? ''\n      : `\\n\\n${noPreviousActionsText}`;\n\n  // Build memories text to include in the message\n  const memoriesText = conversationHistory.memoriesToText();\n  const memoriesSection = memoriesText ? `\\n\\n${memoriesText}` : '';\n\n  if (conversationHistory.pendingFeedbackMessage) {\n    latestFeedbackMessage = {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `${conversationHistory.pendingFeedbackMessage}. The previous action has been executed, here is the latest screenshot. Please continue according to the instruction.${memoriesSection}${executionProgressSection}`,\n        },\n        {\n          type: 'image',\n          url: imagePayload,\n        },\n      ],\n    };\n\n    conversationHistory.resetPendingFeedbackMessageIfExists();\n  } else {\n    latestFeedbackMessage = {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `This is the current screenshot.${memoriesSection}${executionProgressSection}`,\n        },\n        {\n          type: 'image',\n          url: imagePayload,\n        },\n      ],\n    };\n  }\n  conversationHistory.appendMessage(latestFeedbackMessage);\n\n  // Compress history if it exceeds the threshold to avoid context overflow\n  conversationHistory.compressHistory(50, 20);\n\n  const historyLog = conversationHistory.snapshot(opts.imagesIncludeCount);\n\n  const msgs: ModelCallMessages = [\n    { role: 'system', content: systemPrompt },\n    ...instruction,\n    ...historyLog,\n  ];\n\n  const {\n    response: {\n      content: rawResponse,\n      usage,\n      reasoning_content,\n      rawAssistantOutput,\n    },\n    planFromAI,\n    actions,\n  } = await callAndParsePlanningResponse({\n    messages: msgs,\n    modelRuntime,\n    abortSignal: opts.abortSignal,\n    includeLocateInPlanning: opts.includeLocateInPlanning,\n    actionSpace: opts.actionSpace,\n    locateResultCodec,\n    locateResultContext: {\n      preparedSize: preparedImage.preparedSize,\n      contentSize: preparedImage.contentSize,\n    },\n    includeThought,\n    includeLog,\n  });\n\n  // YAML is derived from validated actions outside the model-response retry scope.\n  // dumpActionParam omits runtime-only locatedPixelResult fields.\n  const yamlFlow = buildYamlFlowFromPlans(actions, opts.actionSpace);\n\n  let shouldContinuePlanning = true;\n\n  // Check if task is completed via <complete> tag\n  if (planFromAI.finalizeSuccess !== undefined) {\n    debug('task completed via <complete> tag, stop planning');\n    shouldContinuePlanning = false;\n    // Mark all sub-goals as finished when goal is completed in planning deep-think mode.\n    if (includeSubGoals) {\n      conversationHistory.markAllSubGoalsFinished();\n    }\n  }\n\n  const returnValue: PlanningAIResponse = {\n    ...planFromAI,\n    actions,\n    rawResponse,\n    rawAssistantOutput,\n    usage,\n    reasoning_content,\n    yamlFlow,\n    shouldContinuePlanning,\n  };\n\n  assert(planFromAI, \"can't get plans from AI\");\n\n  // TODO: The plan log is recorded before its action has executed, so a failed\n  // action may still appear in the next round as an action already performed.\n  // Move this write to the successful action execution path in TaskExecutor.action.\n  // Update sub-goals in conversation history only in planning deep-think mode.\n  if (includeSubGoals) {\n    if (planFromAI.updateSubGoals?.length) {\n      conversationHistory.mergeSubGoals(planFromAI.updateSubGoals);\n    }\n    if (planFromAI.markFinishedIndexes?.length) {\n      for (const index of planFromAI.markFinishedIndexes) {\n        conversationHistory.markSubGoalFinished(index);\n      }\n    }\n    // Append the planning log to the currently running sub-goal\n    if (planFromAI.log) {\n      conversationHistory.appendSubGoalLog(planFromAI.log);\n    }\n  } else {\n    // Without planning deep-think mode, accumulate logs as historical execution steps.\n    if (planFromAI.log) {\n      conversationHistory.appendHistoricalLog(planFromAI.log);\n    }\n  }\n\n  // Append memory to conversation history if present\n  if (planFromAI.memory) {\n    conversationHistory.appendMemory(planFromAI.memory);\n  }\n\n  const shouldReplayRawOutput =\n    (rawAssistantOutput?.type === 'responses' &&\n      modelRuntime.adapter.responses.replayRawAssistantOutput) ||\n    (rawAssistantOutput?.type === 'chat-completion' &&\n      modelRuntime.adapter.chatCompletion.replayRawAssistantMessage);\n  if (rawAssistantOutput && shouldReplayRawOutput) {\n    conversationHistory.appendModelOutput(rawAssistantOutput);\n  } else {\n    if (!rawAssistantOutput) {\n      debug(\n        'Raw assistant output is unavailable; saving only response text in conversation history.',\n      );\n    }\n    conversationHistory.appendMessage({\n      role: 'assistant',\n      content: [\n        {\n          type: 'text',\n          text: rawResponse,\n        },\n      ],\n    });\n  }\n\n  return returnValue;\n}\n"],"names":["debug","getDebug","warnLog","noPreviousActionsText","callAndParsePlanningResponse","options","messages","modelRuntime","abortSignal","includeLocateInPlanning","actionSpace","locateResultCodec","locateResultContext","includeThought","includeLog","assert","actionOutputProtocol","callAiAndParseWithRetry","retryAttempt","previousParseError","callAI","withSemanticRetryFeedback","response","planFromAI","parseStandardPlanningResponse","undefined","Error","actions","parsePlanningActions","parseError","errorMessage","String","AIResponseParseError","standardPlan","userInstruction","opts","context","conversationHistory","adapter","shotSize","screenshotBase64","planningProtocol","planningModelFamilyRequiredForLocateMessage","includeSubGoals","systemPrompt","buildStandardPlanningSystemPrompt","preparedImage","prepareModelImage","imagePayload","actionContext","referenceImageMessages","preparedReferenceImagesToMessages","instruction","latestFeedbackMessage","executionProgressText","executionProgressSection","memoriesText","memoriesSection","historyLog","msgs","rawResponse","usage","reasoning_content","rawAssistantOutput","yamlFlow","buildYamlFlowFromPlans","shouldContinuePlanning","returnValue","index","shouldReplayRawOutput"],"mappings":";;;;;;;;;;;AA8BA,MAAMA,QAAQC,SAAS;AACvB,MAAMC,UAAUD,SAAS,YAAY;IAAE,SAAS;AAAK;AAErD,MAAME,wBACJ;AAeF,eAAeC,6BACbC,OAA4C;IAM5C,MAAM,EACJC,QAAQ,EACRC,YAAY,EACZC,WAAW,EACXC,uBAAuB,EACvBC,WAAW,EACXC,iBAAiB,EACjBC,mBAAmB,EACnBC,cAAc,EACdC,UAAU,EACX,GAAGT;IACJU,OACER,AAAuC,eAAvCA,aAAa,OAAO,CAAC,QAAQ,CAAC,IAAI,EAClC;IAEF,MAAMS,uBACJT,aAAa,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,oBAAoB;IAE7D,OAAOU,wBAAwB;QAC7B,QAAQ,CAACC,cAAcC,qBACrBC,OACEC,0BAA0Bf,UAAUa,qBACpCZ,cACA;gBACEC;gBACA,6BAA6BC;gBAC7B,sBAAsBS;YACxB;QAEJ,eAAe,CAACI;YACd,MAAMC,aAAaC,8BAA8BF,SAAS,OAAO,EAAE;gBACjET;gBACAG;gBACAN;gBACA,WAAWI,aAAa,UAAU;YACpC;YACA,IACE,CAACS,WAAW,MAAM,IAClBA,AAA+BE,WAA/BF,WAAW,eAAe,IAC1B,CAACA,WAAW,KAAK,EAEjB,MAAM,IAAIG,MACR;YAGJ,IAAIH,WAAW,MAAM,IAAIA,AAA+BE,WAA/BF,WAAW,eAAe,EAAgB;gBACjErB,QACE;gBAEFqB,WAAW,eAAe,GAAGE;gBAC7BF,WAAW,eAAe,GAAGE;YAC/B;YAEA,MAAME,UAAUJ,WAAW,MAAM,GAAG;gBAACA,WAAW,MAAM;aAAC,GAAG,EAAE;YAC5DK,qBAAqBD,SAAS;gBAC5B,yBAAyBX,qBAAqB,uBAAuB;gBACrEN;gBACAD;gBACAE;gBACAC;YACF;YACA,OAAO;gBAAEU;gBAAUC;gBAAYI;YAAQ;QACzC;QACA,cAAc,CAACE,YAAYP;YACzB,MAAMQ,eACJD,sBAAsBH,QAAQG,WAAW,OAAO,GAAGE,OAAOF;YAC5D,OAAO,IAAIG,qBACT,CAAC,iBAAiB,EAAEF,cAAc,EAClCR,SAAS,OAAO,EAChBA,SAAS,KAAK,EACdA,SAAS,kBAAkB,EAC3BA,SAAS,iBAAiB;QAE9B;QACA,iBAAiBf,aAAa,MAAM,CAAC,UAAU;QAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;QACrDC;QACA,cAAc,CAACqB;YACb7B,MACE,mDACA6B,sBAAsBH,QAAQG,WAAW,OAAO,GAAGE,OAAOF;QAE9D;IACF;AACF;AAEO,eAAeI,aACpBC,eAAmC,EACnCC,IAAiB;IAEjB,MAAM,EAAEC,OAAO,EAAEC,mBAAmB,EAAE,GAAGF;IACzC,MAAM5B,eAAe4B,KAAK,YAAY;IACtC,MAAM,EAAEG,OAAO,EAAE,GAAG/B;IACpB,MAAM,EAAEgC,QAAQ,EAAE,GAAGH;IACrB,MAAMI,mBAAmBJ,QAAQ,UAAU,CAAC,MAAM;IAClDrB,OACEuB,AAA0B,eAA1BA,QAAQ,QAAQ,CAAC,IAAI,EACrB;IAEF,MAAMG,mBAAmBH,QAAQ,QAAQ,CAAC,QAAQ;IAElD,IAAIH,KAAK,uBAAuB,IAAI,CAAC5B,aAAa,MAAM,CAAC,WAAW,EAClE,MAAM,IAAImB,MACRgB,4CAA4CnC,aAAa,MAAM,CAAC,IAAI;IAIxE,MAAMI,oBAAoBJ,aAAa,MAAM,CAAC,WAAW,GACrD+B,QAAQ,QAAQ,CAAC,iBAAiB,GAClCb;IAGJ,MAAMkB,kBAAkBR,AAAgB,gBAAhBA,KAAK,MAAM;IACnC,MAAMtB,iBAAiBsB,AAAgB,WAAhBA,KAAK,MAAM;IAClC,MAAMrB,aAAaqB,AAAgB,WAAhBA,KAAK,MAAM;IAE9B,IAAIA,KAAK,uBAAuB,IAAI,CAACxB,mBACnC,MAAM,IAAIe,MACRgB,4CAA4CnC,aAAa,MAAM,CAAC,IAAI;IAIxE,MAAMqC,eAAe,MAAMC,kCAAkC;QAC3D,aAAaV,KAAK,WAAW;QAC7BtB;QACAC;QACA6B;QACAF;QACA,GAAIN,KAAK,uBAAuB,IAAIxB,oBAChC;YACE,yBAAyB;YACzB,kBAAkBA,kBAAkB,UAAU;QAChD,IACA;YAAE,yBAAyB;QAAM,CAAC;IACxC;IAEA,MAAMmC,gBAAgB,MAAMC,kBAAkB;QAC5C,aAAaP;QACb,OAAOD,SAAS,KAAK;QACrB,QAAQA,SAAS,MAAM;QACvB,QAAQD,QAAQ,eAAe;IACjC;IACA,MAAMU,eAAeF,cAAc,WAAW;IAE9C,MAAMG,gBAAgBd,KAAK,aAAa,GAAG,GAAGA,KAAK,aAAa,CAAC,EAAE,CAAC,GAAG;IAEvE,MAAMe,yBAAyBC,kCAC7BjB,gBAAgB,eAAe;IAEjC,MAAMkB,cAAqC;QACzC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGH,cAAc,kBAAkB,EAAEf,gBAAgB,IAAI,CAAC,mBAAmB,CAAC;gBACtF;aACD;QACH;WACGgB;KACJ;IAED,IAAIG;IAKJ,MAAMC,wBAAwBX,kBAC1BN,oBAAoB,cAAc,KAClCA,oBAAoB,oBAAoB;IAC5C,MAAMkB,2BAA2BD,wBAC7B,CAAC,IAAI,EAAEA,uBAAuB,GAC9BjB,oBAAoB,sBAAsB,GACxC,KACA,CAAC,IAAI,EAAElC,uBAAuB;IAGpC,MAAMqD,eAAenB,oBAAoB,cAAc;IACvD,MAAMoB,kBAAkBD,eAAe,CAAC,IAAI,EAAEA,cAAc,GAAG;IAE/D,IAAInB,oBAAoB,sBAAsB,EAAE;QAC9CgB,wBAAwB;YACtB,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGhB,oBAAoB,sBAAsB,CAAC,qHAAqH,EAAEoB,kBAAkBF,0BAA0B;gBACzN;gBACA;oBACE,MAAM;oBACN,KAAKP;gBACP;aACD;QACH;QAEAX,oBAAoB,mCAAmC;IACzD,OACEgB,wBAAwB;QACtB,MAAM;QACN,SAAS;YACP;gBACE,MAAM;gBACN,MAAM,CAAC,+BAA+B,EAAEI,kBAAkBF,0BAA0B;YACtF;YACA;gBACE,MAAM;gBACN,KAAKP;YACP;SACD;IACH;IAEFX,oBAAoB,aAAa,CAACgB;IAGlChB,oBAAoB,eAAe,CAAC,IAAI;IAExC,MAAMqB,aAAarB,oBAAoB,QAAQ,CAACF,KAAK,kBAAkB;IAEvE,MAAMwB,OAA0B;QAC9B;YAAE,MAAM;YAAU,SAASf;QAAa;WACrCQ;WACAM;KACJ;IAED,MAAM,EACJ,UAAU,EACR,SAASE,WAAW,EACpBC,KAAK,EACLC,iBAAiB,EACjBC,kBAAkB,EACnB,EACDxC,UAAU,EACVI,OAAO,EACR,GAAG,MAAMvB,6BAA6B;QACrC,UAAUuD;QACVpD;QACA,aAAa4B,KAAK,WAAW;QAC7B,yBAAyBA,KAAK,uBAAuB;QACrD,aAAaA,KAAK,WAAW;QAC7BxB;QACA,qBAAqB;YACnB,cAAcmC,cAAc,YAAY;YACxC,aAAaA,cAAc,WAAW;QACxC;QACAjC;QACAC;IACF;IAIA,MAAMkD,WAAWC,uBAAuBtC,SAASQ,KAAK,WAAW;IAEjE,IAAI+B,yBAAyB;IAG7B,IAAI3C,AAA+BE,WAA/BF,WAAW,eAAe,EAAgB;QAC5CvB,MAAM;QACNkE,yBAAyB;QAEzB,IAAIvB,iBACFN,oBAAoB,uBAAuB;IAE/C;IAEA,MAAM8B,cAAkC;QACtC,GAAG5C,UAAU;QACbI;QACAiC;QACAG;QACAF;QACAC;QACAE;QACAE;IACF;IAEAnD,OAAOQ,YAAY;IAMnB,IAAIoB,iBAAiB;QACnB,IAAIpB,WAAW,cAAc,EAAE,QAC7Bc,oBAAoB,aAAa,CAACd,WAAW,cAAc;QAE7D,IAAIA,WAAW,mBAAmB,EAAE,QAClC,KAAK,MAAM6C,SAAS7C,WAAW,mBAAmB,CAChDc,oBAAoB,mBAAmB,CAAC+B;QAI5C,IAAI7C,WAAW,GAAG,EAChBc,oBAAoB,gBAAgB,CAACd,WAAW,GAAG;IAEvD,OAEE,IAAIA,WAAW,GAAG,EAChBc,oBAAoB,mBAAmB,CAACd,WAAW,GAAG;IAK1D,IAAIA,WAAW,MAAM,EACnBc,oBAAoB,YAAY,CAACd,WAAW,MAAM;IAGpD,MAAM8C,wBACHN,oBAAoB,SAAS,eAC5BxD,aAAa,OAAO,CAAC,SAAS,CAAC,wBAAwB,IACxDwD,oBAAoB,SAAS,qBAC5BxD,aAAa,OAAO,CAAC,cAAc,CAAC,yBAAyB;IACjE,IAAIwD,sBAAsBM,uBACxBhC,oBAAoB,iBAAiB,CAAC0B;SACjC;QACL,IAAI,CAACA,oBACH/D,MACE;QAGJqC,oBAAoB,aAAa,CAAC;YAChC,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAMuB;gBACR;aACD;QACH;IACF;IAEA,OAAOO;AACT"}
+{"version":3,"file":"ai-model/workflows/planning/standard-planning.mjs","sources":["../../../../../src/ai-model/workflows/planning/standard-planning.ts"],"sourcesContent":["import type { ConversationMessage } from '@/ai-model/service-caller/types';\nimport type {\n  PlanningAIResponse,\n  PlanningAction,\n  RawResponsePlanningAIResponse,\n} from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport { buildYamlFlowFromPlans } from '../../../common';\nimport { prepareModelImage } from '../../model-adapter/image-preprocess';\nimport { buildStandardPlanningSystemPrompt } from '../../prompt/planning';\nimport {\n  AIResponseParseError,\n  callGenerativeAI,\n} from '../../service-caller/index';\nimport {\n  callAiAndParseWithRetry,\n  withSemanticRetryFeedback,\n} from '../../service-caller/semantic-retry';\nimport type { ModelCallMessages } from '../../service-caller/types';\nimport type {\n  LocateResultCodec,\n  LocateResultContext,\n} from '../../shared/model-locate-result';\nimport { planningModelFamilyRequiredForLocateMessage } from '../../shared/model-locate-result/errors';\nimport {\n  type PreparedUserPrompt,\n  preparedReferenceImagesToMessages,\n} from '../../shared/multimodal-prompt';\nimport { parsePlanningActions } from './parse-planning-actions';\nimport { parseStandardPlanningResponse } from './standard-planning-parser';\nimport type { PlanOptions } from './types';\n\nconst debug = getDebug('planning');\nconst warnLog = getDebug('planning', { console: true });\n\nconst noPreviousActionsText =\n  'No previous actions have been executed in this aiAct execution yet. If the instruction asks for actions, choose the first action to execute.';\ntype PlanningCallResponse = Awaited<ReturnType<typeof callGenerativeAI>>;\n\ntype CallAndParsePlanningResponseOptions = {\n  messages: ModelCallMessages;\n  modelRuntime: PlanOptions['modelRuntime'];\n  abortSignal?: AbortSignal;\n  includeLocateInPlanning: boolean;\n  actionSpace: PlanOptions['actionSpace'];\n  locateResultCodec?: LocateResultCodec;\n  locateResultContext: LocateResultContext;\n  includeThought: boolean;\n  includeLog: boolean;\n};\n\nasync function callAndParsePlanningResponse(\n  options: CallAndParsePlanningResponseOptions,\n): Promise<{\n  response: PlanningCallResponse;\n  planFromAI: RawResponsePlanningAIResponse;\n  actions: PlanningAction[];\n}> {\n  const {\n    messages,\n    modelRuntime,\n    abortSignal,\n    includeLocateInPlanning,\n    actionSpace,\n    locateResultCodec,\n    locateResultContext,\n    includeThought,\n    includeLog,\n  } = options;\n  assert(\n    modelRuntime.adapter.planning.kind === 'standard',\n    'callAndParsePlanningResponse requires a standard planning adapter',\n  );\n  const actionOutputProtocol =\n    modelRuntime.adapter.planning.protocol.actionOutputProtocol;\n\n  return callAiAndParseWithRetry({\n    callAi: (retryAttempt, previousParseError) =>\n      callGenerativeAI(\n        withSemanticRetryFeedback(messages, previousParseError),\n        modelRuntime,\n        {\n          abortSignal,\n          requiresOriginalImageDetail: includeLocateInPlanning,\n          semanticRetryAttempt: retryAttempt,\n        },\n      ),\n    parseResponse: (response) => {\n      const planFromAI = parseStandardPlanningResponse(response.content, {\n        includeThought,\n        actionOutputProtocol,\n        actionSpace,\n        logSource: includeLog ? 'model' : 'action',\n      });\n      if (\n        !planFromAI.action &&\n        planFromAI.finalizeSuccess === undefined &&\n        !planFromAI.error\n      ) {\n        throw new Error(\n          'Incomplete planning response: provide an action, <complete>, or <error>. A <planning> explanation alone is not sufficient.',\n        );\n      }\n      if (planFromAI.action && planFromAI.finalizeSuccess !== undefined) {\n        warnLog(\n          'Planning response included both an action and <complete>; ignoring <complete> output.',\n        );\n        planFromAI.finalizeMessage = undefined;\n        planFromAI.finalizeSuccess = undefined;\n      }\n\n      const actions = planFromAI.action ? [planFromAI.action] : [];\n      parsePlanningActions(actions, {\n        parseRawLocateParameter: actionOutputProtocol.parseRawLocateParameter,\n        actionSpace,\n        includeLocateInPlanning,\n        locateResultCodec,\n        locateResultContext,\n      });\n      return { response, planFromAI, actions };\n    },\n    toParseError: (parseError, response) => {\n      const errorMessage =\n        parseError instanceof Error ? parseError.message : String(parseError);\n      return new AIResponseParseError(\n        `XML parse error: ${errorMessage}`,\n        response.content,\n        response.usage,\n        response.rawAssistantOutput,\n        response.reasoning_content,\n      );\n    },\n    parseRetryTimes: modelRuntime.config.retryCount,\n    parseRetryInterval: modelRuntime.config.retryInterval,\n    abortSignal,\n    onParseRetry: (parseError) => {\n      debug(\n        'retrying plan after response parsing failed: %s',\n        parseError instanceof Error ? parseError.message : String(parseError),\n      );\n    },\n  });\n}\n\nexport async function standardPlan(\n  userInstruction: PreparedUserPrompt,\n  opts: PlanOptions,\n): Promise<PlanningAIResponse> {\n  const { context, conversationHistory } = opts;\n  const modelRuntime = opts.modelRuntime;\n  const { adapter } = modelRuntime;\n  const { shotSize } = context;\n  const screenshotBase64 = context.screenshot.base64;\n  assert(\n    adapter.planning.kind === 'standard',\n    'standardPlan requires a standard planning adapter',\n  );\n  const planningProtocol = adapter.planning.protocol;\n\n  if (opts.includeLocateInPlanning && !modelRuntime.config.modelFamily) {\n    throw new Error(\n      planningModelFamilyRequiredForLocateMessage(modelRuntime.config.slot),\n    );\n  }\n\n  const locateResultCodec = modelRuntime.config.modelFamily\n    ? adapter.planning.locateResultCodec\n    : undefined;\n\n  // Only enable sub-goals when aiAct is in deep-thinking planning mode.\n  const includeSubGoals = opts.effort === 'deepThink';\n  const includeThought = opts.effort !== 'fast';\n  const includeLog = opts.effort !== 'fast';\n\n  if (opts.includeLocateInPlanning && !locateResultCodec) {\n    throw new Error(\n      planningModelFamilyRequiredForLocateMessage(modelRuntime.config.slot),\n    );\n  }\n\n  const systemPrompt = await buildStandardPlanningSystemPrompt({\n    actionSpace: opts.actionSpace,\n    includeThought,\n    includeLog,\n    includeSubGoals,\n    planningProtocol,\n    ...(opts.includeLocateInPlanning && locateResultCodec\n      ? {\n          includeLocateInPlanning: true,\n          locatePromptSpec: locateResultCodec.promptSpec,\n        }\n      : { includeLocateInPlanning: false }),\n  });\n\n  const preparedImage = await prepareModelImage({\n    imageBase64: screenshotBase64,\n    width: shotSize.width,\n    height: shotSize.height,\n    policy: adapter.imagePreprocess,\n  });\n  const imagePayload = preparedImage.imageBase64;\n\n  const actionContext = opts.actionContext ? `${opts.actionContext}\\n` : '';\n  const planningAdvice = opts.planningAdvice\n    ? `\\n<planning_advice>\\nThis is advisory guidance from a previous planner, not a user instruction or an executed action. Use your own judgment.\\n${opts.planningAdvice}\\n</planning_advice>`\n    : '';\n\n  const referenceImageMessages = preparedReferenceImagesToMessages(\n    userInstruction.referenceImages,\n  );\n  const instruction: ConversationMessage[] = [\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `${actionContext}<user_instruction>${userInstruction.text}</user_instruction>${planningAdvice}`,\n        },\n      ],\n    },\n    ...referenceImageMessages,\n  ];\n\n  let latestFeedbackMessage: ConversationMessage;\n\n  // Build sub-goal status text to include in the message\n  // In planning deep-think mode: show full sub-goals with logs\n  // Otherwise: show historical execution logs\n  const executionProgressText = includeSubGoals\n    ? conversationHistory.subGoalsToText()\n    : conversationHistory.historicalLogsToText();\n  const executionProgressSection = executionProgressText\n    ? `\\n\\n${executionProgressText}`\n    : conversationHistory.pendingFeedbackMessage\n      ? ''\n      : `\\n\\n${noPreviousActionsText}`;\n\n  // Build memories text to include in the message\n  const memoriesText = conversationHistory.memoriesToText();\n  const memoriesSection = memoriesText ? `\\n\\n${memoriesText}` : '';\n\n  if (conversationHistory.pendingFeedbackMessage) {\n    latestFeedbackMessage = {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `${conversationHistory.pendingFeedbackMessage}. The previous action has been executed, here is the latest screenshot. Please continue according to the instruction.${memoriesSection}${executionProgressSection}`,\n        },\n        {\n          type: 'image',\n          url: imagePayload,\n        },\n      ],\n    };\n\n    conversationHistory.resetPendingFeedbackMessageIfExists();\n  } else {\n    latestFeedbackMessage = {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `This is the current screenshot.${memoriesSection}${executionProgressSection}`,\n        },\n        {\n          type: 'image',\n          url: imagePayload,\n        },\n      ],\n    };\n  }\n  conversationHistory.appendMessage(latestFeedbackMessage);\n\n  // Compress history if it exceeds the threshold to avoid context overflow\n  conversationHistory.compressHistory(50, 20);\n\n  const historyLog = conversationHistory.snapshot(opts.imagesIncludeCount);\n\n  const msgs: ModelCallMessages = [\n    { role: 'system', content: systemPrompt },\n    ...instruction,\n    ...historyLog,\n  ];\n\n  const {\n    response: {\n      content: rawResponse,\n      usage,\n      reasoning_content,\n      rawAssistantOutput,\n    },\n    planFromAI,\n    actions,\n  } = await callAndParsePlanningResponse({\n    messages: msgs,\n    modelRuntime,\n    abortSignal: opts.abortSignal,\n    includeLocateInPlanning: opts.includeLocateInPlanning,\n    actionSpace: opts.actionSpace,\n    locateResultCodec,\n    locateResultContext: {\n      preparedSize: preparedImage.preparedSize,\n      contentSize: preparedImage.contentSize,\n    },\n    includeThought,\n    includeLog,\n  });\n\n  // YAML is derived from validated actions outside the model-response retry scope.\n  // dumpActionParam omits runtime-only locatedPixelResult fields.\n  const yamlFlow = buildYamlFlowFromPlans(actions, opts.actionSpace);\n\n  let shouldContinuePlanning = true;\n\n  // Check if task is completed via <complete> tag\n  if (planFromAI.finalizeSuccess !== undefined) {\n    debug('task completed via <complete> tag, stop planning');\n    shouldContinuePlanning = false;\n    // Mark all sub-goals as finished when goal is completed in planning deep-think mode.\n    if (includeSubGoals) {\n      conversationHistory.markAllSubGoalsFinished();\n    }\n  }\n\n  const returnValue: PlanningAIResponse = {\n    ...planFromAI,\n    actions,\n    rawResponse,\n    rawAssistantOutput,\n    usage,\n    reasoning_content,\n    yamlFlow,\n    shouldContinuePlanning,\n  };\n\n  assert(planFromAI, \"can't get plans from AI\");\n\n  // TODO: The plan log is recorded before its action has executed, so a failed\n  // action may still appear in the next round as an action already performed.\n  // Move this write to the successful action execution path in TaskExecutor.action.\n  // Update sub-goals in conversation history only in planning deep-think mode.\n  if (includeSubGoals) {\n    if (planFromAI.updateSubGoals?.length) {\n      conversationHistory.mergeSubGoals(planFromAI.updateSubGoals);\n    }\n    if (planFromAI.markFinishedIndexes?.length) {\n      for (const index of planFromAI.markFinishedIndexes) {\n        conversationHistory.markSubGoalFinished(index);\n      }\n    }\n    // Append the planning log to the currently running sub-goal\n    if (planFromAI.log) {\n      conversationHistory.appendSubGoalLog(planFromAI.log);\n    }\n  } else {\n    // Without planning deep-think mode, accumulate logs as historical execution steps.\n    if (planFromAI.log) {\n      conversationHistory.appendHistoricalLog(planFromAI.log);\n    }\n  }\n\n  // Append memory to conversation history if present\n  if (planFromAI.memory) {\n    conversationHistory.appendMemory(planFromAI.memory);\n  }\n\n  const shouldReplayRawOutput =\n    (rawAssistantOutput?.type === 'responses' &&\n      modelRuntime.adapter.responses.replayRawAssistantOutput) ||\n    (rawAssistantOutput?.type === 'chat-completion' &&\n      modelRuntime.adapter.chatCompletion.replayRawAssistantMessage);\n  if (rawAssistantOutput && shouldReplayRawOutput) {\n    conversationHistory.appendModelOutput(rawAssistantOutput);\n  } else {\n    if (!rawAssistantOutput) {\n      debug(\n        'Raw assistant output is unavailable; saving only response text in conversation history.',\n      );\n    }\n    conversationHistory.appendMessage({\n      role: 'assistant',\n      content: [\n        {\n          type: 'text',\n          text: rawResponse,\n        },\n      ],\n    });\n  }\n\n  return returnValue;\n}\n"],"names":["debug","getDebug","warnLog","noPreviousActionsText","callAndParsePlanningResponse","options","messages","modelRuntime","abortSignal","includeLocateInPlanning","actionSpace","locateResultCodec","locateResultContext","includeThought","includeLog","assert","actionOutputProtocol","callAiAndParseWithRetry","retryAttempt","previousParseError","callGenerativeAI","withSemanticRetryFeedback","response","planFromAI","parseStandardPlanningResponse","undefined","Error","actions","parsePlanningActions","parseError","errorMessage","String","AIResponseParseError","standardPlan","userInstruction","opts","context","conversationHistory","adapter","shotSize","screenshotBase64","planningProtocol","planningModelFamilyRequiredForLocateMessage","includeSubGoals","systemPrompt","buildStandardPlanningSystemPrompt","preparedImage","prepareModelImage","imagePayload","actionContext","planningAdvice","referenceImageMessages","preparedReferenceImagesToMessages","instruction","latestFeedbackMessage","executionProgressText","executionProgressSection","memoriesText","memoriesSection","historyLog","msgs","rawResponse","usage","reasoning_content","rawAssistantOutput","yamlFlow","buildYamlFlowFromPlans","shouldContinuePlanning","returnValue","index","shouldReplayRawOutput"],"mappings":";;;;;;;;;;;AAiCA,MAAMA,QAAQC,SAAS;AACvB,MAAMC,UAAUD,SAAS,YAAY;IAAE,SAAS;AAAK;AAErD,MAAME,wBACJ;AAeF,eAAeC,6BACbC,OAA4C;IAM5C,MAAM,EACJC,QAAQ,EACRC,YAAY,EACZC,WAAW,EACXC,uBAAuB,EACvBC,WAAW,EACXC,iBAAiB,EACjBC,mBAAmB,EACnBC,cAAc,EACdC,UAAU,EACX,GAAGT;IACJU,OACER,AAAuC,eAAvCA,aAAa,OAAO,CAAC,QAAQ,CAAC,IAAI,EAClC;IAEF,MAAMS,uBACJT,aAAa,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,oBAAoB;IAE7D,OAAOU,wBAAwB;QAC7B,QAAQ,CAACC,cAAcC,qBACrBC,iBACEC,0BAA0Bf,UAAUa,qBACpCZ,cACA;gBACEC;gBACA,6BAA6BC;gBAC7B,sBAAsBS;YACxB;QAEJ,eAAe,CAACI;YACd,MAAMC,aAAaC,8BAA8BF,SAAS,OAAO,EAAE;gBACjET;gBACAG;gBACAN;gBACA,WAAWI,aAAa,UAAU;YACpC;YACA,IACE,CAACS,WAAW,MAAM,IAClBA,AAA+BE,WAA/BF,WAAW,eAAe,IAC1B,CAACA,WAAW,KAAK,EAEjB,MAAM,IAAIG,MACR;YAGJ,IAAIH,WAAW,MAAM,IAAIA,AAA+BE,WAA/BF,WAAW,eAAe,EAAgB;gBACjErB,QACE;gBAEFqB,WAAW,eAAe,GAAGE;gBAC7BF,WAAW,eAAe,GAAGE;YAC/B;YAEA,MAAME,UAAUJ,WAAW,MAAM,GAAG;gBAACA,WAAW,MAAM;aAAC,GAAG,EAAE;YAC5DK,qBAAqBD,SAAS;gBAC5B,yBAAyBX,qBAAqB,uBAAuB;gBACrEN;gBACAD;gBACAE;gBACAC;YACF;YACA,OAAO;gBAAEU;gBAAUC;gBAAYI;YAAQ;QACzC;QACA,cAAc,CAACE,YAAYP;YACzB,MAAMQ,eACJD,sBAAsBH,QAAQG,WAAW,OAAO,GAAGE,OAAOF;YAC5D,OAAO,IAAIG,qBACT,CAAC,iBAAiB,EAAEF,cAAc,EAClCR,SAAS,OAAO,EAChBA,SAAS,KAAK,EACdA,SAAS,kBAAkB,EAC3BA,SAAS,iBAAiB;QAE9B;QACA,iBAAiBf,aAAa,MAAM,CAAC,UAAU;QAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;QACrDC;QACA,cAAc,CAACqB;YACb7B,MACE,mDACA6B,sBAAsBH,QAAQG,WAAW,OAAO,GAAGE,OAAOF;QAE9D;IACF;AACF;AAEO,eAAeI,aACpBC,eAAmC,EACnCC,IAAiB;IAEjB,MAAM,EAAEC,OAAO,EAAEC,mBAAmB,EAAE,GAAGF;IACzC,MAAM5B,eAAe4B,KAAK,YAAY;IACtC,MAAM,EAAEG,OAAO,EAAE,GAAG/B;IACpB,MAAM,EAAEgC,QAAQ,EAAE,GAAGH;IACrB,MAAMI,mBAAmBJ,QAAQ,UAAU,CAAC,MAAM;IAClDrB,OACEuB,AAA0B,eAA1BA,QAAQ,QAAQ,CAAC,IAAI,EACrB;IAEF,MAAMG,mBAAmBH,QAAQ,QAAQ,CAAC,QAAQ;IAElD,IAAIH,KAAK,uBAAuB,IAAI,CAAC5B,aAAa,MAAM,CAAC,WAAW,EAClE,MAAM,IAAImB,MACRgB,4CAA4CnC,aAAa,MAAM,CAAC,IAAI;IAIxE,MAAMI,oBAAoBJ,aAAa,MAAM,CAAC,WAAW,GACrD+B,QAAQ,QAAQ,CAAC,iBAAiB,GAClCb;IAGJ,MAAMkB,kBAAkBR,AAAgB,gBAAhBA,KAAK,MAAM;IACnC,MAAMtB,iBAAiBsB,AAAgB,WAAhBA,KAAK,MAAM;IAClC,MAAMrB,aAAaqB,AAAgB,WAAhBA,KAAK,MAAM;IAE9B,IAAIA,KAAK,uBAAuB,IAAI,CAACxB,mBACnC,MAAM,IAAIe,MACRgB,4CAA4CnC,aAAa,MAAM,CAAC,IAAI;IAIxE,MAAMqC,eAAe,MAAMC,kCAAkC;QAC3D,aAAaV,KAAK,WAAW;QAC7BtB;QACAC;QACA6B;QACAF;QACA,GAAIN,KAAK,uBAAuB,IAAIxB,oBAChC;YACE,yBAAyB;YACzB,kBAAkBA,kBAAkB,UAAU;QAChD,IACA;YAAE,yBAAyB;QAAM,CAAC;IACxC;IAEA,MAAMmC,gBAAgB,MAAMC,kBAAkB;QAC5C,aAAaP;QACb,OAAOD,SAAS,KAAK;QACrB,QAAQA,SAAS,MAAM;QACvB,QAAQD,QAAQ,eAAe;IACjC;IACA,MAAMU,eAAeF,cAAc,WAAW;IAE9C,MAAMG,gBAAgBd,KAAK,aAAa,GAAG,GAAGA,KAAK,aAAa,CAAC,EAAE,CAAC,GAAG;IACvE,MAAMe,iBAAiBf,KAAK,cAAc,GACtC,CAAC,8IAA8I,EAAEA,KAAK,cAAc,CAAC,oBAAoB,CAAC,GAC1L;IAEJ,MAAMgB,yBAAyBC,kCAC7BlB,gBAAgB,eAAe;IAEjC,MAAMmB,cAAqC;QACzC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGJ,cAAc,kBAAkB,EAAEf,gBAAgB,IAAI,CAAC,mBAAmB,EAAEgB,gBAAgB;gBACvG;aACD;QACH;WACGC;KACJ;IAED,IAAIG;IAKJ,MAAMC,wBAAwBZ,kBAC1BN,oBAAoB,cAAc,KAClCA,oBAAoB,oBAAoB;IAC5C,MAAMmB,2BAA2BD,wBAC7B,CAAC,IAAI,EAAEA,uBAAuB,GAC9BlB,oBAAoB,sBAAsB,GACxC,KACA,CAAC,IAAI,EAAElC,uBAAuB;IAGpC,MAAMsD,eAAepB,oBAAoB,cAAc;IACvD,MAAMqB,kBAAkBD,eAAe,CAAC,IAAI,EAAEA,cAAc,GAAG;IAE/D,IAAIpB,oBAAoB,sBAAsB,EAAE;QAC9CiB,wBAAwB;YACtB,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGjB,oBAAoB,sBAAsB,CAAC,qHAAqH,EAAEqB,kBAAkBF,0BAA0B;gBACzN;gBACA;oBACE,MAAM;oBACN,KAAKR;gBACP;aACD;QACH;QAEAX,oBAAoB,mCAAmC;IACzD,OACEiB,wBAAwB;QACtB,MAAM;QACN,SAAS;YACP;gBACE,MAAM;gBACN,MAAM,CAAC,+BAA+B,EAAEI,kBAAkBF,0BAA0B;YACtF;YACA;gBACE,MAAM;gBACN,KAAKR;YACP;SACD;IACH;IAEFX,oBAAoB,aAAa,CAACiB;IAGlCjB,oBAAoB,eAAe,CAAC,IAAI;IAExC,MAAMsB,aAAatB,oBAAoB,QAAQ,CAACF,KAAK,kBAAkB;IAEvE,MAAMyB,OAA0B;QAC9B;YAAE,MAAM;YAAU,SAAShB;QAAa;WACrCS;WACAM;KACJ;IAED,MAAM,EACJ,UAAU,EACR,SAASE,WAAW,EACpBC,KAAK,EACLC,iBAAiB,EACjBC,kBAAkB,EACnB,EACDzC,UAAU,EACVI,OAAO,EACR,GAAG,MAAMvB,6BAA6B;QACrC,UAAUwD;QACVrD;QACA,aAAa4B,KAAK,WAAW;QAC7B,yBAAyBA,KAAK,uBAAuB;QACrD,aAAaA,KAAK,WAAW;QAC7BxB;QACA,qBAAqB;YACnB,cAAcmC,cAAc,YAAY;YACxC,aAAaA,cAAc,WAAW;QACxC;QACAjC;QACAC;IACF;IAIA,MAAMmD,WAAWC,uBAAuBvC,SAASQ,KAAK,WAAW;IAEjE,IAAIgC,yBAAyB;IAG7B,IAAI5C,AAA+BE,WAA/BF,WAAW,eAAe,EAAgB;QAC5CvB,MAAM;QACNmE,yBAAyB;QAEzB,IAAIxB,iBACFN,oBAAoB,uBAAuB;IAE/C;IAEA,MAAM+B,cAAkC;QACtC,GAAG7C,UAAU;QACbI;QACAkC;QACAG;QACAF;QACAC;QACAE;QACAE;IACF;IAEApD,OAAOQ,YAAY;IAMnB,IAAIoB,iBAAiB;QACnB,IAAIpB,WAAW,cAAc,EAAE,QAC7Bc,oBAAoB,aAAa,CAACd,WAAW,cAAc;QAE7D,IAAIA,WAAW,mBAAmB,EAAE,QAClC,KAAK,MAAM8C,SAAS9C,WAAW,mBAAmB,CAChDc,oBAAoB,mBAAmB,CAACgC;QAI5C,IAAI9C,WAAW,GAAG,EAChBc,oBAAoB,gBAAgB,CAACd,WAAW,GAAG;IAEvD,OAEE,IAAIA,WAAW,GAAG,EAChBc,oBAAoB,mBAAmB,CAACd,WAAW,GAAG;IAK1D,IAAIA,WAAW,MAAM,EACnBc,oBAAoB,YAAY,CAACd,WAAW,MAAM;IAGpD,MAAM+C,wBACHN,oBAAoB,SAAS,eAC5BzD,aAAa,OAAO,CAAC,SAAS,CAAC,wBAAwB,IACxDyD,oBAAoB,SAAS,qBAC5BzD,aAAa,OAAO,CAAC,cAAc,CAAC,yBAAyB;IACjE,IAAIyD,sBAAsBM,uBACxBjC,oBAAoB,iBAAiB,CAAC2B;SACjC;QACL,IAAI,CAACA,oBACHhE,MACE;QAGJqC,oBAAoB,aAAa,CAAC;YAChC,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAMwB;gBACR;aACD;QACH;IACF;IAEA,OAAOO;AACT"}

package/dist/es/ai-model/workflows/recorder-generation/playwright.mjs

@@ -1,6 +1,6 @@
 import { PLAYWRIGHT_EXAMPLE_CODE } from "@midscene/shared/constants";
 import { getModelRuntime } from "../../models/index.mjs";
-import { callAI, callAIWithStringResponse } from "../../service-caller/index.mjs";
+import { callAIWithStringResponse, callGenerativeAI } from "../../service-caller/index.mjs";
 import { createEventCounts, createMessageContent, extractInputDescriptions, filterEventsByType, getScreenshotsForLLM, prepareEventSummary, processEventsForLLM, validateEvents } from "./common.mjs";
 function createModelRuntime(modelConfig) {
     return getModelRuntime(modelConfig);
@@ -104,7 +104,7 @@ ${PLAYWRIGHT_EXAMPLE_CODE}`;
             content: messageContent
         }
     ];
-    if (options.stream && options.onChunk) return await callAI(prompt, modelRuntime, {
+    if (options.stream && options.onChunk) return await callGenerativeAI(prompt, modelRuntime, {
         stream: true,
         onChunk: options.onChunk
     });

package/dist/es/ai-model/workflows/recorder-generation/playwright.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/workflows/recorder-generation/playwright.mjs","sources":["../../../../../src/ai-model/workflows/recorder-generation/playwright.ts"],"sourcesContent":["import type { ConversationMessage } from '@/ai-model/service-caller/types';\nimport type {\n  StreamingAIResponse,\n  StreamingCodeGenerationOptions,\n} from '@/types';\nimport { PLAYWRIGHT_EXAMPLE_CODE } from '@midscene/shared/constants';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport { getModelRuntime } from '../../models';\nimport { callAI, callAIWithStringResponse } from '../../service-caller';\n// Import shared utilities and types from yaml generation.\nimport {\n  type ChromeRecordedEvent,\n  type EventCounts,\n  type EventSummary,\n  type InputDescription,\n  type ProcessedEvent,\n  createEventCounts,\n  createMessageContent,\n  extractInputDescriptions,\n  filterEventsByType,\n  getScreenshotsForLLM,\n  prepareEventSummary,\n  processEventsForLLM,\n  validateEvents,\n} from './common';\n\n// Playwright-specific interfaces\nexport interface PlaywrightGenerationOptions {\n  testName?: string;\n  includeScreenshots?: boolean;\n  includeTimestamps?: boolean;\n  maxScreenshots?: number;\n  description?: string;\n  viewportSize?: { width: number; height: number };\n  waitForNetworkIdle?: boolean;\n  waitForNetworkIdleTimeout?: number;\n}\n\n// Re-export shared types for backward compatibility\nexport type {\n  ChromeRecordedEvent,\n  EventCounts,\n  InputDescription,\n  ProcessedEvent,\n  EventSummary,\n};\n\n// Re-export shared utilities for backward compatibility\nexport {\n  getScreenshotsForLLM,\n  filterEventsByType,\n  createEventCounts,\n  extractInputDescriptions,\n  processEventsForLLM,\n  prepareEventSummary,\n  createMessageContent,\n  validateEvents,\n};\n\nfunction createModelRuntime(modelConfig: IModelConfig) {\n  return getModelRuntime(modelConfig);\n}\n\n/**\n * Generates Playwright test code from recorded events\n */\nexport const generatePlaywrightTest = async (\n  events: ChromeRecordedEvent[],\n  options: PlaywrightGenerationOptions,\n  modelConfig: IModelConfig,\n): Promise<string> => {\n  const modelRuntime = createModelRuntime(modelConfig);\n\n  // Validate input\n  validateEvents(events);\n\n  // Prepare event summary using shared utilities\n  const summary = prepareEventSummary(events, {\n    testName: options.testName,\n    maxScreenshots: options.maxScreenshots || 3,\n  });\n\n  // Add Playwright-specific options to summary\n  const playwrightSummary = {\n    ...summary,\n    waitForNetworkIdle: options.waitForNetworkIdle !== false,\n    waitForNetworkIdleTimeout: options.waitForNetworkIdleTimeout || 2000,\n    viewportSize: options.viewportSize || { width: 1280, height: 800 },\n  };\n\n  // Get screenshots for visual context\n  const screenshots = getScreenshotsForLLM(events, options.maxScreenshots || 3);\n\n  // Create prompt text\n  const promptText = `Generate a Playwright test using @midscene/web/playwright that reproduces this recorded browser session. The test should be based on the following events and follow the structure of the example provided. Make the test descriptive with appropriate assertions and validations.\n\nEvent Summary:\n${JSON.stringify(playwrightSummary, null, 2)}\n\nGenerated code should:\n1. Import required dependencies\n2. Set up the test with proper configuration\n3. Include a beforeEach hook to navigate to the starting URL\n4. Implement a test that uses Midscene AI methods (aiTap, aiInput, aiAssert, etc.)\n5. Include appropriate assertions and validations\n6. Follow best practices for Playwright tests\n7. Be ready to execute without further modification\n\nImportant: Return ONLY the raw Playwright test code. Do NOT wrap the response in markdown code blocks (no \\`\\`\\`typescript, \\`\\`\\`javascript or \\`\\`\\`). Start directly with the code content.`;\n\n  // Create message content with screenshots\n  const messageContent = createMessageContent(\n    promptText,\n    screenshots,\n    options.includeScreenshots !== false,\n  );\n\n  // Create system prompt\n  const systemPrompt = `You are an expert test automation engineer specializing in Playwright and Midscene. \nYour task is to generate a complete, executable Playwright test using @midscene/web/playwright that reproduces a recorded browser session.\n\n${PLAYWRIGHT_EXAMPLE_CODE}`;\n\n  // Use LLM to generate the Playwright test code\n  const prompt: ConversationMessage[] = [\n    {\n      role: 'system',\n      content: systemPrompt,\n    },\n    {\n      role: 'user',\n      content: messageContent,\n    },\n  ];\n\n  const response = await callAIWithStringResponse(prompt, modelRuntime);\n\n  if (response?.content && typeof response.content === 'string') {\n    return response.content;\n  }\n\n  throw new Error('Failed to generate Playwright test code');\n};\n\n/**\n * Generates Playwright test code from recorded events with streaming support\n */\nexport const generatePlaywrightTestStream = async (\n  events: ChromeRecordedEvent[],\n  options: PlaywrightGenerationOptions & StreamingCodeGenerationOptions,\n  modelConfig: IModelConfig,\n): Promise<StreamingAIResponse> => {\n  const modelRuntime = createModelRuntime(modelConfig);\n\n  // Validate input\n  validateEvents(events);\n\n  // Prepare event summary using shared utilities\n  const summary = prepareEventSummary(events, {\n    testName: options.testName,\n    maxScreenshots: options.maxScreenshots || 3,\n  });\n\n  // Add Playwright-specific options to summary\n  const playwrightSummary = {\n    ...summary,\n    waitForNetworkIdle: options.waitForNetworkIdle !== false,\n    waitForNetworkIdleTimeout: options.waitForNetworkIdleTimeout || 2000,\n    viewportSize: options.viewportSize || { width: 1280, height: 800 },\n  };\n\n  // Get screenshots for visual context\n  const screenshots = getScreenshotsForLLM(events, options.maxScreenshots || 3);\n\n  // Create prompt text\n  const promptText = `Generate a Playwright test using @midscene/web/playwright that reproduces this recorded browser session. The test should be based on the following events and follow the structure of the example provided. Make the test descriptive with appropriate assertions and validations.\n\nEvent Summary:\n${JSON.stringify(playwrightSummary, null, 2)}\n\nGenerated code should:\n1. Import required dependencies\n2. Set up the test with proper configuration\n3. Include a beforeEach hook to navigate to the starting URL\n4. Implement a test that uses Midscene AI methods (aiTap, aiInput, aiAssert, etc.)\n5. Include appropriate assertions and validations\n6. Follow best practices for Playwright tests\n7. Be ready to execute without further modification\n8. can't wrap this test code in markdown code block\n\nImportant: Return ONLY the raw Playwright test code. Do NOT wrap the response in markdown code blocks (no \\`\\`\\`typescript, \\`\\`\\`javascript or \\`\\`\\`). Start directly with the code content.`;\n\n  // Create message content with screenshots\n  const messageContent = createMessageContent(\n    promptText,\n    screenshots,\n    options.includeScreenshots !== false,\n  );\n\n  // Create system prompt\n  const systemPrompt = `You are an expert test automation engineer specializing in Playwright and Midscene. \nYour task is to generate a complete, executable Playwright test using @midscene/web/playwright that reproduces a recorded browser session.\n\n${PLAYWRIGHT_EXAMPLE_CODE}`;\n\n  // Use LLM to generate the Playwright test code with streaming\n  const prompt: ConversationMessage[] = [\n    {\n      role: 'system',\n      content: systemPrompt,\n    },\n    {\n      role: 'user',\n      content: messageContent,\n    },\n  ];\n\n  if (options.stream && options.onChunk) {\n    // Use streaming\n    return await callAI(prompt, modelRuntime, {\n      stream: true,\n      onChunk: options.onChunk,\n    });\n  } else {\n    // Fallback to non-streaming\n    const response = await callAIWithStringResponse(prompt, modelRuntime);\n\n    if (response?.content && typeof response.content === 'string') {\n      return {\n        content: response.content,\n        usage: response.usage,\n        isStreamed: false,\n      };\n    }\n\n    throw new Error('Failed to generate Playwright test code');\n  }\n};\n"],"names":["createModelRuntime","modelConfig","getModelRuntime","generatePlaywrightTest","events","options","modelRuntime","validateEvents","summary","prepareEventSummary","playwrightSummary","screenshots","getScreenshotsForLLM","promptText","JSON","messageContent","createMessageContent","systemPrompt","PLAYWRIGHT_EXAMPLE_CODE","prompt","response","callAIWithStringResponse","Error","generatePlaywrightTestStream","callAI"],"mappings":";;;;AA2DA,SAASA,mBAAmBC,WAAyB;IACnD,OAAOC,gBAAgBD;AACzB;AAKO,MAAME,yBAAyB,OACpCC,QACAC,SACAJ;IAEA,MAAMK,eAAeN,mBAAmBC;IAGxCM,eAAeH;IAGf,MAAMI,UAAUC,oBAAoBL,QAAQ;QAC1C,UAAUC,QAAQ,QAAQ;QAC1B,gBAAgBA,QAAQ,cAAc,IAAI;IAC5C;IAGA,MAAMK,oBAAoB;QACxB,GAAGF,OAAO;QACV,oBAAoBH,AAA+B,UAA/BA,QAAQ,kBAAkB;QAC9C,2BAA2BA,QAAQ,yBAAyB,IAAI;QAChE,cAAcA,QAAQ,YAAY,IAAI;YAAE,OAAO;YAAM,QAAQ;QAAI;IACnE;IAGA,MAAMM,cAAcC,qBAAqBR,QAAQC,QAAQ,cAAc,IAAI;IAG3E,MAAMQ,aAAa,CAAC;;;AAGtB,EAAEC,KAAK,SAAS,CAACJ,mBAAmB,MAAM,GAAG;;;;;;;;;;;8LAWiJ,CAAC;IAG7L,MAAMK,iBAAiBC,qBACrBH,YACAF,aACAN,AAA+B,UAA/BA,QAAQ,kBAAkB;IAI5B,MAAMY,eAAe,CAAC;;;AAGxB,EAAEC,yBAAyB;IAGzB,MAAMC,SAAgC;QACpC;YACE,MAAM;YACN,SAASF;QACX;QACA;YACE,MAAM;YACN,SAASF;QACX;KACD;IAED,MAAMK,WAAW,MAAMC,yBAAyBF,QAAQb;IAExD,IAAIc,UAAU,WAAW,AAA4B,YAA5B,OAAOA,SAAS,OAAO,EAC9C,OAAOA,SAAS,OAAO;IAGzB,MAAM,IAAIE,MAAM;AAClB;AAKO,MAAMC,+BAA+B,OAC1CnB,QACAC,SACAJ;IAEA,MAAMK,eAAeN,mBAAmBC;IAGxCM,eAAeH;IAGf,MAAMI,UAAUC,oBAAoBL,QAAQ;QAC1C,UAAUC,QAAQ,QAAQ;QAC1B,gBAAgBA,QAAQ,cAAc,IAAI;IAC5C;IAGA,MAAMK,oBAAoB;QACxB,GAAGF,OAAO;QACV,oBAAoBH,AAA+B,UAA/BA,QAAQ,kBAAkB;QAC9C,2BAA2BA,QAAQ,yBAAyB,IAAI;QAChE,cAAcA,QAAQ,YAAY,IAAI;YAAE,OAAO;YAAM,QAAQ;QAAI;IACnE;IAGA,MAAMM,cAAcC,qBAAqBR,QAAQC,QAAQ,cAAc,IAAI;IAG3E,MAAMQ,aAAa,CAAC;;;AAGtB,EAAEC,KAAK,SAAS,CAACJ,mBAAmB,MAAM,GAAG;;;;;;;;;;;;8LAYiJ,CAAC;IAG7L,MAAMK,iBAAiBC,qBACrBH,YACAF,aACAN,AAA+B,UAA/BA,QAAQ,kBAAkB;IAI5B,MAAMY,eAAe,CAAC;;;AAGxB,EAAEC,yBAAyB;IAGzB,MAAMC,SAAgC;QACpC;YACE,MAAM;YACN,SAASF;QACX;QACA;YACE,MAAM;YACN,SAASF;QACX;KACD;IAED,IAAIV,QAAQ,MAAM,IAAIA,QAAQ,OAAO,EAEnC,OAAO,MAAMmB,OAAOL,QAAQb,cAAc;QACxC,QAAQ;QACR,SAASD,QAAQ,OAAO;IAC1B;IACK;QAEL,MAAMe,WAAW,MAAMC,yBAAyBF,QAAQb;QAExD,IAAIc,UAAU,WAAW,AAA4B,YAA5B,OAAOA,SAAS,OAAO,EAC9C,OAAO;YACL,SAASA,SAAS,OAAO;YACzB,OAAOA,SAAS,KAAK;YACrB,YAAY;QACd;QAGF,MAAM,IAAIE,MAAM;IAClB;AACF"}
+{"version":3,"file":"ai-model/workflows/recorder-generation/playwright.mjs","sources":["../../../../../src/ai-model/workflows/recorder-generation/playwright.ts"],"sourcesContent":["import type { ConversationMessage } from '@/ai-model/service-caller/types';\nimport type {\n  StreamingAIResponse,\n  StreamingCodeGenerationOptions,\n} from '@/types';\nimport { PLAYWRIGHT_EXAMPLE_CODE } from '@midscene/shared/constants';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport { getModelRuntime } from '../../models';\nimport {\n  callAIWithStringResponse,\n  callGenerativeAI,\n} from '../../service-caller';\n// Import shared utilities and types from yaml generation.\nimport {\n  type ChromeRecordedEvent,\n  type EventCounts,\n  type EventSummary,\n  type InputDescription,\n  type ProcessedEvent,\n  createEventCounts,\n  createMessageContent,\n  extractInputDescriptions,\n  filterEventsByType,\n  getScreenshotsForLLM,\n  prepareEventSummary,\n  processEventsForLLM,\n  validateEvents,\n} from './common';\n\n// Playwright-specific interfaces\nexport interface PlaywrightGenerationOptions {\n  testName?: string;\n  includeScreenshots?: boolean;\n  includeTimestamps?: boolean;\n  maxScreenshots?: number;\n  description?: string;\n  viewportSize?: { width: number; height: number };\n  waitForNetworkIdle?: boolean;\n  waitForNetworkIdleTimeout?: number;\n}\n\n// Re-export shared types for backward compatibility\nexport type {\n  ChromeRecordedEvent,\n  EventCounts,\n  InputDescription,\n  ProcessedEvent,\n  EventSummary,\n};\n\n// Re-export shared utilities for backward compatibility\nexport {\n  getScreenshotsForLLM,\n  filterEventsByType,\n  createEventCounts,\n  extractInputDescriptions,\n  processEventsForLLM,\n  prepareEventSummary,\n  createMessageContent,\n  validateEvents,\n};\n\nfunction createModelRuntime(modelConfig: IModelConfig) {\n  return getModelRuntime(modelConfig);\n}\n\n/**\n * Generates Playwright test code from recorded events\n */\nexport const generatePlaywrightTest = async (\n  events: ChromeRecordedEvent[],\n  options: PlaywrightGenerationOptions,\n  modelConfig: IModelConfig,\n): Promise<string> => {\n  const modelRuntime = createModelRuntime(modelConfig);\n\n  // Validate input\n  validateEvents(events);\n\n  // Prepare event summary using shared utilities\n  const summary = prepareEventSummary(events, {\n    testName: options.testName,\n    maxScreenshots: options.maxScreenshots || 3,\n  });\n\n  // Add Playwright-specific options to summary\n  const playwrightSummary = {\n    ...summary,\n    waitForNetworkIdle: options.waitForNetworkIdle !== false,\n    waitForNetworkIdleTimeout: options.waitForNetworkIdleTimeout || 2000,\n    viewportSize: options.viewportSize || { width: 1280, height: 800 },\n  };\n\n  // Get screenshots for visual context\n  const screenshots = getScreenshotsForLLM(events, options.maxScreenshots || 3);\n\n  // Create prompt text\n  const promptText = `Generate a Playwright test using @midscene/web/playwright that reproduces this recorded browser session. The test should be based on the following events and follow the structure of the example provided. Make the test descriptive with appropriate assertions and validations.\n\nEvent Summary:\n${JSON.stringify(playwrightSummary, null, 2)}\n\nGenerated code should:\n1. Import required dependencies\n2. Set up the test with proper configuration\n3. Include a beforeEach hook to navigate to the starting URL\n4. Implement a test that uses Midscene AI methods (aiTap, aiInput, aiAssert, etc.)\n5. Include appropriate assertions and validations\n6. Follow best practices for Playwright tests\n7. Be ready to execute without further modification\n\nImportant: Return ONLY the raw Playwright test code. Do NOT wrap the response in markdown code blocks (no \\`\\`\\`typescript, \\`\\`\\`javascript or \\`\\`\\`). Start directly with the code content.`;\n\n  // Create message content with screenshots\n  const messageContent = createMessageContent(\n    promptText,\n    screenshots,\n    options.includeScreenshots !== false,\n  );\n\n  // Create system prompt\n  const systemPrompt = `You are an expert test automation engineer specializing in Playwright and Midscene. \nYour task is to generate a complete, executable Playwright test using @midscene/web/playwright that reproduces a recorded browser session.\n\n${PLAYWRIGHT_EXAMPLE_CODE}`;\n\n  // Use LLM to generate the Playwright test code\n  const prompt: ConversationMessage[] = [\n    {\n      role: 'system',\n      content: systemPrompt,\n    },\n    {\n      role: 'user',\n      content: messageContent,\n    },\n  ];\n\n  const response = await callAIWithStringResponse(prompt, modelRuntime);\n\n  if (response?.content && typeof response.content === 'string') {\n    return response.content;\n  }\n\n  throw new Error('Failed to generate Playwright test code');\n};\n\n/**\n * Generates Playwright test code from recorded events with streaming support\n */\nexport const generatePlaywrightTestStream = async (\n  events: ChromeRecordedEvent[],\n  options: PlaywrightGenerationOptions & StreamingCodeGenerationOptions,\n  modelConfig: IModelConfig,\n): Promise<StreamingAIResponse> => {\n  const modelRuntime = createModelRuntime(modelConfig);\n\n  // Validate input\n  validateEvents(events);\n\n  // Prepare event summary using shared utilities\n  const summary = prepareEventSummary(events, {\n    testName: options.testName,\n    maxScreenshots: options.maxScreenshots || 3,\n  });\n\n  // Add Playwright-specific options to summary\n  const playwrightSummary = {\n    ...summary,\n    waitForNetworkIdle: options.waitForNetworkIdle !== false,\n    waitForNetworkIdleTimeout: options.waitForNetworkIdleTimeout || 2000,\n    viewportSize: options.viewportSize || { width: 1280, height: 800 },\n  };\n\n  // Get screenshots for visual context\n  const screenshots = getScreenshotsForLLM(events, options.maxScreenshots || 3);\n\n  // Create prompt text\n  const promptText = `Generate a Playwright test using @midscene/web/playwright that reproduces this recorded browser session. The test should be based on the following events and follow the structure of the example provided. Make the test descriptive with appropriate assertions and validations.\n\nEvent Summary:\n${JSON.stringify(playwrightSummary, null, 2)}\n\nGenerated code should:\n1. Import required dependencies\n2. Set up the test with proper configuration\n3. Include a beforeEach hook to navigate to the starting URL\n4. Implement a test that uses Midscene AI methods (aiTap, aiInput, aiAssert, etc.)\n5. Include appropriate assertions and validations\n6. Follow best practices for Playwright tests\n7. Be ready to execute without further modification\n8. can't wrap this test code in markdown code block\n\nImportant: Return ONLY the raw Playwright test code. Do NOT wrap the response in markdown code blocks (no \\`\\`\\`typescript, \\`\\`\\`javascript or \\`\\`\\`). Start directly with the code content.`;\n\n  // Create message content with screenshots\n  const messageContent = createMessageContent(\n    promptText,\n    screenshots,\n    options.includeScreenshots !== false,\n  );\n\n  // Create system prompt\n  const systemPrompt = `You are an expert test automation engineer specializing in Playwright and Midscene. \nYour task is to generate a complete, executable Playwright test using @midscene/web/playwright that reproduces a recorded browser session.\n\n${PLAYWRIGHT_EXAMPLE_CODE}`;\n\n  // Use LLM to generate the Playwright test code with streaming\n  const prompt: ConversationMessage[] = [\n    {\n      role: 'system',\n      content: systemPrompt,\n    },\n    {\n      role: 'user',\n      content: messageContent,\n    },\n  ];\n\n  if (options.stream && options.onChunk) {\n    // Use streaming\n    return await callGenerativeAI(prompt, modelRuntime, {\n      stream: true,\n      onChunk: options.onChunk,\n    });\n  } else {\n    // Fallback to non-streaming\n    const response = await callAIWithStringResponse(prompt, modelRuntime);\n\n    if (response?.content && typeof response.content === 'string') {\n      return {\n        content: response.content,\n        usage: response.usage,\n        isStreamed: false,\n      };\n    }\n\n    throw new Error('Failed to generate Playwright test code');\n  }\n};\n"],"names":["createModelRuntime","modelConfig","getModelRuntime","generatePlaywrightTest","events","options","modelRuntime","validateEvents","summary","prepareEventSummary","playwrightSummary","screenshots","getScreenshotsForLLM","promptText","JSON","messageContent","createMessageContent","systemPrompt","PLAYWRIGHT_EXAMPLE_CODE","prompt","response","callAIWithStringResponse","Error","generatePlaywrightTestStream","callGenerativeAI"],"mappings":";;;;AA8DA,SAASA,mBAAmBC,WAAyB;IACnD,OAAOC,gBAAgBD;AACzB;AAKO,MAAME,yBAAyB,OACpCC,QACAC,SACAJ;IAEA,MAAMK,eAAeN,mBAAmBC;IAGxCM,eAAeH;IAGf,MAAMI,UAAUC,oBAAoBL,QAAQ;QAC1C,UAAUC,QAAQ,QAAQ;QAC1B,gBAAgBA,QAAQ,cAAc,IAAI;IAC5C;IAGA,MAAMK,oBAAoB;QACxB,GAAGF,OAAO;QACV,oBAAoBH,AAA+B,UAA/BA,QAAQ,kBAAkB;QAC9C,2BAA2BA,QAAQ,yBAAyB,IAAI;QAChE,cAAcA,QAAQ,YAAY,IAAI;YAAE,OAAO;YAAM,QAAQ;QAAI;IACnE;IAGA,MAAMM,cAAcC,qBAAqBR,QAAQC,QAAQ,cAAc,IAAI;IAG3E,MAAMQ,aAAa,CAAC;;;AAGtB,EAAEC,KAAK,SAAS,CAACJ,mBAAmB,MAAM,GAAG;;;;;;;;;;;8LAWiJ,CAAC;IAG7L,MAAMK,iBAAiBC,qBACrBH,YACAF,aACAN,AAA+B,UAA/BA,QAAQ,kBAAkB;IAI5B,MAAMY,eAAe,CAAC;;;AAGxB,EAAEC,yBAAyB;IAGzB,MAAMC,SAAgC;QACpC;YACE,MAAM;YACN,SAASF;QACX;QACA;YACE,MAAM;YACN,SAASF;QACX;KACD;IAED,MAAMK,WAAW,MAAMC,yBAAyBF,QAAQb;IAExD,IAAIc,UAAU,WAAW,AAA4B,YAA5B,OAAOA,SAAS,OAAO,EAC9C,OAAOA,SAAS,OAAO;IAGzB,MAAM,IAAIE,MAAM;AAClB;AAKO,MAAMC,+BAA+B,OAC1CnB,QACAC,SACAJ;IAEA,MAAMK,eAAeN,mBAAmBC;IAGxCM,eAAeH;IAGf,MAAMI,UAAUC,oBAAoBL,QAAQ;QAC1C,UAAUC,QAAQ,QAAQ;QAC1B,gBAAgBA,QAAQ,cAAc,IAAI;IAC5C;IAGA,MAAMK,oBAAoB;QACxB,GAAGF,OAAO;QACV,oBAAoBH,AAA+B,UAA/BA,QAAQ,kBAAkB;QAC9C,2BAA2BA,QAAQ,yBAAyB,IAAI;QAChE,cAAcA,QAAQ,YAAY,IAAI;YAAE,OAAO;YAAM,QAAQ;QAAI;IACnE;IAGA,MAAMM,cAAcC,qBAAqBR,QAAQC,QAAQ,cAAc,IAAI;IAG3E,MAAMQ,aAAa,CAAC;;;AAGtB,EAAEC,KAAK,SAAS,CAACJ,mBAAmB,MAAM,GAAG;;;;;;;;;;;;8LAYiJ,CAAC;IAG7L,MAAMK,iBAAiBC,qBACrBH,YACAF,aACAN,AAA+B,UAA/BA,QAAQ,kBAAkB;IAI5B,MAAMY,eAAe,CAAC;;;AAGxB,EAAEC,yBAAyB;IAGzB,MAAMC,SAAgC;QACpC;YACE,MAAM;YACN,SAASF;QACX;QACA;YACE,MAAM;YACN,SAASF;QACX;KACD;IAED,IAAIV,QAAQ,MAAM,IAAIA,QAAQ,OAAO,EAEnC,OAAO,MAAMmB,iBAAiBL,QAAQb,cAAc;QAClD,QAAQ;QACR,SAASD,QAAQ,OAAO;IAC1B;IACK;QAEL,MAAMe,WAAW,MAAMC,yBAAyBF,QAAQb;QAExD,IAAIc,UAAU,WAAW,AAA4B,YAA5B,OAAOA,SAAS,OAAO,EAC9C,OAAO;YACL,SAASA,SAAS,OAAO;YACzB,OAAOA,SAAS,KAAK;YACrB,YAAY;QACd;QAGF,MAAM,IAAIE,MAAM;IAClB;AACF"}

package/dist/es/ai-model/workflows/recorder-generation/yaml.mjs

@@ -1,7 +1,7 @@
 import { YAML_EXAMPLE_CODE } from "@midscene/shared/constants";
 import { stringifyMidsceneRecorderTargetBlock } from "@midscene/shared/recorder";
 import { getModelRuntime } from "../../models/index.mjs";
-import { callAI, callAIWithStringResponse } from "../../service-caller/index.mjs";
+import { callAIWithStringResponse, callGenerativeAI } from "../../service-caller/index.mjs";
 import { createEventCounts, createMessageContent, extractInputDescriptions, filterEventsByType, getScreenshotsForLLM, prepareEventSummary, prepareRecorderGenerationContext, processEventsForLLM, validateEvents } from "./common.mjs";
 const getYamlLanguageInstruction = (language)=>{
     const normalizedLanguage = language?.trim();
@@ -130,7 +130,7 @@ const generateRecorderYamlTestStream = async (input, options, modelConfig)=>{
     try {
         const prompt = createRecorderYamlPrompt(input);
         const modelRuntime = createModelRuntime(modelConfig);
-        if (options.stream && options.onChunk) return await callAI(prompt, modelRuntime, {
+        if (options.stream && options.onChunk) return await callGenerativeAI(prompt, modelRuntime, {
             stream: true,
             onChunk: options.onChunk
         });