@midscene/core 1.10.2 → 1.10.3

package/dist/es/ai-model/models/doubao.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/models/doubao.mjs","sources":["../../../../src/ai-model/models/doubao.ts"],"sourcesContent":["import type { TModelFamily } from '@midscene/shared/env';\nimport { assert } from '@midscene/shared/utils';\nimport { jsonrepair } from 'jsonrepair';\nimport type {\n  ChatCompletionCallContext,\n  ChatCompletionParamsResult,\n  JsonParserContext,\n  JsonParserSource,\n  ModelAdapterDefinition,\n} from '../model-adapter/types';\nimport {\n  extractJSONFromCodeBlock,\n  safeParseJson,\n} from '../service-caller/json';\nimport {\n  type LocateResultValue,\n  createLocateResultValue,\n  unwrapCoordinateListLikeInput,\n} from '../shared/model-locate-result';\n\nconst doubaoBboxCoordinatesMeta = {\n  shape: 'bbox',\n  order: 'xy',\n  normalizedBy: 1000,\n} as const;\nconst doubaoPointCoordinatesMeta = {\n  shape: 'point',\n  order: 'xy',\n  normalizedBy: 1000,\n} as const;\n\nexport function normalizeDoubaoJsonObject(\n  obj: any,\n  context: Pick<JsonParserContext, 'preserveStringValueKeys'> = {},\n): any {\n  if (obj === null || obj === undefined) {\n    return obj;\n  }\n\n  if (Array.isArray(obj)) {\n    return obj.map((item) => normalizeDoubaoJsonObject(item, context));\n  }\n\n  if (typeof obj === 'object') {\n    const normalized: any = {};\n    for (const [key, value] of Object.entries(obj)) {\n      const trimmedKey = key.trim();\n      const preserveStringValue =\n        context.preserveStringValueKeys?.includes(trimmedKey) ?? false;\n      const normalizedValue =\n        typeof value === 'string'\n          ? preserveStringValue\n            ? value\n            : value.trim()\n          : normalizeDoubaoJsonObject(value, context);\n      normalized[trimmedKey] = normalizedValue;\n    }\n    return normalized;\n  }\n\n  return typeof obj === 'string' ? obj.trim() : obj;\n}\n\nexport function shouldRepairDoubaoLocateJson(source: JsonParserSource) {\n  return (\n    source === 'locate' ||\n    source === 'section-locator' ||\n    source === 'planning-action-param'\n  );\n}\n\nexport function preprocessDoubaoLocateJson(input: string) {\n  if (input.includes('bbox')) {\n    while (/\\d+\\s+\\d+/.test(input)) {\n      input = input.replace(/(\\d+)\\s+(\\d+)/g, '$1,$2');\n    }\n  }\n  return input;\n}\n\nconst doubaoJsonParser: ModelAdapterDefinition['jsonParser'] = (\n  raw,\n  context = { source: 'generic-object' },\n) => {\n  const { source } = context;\n  try {\n    return safeParseJson(raw, context);\n  } catch (firstError) {\n    if (!shouldRepairDoubaoLocateJson(source)) {\n      throw firstError;\n    }\n\n    const jsonString = preprocessDoubaoLocateJson(\n      extractJSONFromCodeBlock(raw),\n    );\n    try {\n      return normalizeDoubaoJsonObject(\n        JSON.parse(jsonrepair(jsonString)),\n        context,\n      );\n    } catch (error) {\n      throw Error(\n        `failed to parse LLM response into JSON. Error - ${String(\n          error ?? firstError ?? 'unknown error',\n        )}. Response - \\n ${raw}`,\n      );\n    }\n  }\n};\n\nexport function parseDoubaoRawLocateValue(input: unknown): LocateResultValue {\n  const bbox = unwrapCoordinateListLikeInput(input as any);\n  if (typeof bbox === 'string') {\n    assert(\n      /^(\\d+)\\s(\\d+)\\s(\\d+)\\s(\\d+)$/.test(bbox.trim()),\n      `invalid bbox data string for doubao-vision mode: ${bbox}`,\n    );\n    const splitted = bbox.split(' ');\n    if (splitted.length === 4) {\n      return createLocateResultValue(doubaoBboxCoordinatesMeta, [\n        Number(splitted[0]),\n        Number(splitted[1]),\n        Number(splitted[2]),\n        Number(splitted[3]),\n      ]);\n    }\n    throw new Error(`invalid bbox data string for doubao-vision mode: ${bbox}`);\n  }\n\n  let bboxList: number[] = [];\n  if (Array.isArray(bbox) && typeof bbox[0] === 'string') {\n    bbox.forEach((item) => {\n      if (typeof item === 'string' && item.includes(',')) {\n        const [x, y] = item.split(',');\n        bboxList.push(Number(x.trim()), Number(y.trim()));\n      } else if (typeof item === 'string' && item.includes(' ')) {\n        const [x, y] = item.split(' ');\n        bboxList.push(Number(x.trim()), Number(y.trim()));\n      } else {\n        bboxList.push(Number(item));\n      }\n    });\n  } else {\n    bboxList = bbox as number[];\n  }\n\n  if (bboxList.length === 4 || bboxList.length === 5) {\n    return createLocateResultValue(doubaoBboxCoordinatesMeta, [\n      bboxList[0],\n      bboxList[1],\n      bboxList[2],\n      bboxList[3],\n    ]);\n  }\n\n  if (\n    bboxList.length === 6 ||\n    bboxList.length === 2 ||\n    bboxList.length === 3 ||\n    bboxList.length === 7\n  ) {\n    return createLocateResultValue(doubaoPointCoordinatesMeta, [\n      bboxList[0],\n      bboxList[1],\n    ]);\n  }\n\n  if (bbox.length === 8) {\n    return createLocateResultValue(doubaoBboxCoordinatesMeta, [\n      bboxList[0],\n      bboxList[1],\n      bboxList[4],\n      bboxList[5],\n    ]);\n  }\n\n  const msg = `invalid bbox data for doubao-vision mode: ${JSON.stringify(bbox)} `;\n  throw new Error(msg);\n}\n\nconst buildDoubaoChatCompletionParams = (\n  input: ChatCompletionCallContext,\n): ChatCompletionParamsResult => {\n  const { midsceneDefaults, userConfig } = input;\n  const { reasoningEnabled, reasoningEffort } = userConfig;\n  const commonOverrideConfig: Record<string, unknown> = {};\n\n  if (userConfig.temperature !== undefined) {\n    commonOverrideConfig.temperature = userConfig.temperature;\n  }\n\n  const modelSpecificConfig: Record<string, unknown> = {};\n\n  if (reasoningEnabled !== 'default') {\n    modelSpecificConfig.thinking = {\n      type: (reasoningEnabled ?? false) ? 'enabled' : 'disabled',\n    };\n    if (reasoningEffort) {\n      modelSpecificConfig.reasoning_effort = reasoningEffort;\n    }\n  }\n\n  return {\n    config: {\n      ...midsceneDefaults,\n      ...commonOverrideConfig,\n      ...modelSpecificConfig,\n    },\n  };\n};\n\nconst doubaoVisionAdapter: ModelAdapterDefinition = {\n  jsonParser: doubaoJsonParser,\n  chatCompletion: {\n    unsupportedUserConfig: ['reasoningBudget'],\n    buildChatCompletionParams: buildDoubaoChatCompletionParams,\n    useReasoningAsContentFallback: true,\n  },\n  locate: {\n    resultAdapter: {\n      coordinates: doubaoBboxCoordinatesMeta,\n      parseRawLocateValue: parseDoubaoRawLocateValue,\n    },\n  },\n};\n\nexport const doubaoAdapters = {\n  'doubao-vision': doubaoVisionAdapter,\n  'doubao-seed': doubaoVisionAdapter,\n} satisfies Pick<\n  Record<TModelFamily, ModelAdapterDefinition>,\n  'doubao-vision' | 'doubao-seed'\n>;\n"],"names":["doubaoBboxCoordinatesMeta","doubaoPointCoordinatesMeta","normalizeDoubaoJsonObject","obj","context","Array","item","normalized","key","value","Object","trimmedKey","preserveStringValue","normalizedValue","shouldRepairDoubaoLocateJson","source","preprocessDoubaoLocateJson","input","doubaoJsonParser","raw","safeParseJson","firstError","jsonString","extractJSONFromCodeBlock","JSON","jsonrepair","error","Error","String","parseDoubaoRawLocateValue","bbox","unwrapCoordinateListLikeInput","assert","splitted","createLocateResultValue","Number","bboxList","x","y","msg","buildDoubaoChatCompletionParams","midsceneDefaults","userConfig","reasoningEnabled","reasoningEffort","commonOverrideConfig","undefined","modelSpecificConfig","doubaoVisionAdapter","doubaoAdapters"],"mappings":";;;;AAoBA,MAAMA,4BAA4B;IAChC,OAAO;IACP,OAAO;IACP,cAAc;AAChB;AACA,MAAMC,6BAA6B;IACjC,OAAO;IACP,OAAO;IACP,cAAc;AAChB;AAEO,SAASC,0BACdC,GAAQ,EACRC,UAA8D,CAAC,CAAC;IAEhE,IAAID,QAAAA,KACF,OAAOA;IAGT,IAAIE,MAAM,OAAO,CAACF,MAChB,OAAOA,IAAI,GAAG,CAAC,CAACG,OAASJ,0BAA0BI,MAAMF;IAG3D,IAAI,AAAe,YAAf,OAAOD,KAAkB;QAC3B,MAAMI,aAAkB,CAAC;QACzB,KAAK,MAAM,CAACC,KAAKC,MAAM,IAAIC,OAAO,OAAO,CAACP,KAAM;YAC9C,MAAMQ,aAAaH,IAAI,IAAI;YAC3B,MAAMI,sBACJR,QAAQ,uBAAuB,EAAE,SAASO,eAAe;YAC3D,MAAME,kBACJ,AAAiB,YAAjB,OAAOJ,QACHG,sBACEH,QACAA,MAAM,IAAI,KACZP,0BAA0BO,OAAOL;YACvCG,UAAU,CAACI,WAAW,GAAGE;QAC3B;QACA,OAAON;IACT;IAEA,OAAO,AAAe,YAAf,OAAOJ,MAAmBA,IAAI,IAAI,KAAKA;AAChD;AAEO,SAASW,6BAA6BC,MAAwB;IACnE,OACEA,AAAW,aAAXA,UACAA,AAAW,sBAAXA,UACAA,AAAW,4BAAXA;AAEJ;AAEO,SAASC,2BAA2BC,KAAa;IACtD,IAAIA,MAAM,QAAQ,CAAC,SACjB,MAAO,YAAY,IAAI,CAACA,OACtBA,QAAQA,MAAM,OAAO,CAAC,kBAAkB;IAG5C,OAAOA;AACT;AAEA,MAAMC,mBAAyD,CAC7DC,KACAf,UAAU;IAAE,QAAQ;AAAiB,CAAC;IAEtC,MAAM,EAAEW,MAAM,EAAE,GAAGX;IACnB,IAAI;QACF,OAAOgB,cAAcD,KAAKf;IAC5B,EAAE,OAAOiB,YAAY;QACnB,IAAI,CAACP,6BAA6BC,SAChC,MAAMM;QAGR,MAAMC,aAAaN,2BACjBO,yBAAyBJ;QAE3B,IAAI;YACF,OAAOjB,0BACLsB,KAAK,KAAK,CAACC,WAAWH,cACtBlB;QAEJ,EAAE,OAAOsB,OAAO;YACd,MAAMC,MACJ,CAAC,gDAAgD,EAAEC,OACjDF,SAASL,cAAc,iBACvB,gBAAgB,EAAEF,KAAK;QAE7B;IACF;AACF;AAEO,SAASU,0BAA0BZ,KAAc;IACtD,MAAMa,OAAOC,8BAA8Bd;IAC3C,IAAI,AAAgB,YAAhB,OAAOa,MAAmB;QAC5BE,OACE,+BAA+B,IAAI,CAACF,KAAK,IAAI,KAC7C,CAAC,iDAAiD,EAAEA,MAAM;QAE5D,MAAMG,WAAWH,KAAK,KAAK,CAAC;QAC5B,IAAIG,AAAoB,MAApBA,SAAS,MAAM,EACjB,OAAOC,wBAAwBlC,2BAA2B;YACxDmC,OAAOF,QAAQ,CAAC,EAAE;YAClBE,OAAOF,QAAQ,CAAC,EAAE;YAClBE,OAAOF,QAAQ,CAAC,EAAE;YAClBE,OAAOF,QAAQ,CAAC,EAAE;SACnB;QAEH,MAAM,IAAIN,MAAM,CAAC,iDAAiD,EAAEG,MAAM;IAC5E;IAEA,IAAIM,WAAqB,EAAE;IAC3B,IAAI/B,MAAM,OAAO,CAACyB,SAAS,AAAmB,YAAnB,OAAOA,IAAI,CAAC,EAAE,EACvCA,KAAK,OAAO,CAAC,CAACxB;QACZ,IAAI,AAAgB,YAAhB,OAAOA,QAAqBA,KAAK,QAAQ,CAAC,MAAM;YAClD,MAAM,CAAC+B,GAAGC,EAAE,GAAGhC,KAAK,KAAK,CAAC;YAC1B8B,SAAS,IAAI,CAACD,OAAOE,EAAE,IAAI,KAAKF,OAAOG,EAAE,IAAI;QAC/C,OAAO,IAAI,AAAgB,YAAhB,OAAOhC,QAAqBA,KAAK,QAAQ,CAAC,MAAM;YACzD,MAAM,CAAC+B,GAAGC,EAAE,GAAGhC,KAAK,KAAK,CAAC;YAC1B8B,SAAS,IAAI,CAACD,OAAOE,EAAE,IAAI,KAAKF,OAAOG,EAAE,IAAI;QAC/C,OACEF,SAAS,IAAI,CAACD,OAAO7B;IAEzB;SAEA8B,WAAWN;IAGb,IAAIM,AAAoB,MAApBA,SAAS,MAAM,IAAUA,AAAoB,MAApBA,SAAS,MAAM,EAC1C,OAAOF,wBAAwBlC,2BAA2B;QACxDoC,QAAQ,CAAC,EAAE;QACXA,QAAQ,CAAC,EAAE;QACXA,QAAQ,CAAC,EAAE;QACXA,QAAQ,CAAC,EAAE;KACZ;IAGH,IACEA,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,EAEf,OAAOF,wBAAwBjC,4BAA4B;QACzDmC,QAAQ,CAAC,EAAE;QACXA,QAAQ,CAAC,EAAE;KACZ;IAGH,IAAIN,AAAgB,MAAhBA,KAAK,MAAM,EACb,OAAOI,wBAAwBlC,2BAA2B;QACxDoC,QAAQ,CAAC,EAAE;QACXA,QAAQ,CAAC,EAAE;QACXA,QAAQ,CAAC,EAAE;QACXA,QAAQ,CAAC,EAAE;KACZ;IAGH,MAAMG,MAAM,CAAC,0CAA0C,EAAEf,KAAK,SAAS,CAACM,MAAM,CAAC,CAAC;IAChF,MAAM,IAAIH,MAAMY;AAClB;AAEA,MAAMC,kCAAkC,CACtCvB;IAEA,MAAM,EAAEwB,gBAAgB,EAAEC,UAAU,EAAE,GAAGzB;IACzC,MAAM,EAAE0B,gBAAgB,EAAEC,eAAe,EAAE,GAAGF;IAC9C,MAAMG,uBAAgD,CAAC;IAEvD,IAAIH,AAA2BI,WAA3BJ,WAAW,WAAW,EACxBG,qBAAqB,WAAW,GAAGH,WAAW,WAAW;IAG3D,MAAMK,sBAA+C,CAAC;IAEtD,IAAIJ,AAAqB,cAArBA,kBAAgC;QAClCI,oBAAoB,QAAQ,GAAG;YAC7B,MAAOJ,oBAAoB,QAAS,YAAY;QAClD;QACA,IAAIC,iBACFG,oBAAoB,gBAAgB,GAAGH;IAE3C;IAEA,OAAO;QACL,QAAQ;YACN,GAAGH,gBAAgB;YACnB,GAAGI,oBAAoB;YACvB,GAAGE,mBAAmB;QACxB;IACF;AACF;AAEA,MAAMC,sBAA8C;IAClD,YAAY9B;IACZ,gBAAgB;QACd,uBAAuB;YAAC;SAAkB;QAC1C,2BAA2BsB;QAC3B,+BAA+B;IACjC;IACA,QAAQ;QACN,eAAe;YACb,aAAaxC;YACb,qBAAqB6B;QACvB;IACF;AACF;AAEO,MAAMoB,iBAAiB;IAC5B,iBAAiBD;IACjB,eAAeA;AACjB"}
+{"version":3,"file":"ai-model/models/doubao.mjs","sources":["../../../../src/ai-model/models/doubao.ts"],"sourcesContent":["import type { TModelFamily } from '@midscene/shared/env';\nimport type {\n  ChatCompletionCallContext,\n  ChatCompletionParamsResult,\n  ModelAdapterDefinition,\n} from '../model-adapter/types';\nimport { parseModelResponseJson } from '../service-caller/json';\nimport {\n  type LocateResultValue,\n  createLocateResultValue,\n  unwrapCoordinateListLikeInput,\n} from '../shared/model-locate-result';\n\nconst doubaoBboxCoordinatesMeta = {\n  shape: 'bbox',\n  order: 'xy',\n  normalizedBy: 1000,\n} as const;\nconst doubaoPointCoordinatesMeta = {\n  shape: 'point',\n  order: 'xy',\n  normalizedBy: 1000,\n} as const;\n\nfunction parseNumbersFromBboxString(input: string): number[] {\n  return (input.match(/\\d+/g) ?? []).map(Number).filter(Number.isFinite);\n}\n\nfunction parseLeadingNumberFromString(input: string): number | undefined {\n  const match = input.match(/^\\s*(\\d+)/);\n  return match ? Number(match[1]) : undefined;\n}\n\n/**\n * Clean coordinate strings can contain multiple positive integers, e.g.\n * - \"123 100\"\n * - \"123,100\"\n * - \"277; 664 291;\"\n * Dirty strings like \"345<\" are handled by taking the leading number and\n * dropping the remaining array items.\n */\nfunction isCleanCoordinateString(input: string): boolean {\n  return /^\\s*\\d+(?:[\\s,;]+\\d+)*\\s*[,;]?\\s*$/.test(input);\n}\n\nfunction parseNumbersFromBboxArray(input: unknown[]): number[] {\n  const numbers: number[] = [];\n\n  for (const item of input) {\n    if (typeof item === 'number') {\n      numbers.push(item);\n      continue;\n    }\n\n    if (typeof item === 'string') {\n      if (isCleanCoordinateString(item)) {\n        numbers.push(...parseNumbersFromBboxString(item));\n        continue;\n      } else {\n        const leadingNumber = parseLeadingNumberFromString(item);\n        if (leadingNumber !== undefined) {\n          numbers.push(leadingNumber);\n        }\n        // Once a dirty string appears, the remaining array items usually belong\n        // to broken JSON repair output, e.g. [410, 295, 885, \"345<\", \"/bbox>...\"].\n        break;\n      }\n    }\n\n    break;\n  }\n\n  return numbers.filter(Number.isFinite);\n}\n\n/**\n * Some models return a tagged bbox tuple, e.g.\n * - [\"bbox\", [782, 541, 815, 559]]\n * - [\"bbox\", \"782, 541, 815, 559\"]\n */\nfunction parseNumbersFromTaggedBboxArray(input: unknown[]): number[] | null {\n  if (input.length < 2) {\n    return null;\n  }\n\n  const taggedBbox = input[1];\n  if (typeof taggedBbox === 'string') {\n    return parseNumbersFromBboxString(taggedBbox);\n  }\n\n  if (Array.isArray(taggedBbox)) {\n    return parseNumbersFromBboxArray(taggedBbox);\n  }\n\n  return null;\n}\n\nexport function parseDoubaoRawLocateValue(input: unknown): LocateResultValue {\n  const bbox = unwrapCoordinateListLikeInput(input as any);\n  let bboxList: number[] = [];\n\n  if (typeof bbox === 'string') {\n    /**\n     * Some models return bbox as a string, e.g.\n     * - { \"bbox\": \"[336, 163, 717, 200]\" }.\n     * - { \"bbox\": \"336, 163, 717, 200\" }.\n     * - { \"bbox\": \"336 163 717 200\" }.\n     */\n    bboxList = parseNumbersFromBboxString(bbox);\n    if (bboxList.length !== 4) {\n      throw new Error(\n        `invalid bbox data string for doubao-vision mode: ${bbox}`,\n      );\n    }\n  } else if (Array.isArray(bbox)) {\n    if (typeof bbox[0] === 'string' && bbox[0].trim() === 'bbox') {\n      bboxList = parseNumbersFromTaggedBboxArray(bbox) ?? [];\n    } else {\n      bboxList = parseNumbersFromBboxArray(bbox);\n    }\n  } else {\n    bboxList = bbox as number[];\n  }\n\n  if (bboxList.length === 4 || bboxList.length === 5) {\n    return createLocateResultValue(doubaoBboxCoordinatesMeta, [\n      bboxList[0],\n      bboxList[1],\n      bboxList[2],\n      bboxList[3],\n    ]);\n  }\n\n  if (\n    bboxList.length === 6 ||\n    bboxList.length === 2 ||\n    bboxList.length === 3 ||\n    bboxList.length === 7\n  ) {\n    return createLocateResultValue(doubaoPointCoordinatesMeta, [\n      bboxList[0],\n      bboxList[1],\n    ]);\n  }\n\n  if (bboxList.length === 8) {\n    return createLocateResultValue(doubaoBboxCoordinatesMeta, [\n      bboxList[0],\n      bboxList[1],\n      bboxList[4],\n      bboxList[5],\n    ]);\n  }\n\n  const msg = `invalid bbox data for doubao-vision mode: ${JSON.stringify(bbox)} `;\n  throw new Error(msg);\n}\n\nconst buildDoubaoChatCompletionParams = (\n  input: ChatCompletionCallContext,\n): ChatCompletionParamsResult => {\n  const { midsceneDefaults, userConfig } = input;\n  const { reasoningEnabled, reasoningEffort } = userConfig;\n  const commonOverrideConfig: Record<string, unknown> = {};\n\n  if (userConfig.temperature !== undefined) {\n    commonOverrideConfig.temperature = userConfig.temperature;\n  }\n\n  const modelSpecificConfig: Record<string, unknown> = {};\n\n  if (reasoningEnabled !== 'default') {\n    modelSpecificConfig.thinking = {\n      type: (reasoningEnabled ?? false) ? 'enabled' : 'disabled',\n    };\n    if (reasoningEffort) {\n      modelSpecificConfig.reasoning_effort = reasoningEffort;\n    }\n  }\n\n  return {\n    config: {\n      ...midsceneDefaults,\n      ...commonOverrideConfig,\n      ...modelSpecificConfig,\n    },\n  };\n};\n\nconst doubaoVisionAdapter: ModelAdapterDefinition = {\n  jsonParser: parseModelResponseJson,\n  chatCompletion: {\n    unsupportedUserConfig: ['reasoningBudget'],\n    buildChatCompletionParams: buildDoubaoChatCompletionParams,\n    useReasoningAsContentFallback: true,\n  },\n  locate: {\n    resultAdapter: {\n      coordinates: doubaoBboxCoordinatesMeta,\n      parseRawLocateValue: parseDoubaoRawLocateValue,\n    },\n  },\n};\n\nexport const doubaoAdapters = {\n  'doubao-vision': doubaoVisionAdapter,\n  'doubao-seed': doubaoVisionAdapter,\n} satisfies Pick<\n  Record<TModelFamily, ModelAdapterDefinition>,\n  'doubao-vision' | 'doubao-seed'\n>;\n"],"names":["doubaoBboxCoordinatesMeta","doubaoPointCoordinatesMeta","parseNumbersFromBboxString","input","Number","parseLeadingNumberFromString","match","undefined","isCleanCoordinateString","parseNumbersFromBboxArray","numbers","item","leadingNumber","parseNumbersFromTaggedBboxArray","taggedBbox","Array","parseDoubaoRawLocateValue","bbox","unwrapCoordinateListLikeInput","bboxList","Error","createLocateResultValue","msg","JSON","buildDoubaoChatCompletionParams","midsceneDefaults","userConfig","reasoningEnabled","reasoningEffort","commonOverrideConfig","modelSpecificConfig","doubaoVisionAdapter","parseModelResponseJson","doubaoAdapters"],"mappings":";;AAaA,MAAMA,4BAA4B;IAChC,OAAO;IACP,OAAO;IACP,cAAc;AAChB;AACA,MAAMC,6BAA6B;IACjC,OAAO;IACP,OAAO;IACP,cAAc;AAChB;AAEA,SAASC,2BAA2BC,KAAa;IAC/C,OAAQA,AAAAA,CAAAA,MAAM,KAAK,CAAC,WAAW,EAAC,EAAG,GAAG,CAACC,QAAQ,MAAM,CAACA,OAAO,QAAQ;AACvE;AAEA,SAASC,6BAA6BF,KAAa;IACjD,MAAMG,QAAQH,MAAM,KAAK,CAAC;IAC1B,OAAOG,QAAQF,OAAOE,KAAK,CAAC,EAAE,IAAIC;AACpC;AAUA,SAASC,wBAAwBL,KAAa;IAC5C,OAAO,qCAAqC,IAAI,CAACA;AACnD;AAEA,SAASM,0BAA0BN,KAAgB;IACjD,MAAMO,UAAoB,EAAE;IAE5B,KAAK,MAAMC,QAAQR,MAAO;QACxB,IAAI,AAAgB,YAAhB,OAAOQ,MAAmB;YAC5BD,QAAQ,IAAI,CAACC;YACb;QACF;QAEA,IAAI,AAAgB,YAAhB,OAAOA,MACT,IAAIH,wBAAwBG,OAAO;YACjCD,QAAQ,IAAI,IAAIR,2BAA2BS;YAC3C;QACF,OAAO;YACL,MAAMC,gBAAgBP,6BAA6BM;YACnD,IAAIC,AAAkBL,WAAlBK,eACFF,QAAQ,IAAI,CAACE;QAKjB;QAGF;IACF;IAEA,OAAOF,QAAQ,MAAM,CAACN,OAAO,QAAQ;AACvC;AAOA,SAASS,gCAAgCV,KAAgB;IACvD,IAAIA,MAAM,MAAM,GAAG,GACjB,OAAO;IAGT,MAAMW,aAAaX,KAAK,CAAC,EAAE;IAC3B,IAAI,AAAsB,YAAtB,OAAOW,YACT,OAAOZ,2BAA2BY;IAGpC,IAAIC,MAAM,OAAO,CAACD,aAChB,OAAOL,0BAA0BK;IAGnC,OAAO;AACT;AAEO,SAASE,0BAA0Bb,KAAc;IACtD,MAAMc,OAAOC,8BAA8Bf;IAC3C,IAAIgB,WAAqB,EAAE;IAE3B,IAAI,AAAgB,YAAhB,OAAOF,MAAmB;QAO5BE,WAAWjB,2BAA2Be;QACtC,IAAIE,AAAoB,MAApBA,SAAS,MAAM,EACjB,MAAM,IAAIC,MACR,CAAC,iDAAiD,EAAEH,MAAM;IAGhE,OAEIE,WAFOJ,MAAM,OAAO,CAACE,QACnB,AAAmB,YAAnB,OAAOA,IAAI,CAAC,EAAE,IAAiBA,AAAmB,WAAnBA,IAAI,CAAC,EAAE,CAAC,IAAI,KAClCJ,gCAAgCI,SAAS,EAAE,GAE3CR,0BAA0BQ,QAG5BA;IAGb,IAAIE,AAAoB,MAApBA,SAAS,MAAM,IAAUA,AAAoB,MAApBA,SAAS,MAAM,EAC1C,OAAOE,wBAAwBrB,2BAA2B;QACxDmB,QAAQ,CAAC,EAAE;QACXA,QAAQ,CAAC,EAAE;QACXA,QAAQ,CAAC,EAAE;QACXA,QAAQ,CAAC,EAAE;KACZ;IAGH,IACEA,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,EAEf,OAAOE,wBAAwBpB,4BAA4B;QACzDkB,QAAQ,CAAC,EAAE;QACXA,QAAQ,CAAC,EAAE;KACZ;IAGH,IAAIA,AAAoB,MAApBA,SAAS,MAAM,EACjB,OAAOE,wBAAwBrB,2BAA2B;QACxDmB,QAAQ,CAAC,EAAE;QACXA,QAAQ,CAAC,EAAE;QACXA,QAAQ,CAAC,EAAE;QACXA,QAAQ,CAAC,EAAE;KACZ;IAGH,MAAMG,MAAM,CAAC,0CAA0C,EAAEC,KAAK,SAAS,CAACN,MAAM,CAAC,CAAC;IAChF,MAAM,IAAIG,MAAME;AAClB;AAEA,MAAME,kCAAkC,CACtCrB;IAEA,MAAM,EAAEsB,gBAAgB,EAAEC,UAAU,EAAE,GAAGvB;IACzC,MAAM,EAAEwB,gBAAgB,EAAEC,eAAe,EAAE,GAAGF;IAC9C,MAAMG,uBAAgD,CAAC;IAEvD,IAAIH,AAA2BnB,WAA3BmB,WAAW,WAAW,EACxBG,qBAAqB,WAAW,GAAGH,WAAW,WAAW;IAG3D,MAAMI,sBAA+C,CAAC;IAEtD,IAAIH,AAAqB,cAArBA,kBAAgC;QAClCG,oBAAoB,QAAQ,GAAG;YAC7B,MAAOH,oBAAoB,QAAS,YAAY;QAClD;QACA,IAAIC,iBACFE,oBAAoB,gBAAgB,GAAGF;IAE3C;IAEA,OAAO;QACL,QAAQ;YACN,GAAGH,gBAAgB;YACnB,GAAGI,oBAAoB;YACvB,GAAGC,mBAAmB;QACxB;IACF;AACF;AAEA,MAAMC,sBAA8C;IAClD,YAAYC;IACZ,gBAAgB;QACd,uBAAuB;YAAC;SAAkB;QAC1C,2BAA2BR;QAC3B,+BAA+B;IACjC;IACA,QAAQ;QACN,eAAe;YACb,aAAaxB;YACb,qBAAqBgB;QACvB;IACF;AACF;AAEO,MAAMiB,iBAAiB;IAC5B,iBAAiBF;IACjB,eAAeA;AACjB"}

package/dist/es/ai-model/models/glm.mjs

@@ -1,8 +1,12 @@
+import { isLocateIntent } from "./utils/intent.mjs";
 const buildGlmChatCompletionParams = (input)=>{
     const { midsceneDefaults, userConfig } = input;
     const { reasoningEnabled } = userConfig;
     const commonOverrideConfig = {};
     if (void 0 !== userConfig.temperature) commonOverrideConfig.temperature = userConfig.temperature;
+    if (isLocateIntent(input.intent)) commonOverrideConfig.response_format = {
+        type: 'json_object'
+    };
     const modelSpecificConfig = {};
     if ('default' !== reasoningEnabled) modelSpecificConfig.thinking = {
         type: reasoningEnabled ?? false ? 'enabled' : 'disabled'

package/dist/es/ai-model/models/glm.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/models/glm.mjs","sources":["../../../../src/ai-model/models/glm.ts"],"sourcesContent":["import type { TModelFamily } from '@midscene/shared/env';\nimport type {\n  ChatCompletionCallContext,\n  ChatCompletionParamsResult,\n  ModelAdapterDefinition,\n} from '../model-adapter/types';\n\nconst buildGlmChatCompletionParams = (\n  input: ChatCompletionCallContext,\n): ChatCompletionParamsResult => {\n  const { midsceneDefaults, userConfig } = input;\n  const { reasoningEnabled } = userConfig;\n  const commonOverrideConfig: Record<string, unknown> = {};\n\n  if (userConfig.temperature !== undefined) {\n    commonOverrideConfig.temperature = userConfig.temperature;\n  }\n\n  const modelSpecificConfig: Record<string, unknown> = {};\n\n  if (reasoningEnabled !== 'default') {\n    modelSpecificConfig.thinking = {\n      type: (reasoningEnabled ?? false) ? 'enabled' : 'disabled',\n    };\n  }\n\n  return {\n    config: {\n      ...midsceneDefaults,\n      ...commonOverrideConfig,\n      ...modelSpecificConfig,\n    },\n  };\n};\n\nexport const glmAdapters = {\n  'glm-v': {\n    chatCompletion: {\n      unsupportedUserConfig: ['reasoningEffort', 'reasoningBudget'],\n      buildChatCompletionParams: buildGlmChatCompletionParams,\n      useReasoningAsContentFallback: true,\n    },\n    locate: {\n      resultAdapter: {\n        coordinates: { shape: 'bbox', order: 'xy', normalizedBy: 1000 },\n      },\n    },\n  },\n} satisfies Pick<Record<TModelFamily, ModelAdapterDefinition>, 'glm-v'>;\n"],"names":["buildGlmChatCompletionParams","input","midsceneDefaults","userConfig","reasoningEnabled","commonOverrideConfig","undefined","modelSpecificConfig","glmAdapters"],"mappings":"AAOA,MAAMA,+BAA+B,CACnCC;IAEA,MAAM,EAAEC,gBAAgB,EAAEC,UAAU,EAAE,GAAGF;IACzC,MAAM,EAAEG,gBAAgB,EAAE,GAAGD;IAC7B,MAAME,uBAAgD,CAAC;IAEvD,IAAIF,AAA2BG,WAA3BH,WAAW,WAAW,EACxBE,qBAAqB,WAAW,GAAGF,WAAW,WAAW;IAG3D,MAAMI,sBAA+C,CAAC;IAEtD,IAAIH,AAAqB,cAArBA,kBACFG,oBAAoB,QAAQ,GAAG;QAC7B,MAAOH,oBAAoB,QAAS,YAAY;IAClD;IAGF,OAAO;QACL,QAAQ;YACN,GAAGF,gBAAgB;YACnB,GAAGG,oBAAoB;YACvB,GAAGE,mBAAmB;QACxB;IACF;AACF;AAEO,MAAMC,cAAc;IACzB,SAAS;QACP,gBAAgB;YACd,uBAAuB;gBAAC;gBAAmB;aAAkB;YAC7D,2BAA2BR;YAC3B,+BAA+B;QACjC;QACA,QAAQ;YACN,eAAe;gBACb,aAAa;oBAAE,OAAO;oBAAQ,OAAO;oBAAM,cAAc;gBAAK;YAChE;QACF;IACF;AACF"}
+{"version":3,"file":"ai-model/models/glm.mjs","sources":["../../../../src/ai-model/models/glm.ts"],"sourcesContent":["import type { TModelFamily } from '@midscene/shared/env';\nimport type {\n  ChatCompletionCallContext,\n  ChatCompletionParamsResult,\n  ModelAdapterDefinition,\n} from '../model-adapter/types';\nimport { isLocateIntent } from './utils/intent';\n\nconst buildGlmChatCompletionParams = (\n  input: ChatCompletionCallContext,\n): ChatCompletionParamsResult => {\n  const { midsceneDefaults, userConfig } = input;\n  const { reasoningEnabled } = userConfig;\n  const commonOverrideConfig: Record<string, unknown> = {};\n\n  if (userConfig.temperature !== undefined) {\n    commonOverrideConfig.temperature = userConfig.temperature;\n  }\n\n  // Zhipu structured output JSON mode:\n  // https://docs.bigmodel.cn/cn/guide/capabilities/struct-output\n  if (isLocateIntent(input.intent)) {\n    commonOverrideConfig.response_format = { type: 'json_object' };\n  }\n\n  const modelSpecificConfig: Record<string, unknown> = {};\n\n  if (reasoningEnabled !== 'default') {\n    modelSpecificConfig.thinking = {\n      type: (reasoningEnabled ?? false) ? 'enabled' : 'disabled',\n    };\n  }\n\n  return {\n    config: {\n      ...midsceneDefaults,\n      ...commonOverrideConfig,\n      ...modelSpecificConfig,\n    },\n  };\n};\n\nexport const glmAdapters = {\n  'glm-v': {\n    chatCompletion: {\n      unsupportedUserConfig: ['reasoningEffort', 'reasoningBudget'],\n      buildChatCompletionParams: buildGlmChatCompletionParams,\n      useReasoningAsContentFallback: true,\n    },\n    locate: {\n      resultAdapter: {\n        coordinates: { shape: 'bbox', order: 'xy', normalizedBy: 1000 },\n      },\n    },\n  },\n} satisfies Pick<Record<TModelFamily, ModelAdapterDefinition>, 'glm-v'>;\n"],"names":["buildGlmChatCompletionParams","input","midsceneDefaults","userConfig","reasoningEnabled","commonOverrideConfig","undefined","isLocateIntent","modelSpecificConfig","glmAdapters"],"mappings":";AAQA,MAAMA,+BAA+B,CACnCC;IAEA,MAAM,EAAEC,gBAAgB,EAAEC,UAAU,EAAE,GAAGF;IACzC,MAAM,EAAEG,gBAAgB,EAAE,GAAGD;IAC7B,MAAME,uBAAgD,CAAC;IAEvD,IAAIF,AAA2BG,WAA3BH,WAAW,WAAW,EACxBE,qBAAqB,WAAW,GAAGF,WAAW,WAAW;IAK3D,IAAII,eAAeN,MAAM,MAAM,GAC7BI,qBAAqB,eAAe,GAAG;QAAE,MAAM;IAAc;IAG/D,MAAMG,sBAA+C,CAAC;IAEtD,IAAIJ,AAAqB,cAArBA,kBACFI,oBAAoB,QAAQ,GAAG;QAC7B,MAAOJ,oBAAoB,QAAS,YAAY;IAClD;IAGF,OAAO;QACL,QAAQ;YACN,GAAGF,gBAAgB;YACnB,GAAGG,oBAAoB;YACvB,GAAGG,mBAAmB;QACxB;IACF;AACF;AAEO,MAAMC,cAAc;IACzB,SAAS;QACP,gBAAgB;YACd,uBAAuB;gBAAC;gBAAmB;aAAkB;YAC7D,2BAA2BT;YAC3B,+BAA+B;QACjC;QACA,QAAQ;YACN,eAAe;gBACb,aAAa;oBAAE,OAAO;oBAAQ,OAAO;oBAAM,cAAc;gBAAK;YAChE;QACF;IACF;AACF"}

package/dist/es/ai-model/models/gpt.mjs

@@ -1,9 +1,13 @@
-const originalImageDetailForDefaultIntent = (input)=>'default' === input.intent || input.requiresOriginalImageDetail ? 'original' : void 0;
+import { isLocateIntent } from "./utils/intent.mjs";
+const originalImageDetailForDefaultIntent = (input)=>isLocateIntent(input.intent) || input.requiresOriginalImageDetail ? 'original' : void 0;
 const buildGpt5ChatCompletionParams = (input)=>{
     const { midsceneDefaults, userConfig } = input;
     const { reasoningEnabled, reasoningEffort } = userConfig;
     const commonOverrideConfig = {};
     if (void 0 !== userConfig.temperature) commonOverrideConfig.temperature = userConfig.temperature;
+    if (isLocateIntent(input.intent)) commonOverrideConfig.response_format = {
+        type: 'json_object'
+    };
     const effectiveReasoningEffort = true === reasoningEnabled ? reasoningEffort ?? 'medium' : 'none';
     return {
         config: {

package/dist/es/ai-model/models/gpt.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/models/gpt.mjs","sources":["../../../../src/ai-model/models/gpt.ts"],"sourcesContent":["import type { TModelFamily } from '@midscene/shared/env';\nimport type {\n  ChatCompletionCallContext,\n  ChatCompletionParamsResult,\n  ImageDetail,\n  ModelAdapterDefinition,\n} from '../model-adapter/types';\n\nconst originalImageDetailForDefaultIntent = (\n  input: ChatCompletionCallContext,\n): ImageDetail | undefined =>\n  input.intent === 'default' || input.requiresOriginalImageDetail\n    ? 'original'\n    : undefined;\n\nconst buildGpt5ChatCompletionParams = (\n  input: ChatCompletionCallContext,\n): ChatCompletionParamsResult => {\n  const { midsceneDefaults, userConfig } = input;\n  const { reasoningEnabled, reasoningEffort } = userConfig;\n  const commonOverrideConfig: Record<string, unknown> = {};\n\n  if (userConfig.temperature !== undefined) {\n    commonOverrideConfig.temperature = userConfig.temperature;\n  }\n\n  const effectiveReasoningEffort =\n    reasoningEnabled === true ? (reasoningEffort ?? 'medium') : 'none';\n\n  return {\n    config: {\n      ...midsceneDefaults,\n      ...commonOverrideConfig,\n      reasoning_effort: effectiveReasoningEffort,\n    },\n  };\n};\n\nexport const gptAdapters = {\n  'gpt-5': {\n    chatCompletion: {\n      unsupportedUserConfig: ['reasoningBudget'],\n      buildChatCompletionParams: buildGpt5ChatCompletionParams,\n      resolveImageDetail: originalImageDetailForDefaultIntent,\n    },\n    locate: {\n      resultAdapter: {\n        coordinates: { shape: 'bbox', order: 'xy' },\n      },\n    },\n  },\n} satisfies Pick<Record<TModelFamily, ModelAdapterDefinition>, 'gpt-5'>;\n"],"names":["originalImageDetailForDefaultIntent","input","undefined","buildGpt5ChatCompletionParams","midsceneDefaults","userConfig","reasoningEnabled","reasoningEffort","commonOverrideConfig","effectiveReasoningEffort","gptAdapters"],"mappings":"AAQA,MAAMA,sCAAsC,CAC1CC,QAEAA,AAAiB,cAAjBA,MAAM,MAAM,IAAkBA,MAAM,2BAA2B,GAC3D,aACAC;AAEN,MAAMC,gCAAgC,CACpCF;IAEA,MAAM,EAAEG,gBAAgB,EAAEC,UAAU,EAAE,GAAGJ;IACzC,MAAM,EAAEK,gBAAgB,EAAEC,eAAe,EAAE,GAAGF;IAC9C,MAAMG,uBAAgD,CAAC;IAEvD,IAAIH,AAA2BH,WAA3BG,WAAW,WAAW,EACxBG,qBAAqB,WAAW,GAAGH,WAAW,WAAW;IAG3D,MAAMI,2BACJH,AAAqB,SAArBA,mBAA6BC,mBAAmB,WAAY;IAE9D,OAAO;QACL,QAAQ;YACN,GAAGH,gBAAgB;YACnB,GAAGI,oBAAoB;YACvB,kBAAkBC;QACpB;IACF;AACF;AAEO,MAAMC,cAAc;IACzB,SAAS;QACP,gBAAgB;YACd,uBAAuB;gBAAC;aAAkB;YAC1C,2BAA2BP;YAC3B,oBAAoBH;QACtB;QACA,QAAQ;YACN,eAAe;gBACb,aAAa;oBAAE,OAAO;oBAAQ,OAAO;gBAAK;YAC5C;QACF;IACF;AACF"}
+{"version":3,"file":"ai-model/models/gpt.mjs","sources":["../../../../src/ai-model/models/gpt.ts"],"sourcesContent":["import type { TModelFamily } from '@midscene/shared/env';\nimport type {\n  ChatCompletionCallContext,\n  ChatCompletionParamsResult,\n  ImageDetail,\n  ModelAdapterDefinition,\n} from '../model-adapter/types';\nimport { isLocateIntent } from './utils/intent';\n\nconst originalImageDetailForDefaultIntent = (\n  input: ChatCompletionCallContext,\n): ImageDetail | undefined =>\n  isLocateIntent(input.intent) || input.requiresOriginalImageDetail\n    ? 'original'\n    : undefined;\n\nconst buildGpt5ChatCompletionParams = (\n  input: ChatCompletionCallContext,\n): ChatCompletionParamsResult => {\n  const { midsceneDefaults, userConfig } = input;\n  const { reasoningEnabled, reasoningEffort } = userConfig;\n  const commonOverrideConfig: Record<string, unknown> = {};\n\n  if (userConfig.temperature !== undefined) {\n    commonOverrideConfig.temperature = userConfig.temperature;\n  }\n\n  // OpenAI Chat Completions JSON mode:\n  // https://platform.openai.com/docs/guides/structured-outputs?api-mode=chat#json-mode\n  if (isLocateIntent(input.intent)) {\n    commonOverrideConfig.response_format = { type: 'json_object' };\n  }\n\n  const effectiveReasoningEffort =\n    reasoningEnabled === true ? (reasoningEffort ?? 'medium') : 'none';\n\n  return {\n    config: {\n      ...midsceneDefaults,\n      ...commonOverrideConfig,\n      reasoning_effort: effectiveReasoningEffort,\n    },\n  };\n};\n\nexport const gptAdapters = {\n  'gpt-5': {\n    chatCompletion: {\n      unsupportedUserConfig: ['reasoningBudget'],\n      buildChatCompletionParams: buildGpt5ChatCompletionParams,\n      resolveImageDetail: originalImageDetailForDefaultIntent,\n    },\n    locate: {\n      resultAdapter: {\n        coordinates: { shape: 'bbox', order: 'xy' },\n      },\n    },\n  },\n} satisfies Pick<Record<TModelFamily, ModelAdapterDefinition>, 'gpt-5'>;\n"],"names":["originalImageDetailForDefaultIntent","input","isLocateIntent","undefined","buildGpt5ChatCompletionParams","midsceneDefaults","userConfig","reasoningEnabled","reasoningEffort","commonOverrideConfig","effectiveReasoningEffort","gptAdapters"],"mappings":";AASA,MAAMA,sCAAsC,CAC1CC,QAEAC,eAAeD,MAAM,MAAM,KAAKA,MAAM,2BAA2B,GAC7D,aACAE;AAEN,MAAMC,gCAAgC,CACpCH;IAEA,MAAM,EAAEI,gBAAgB,EAAEC,UAAU,EAAE,GAAGL;IACzC,MAAM,EAAEM,gBAAgB,EAAEC,eAAe,EAAE,GAAGF;IAC9C,MAAMG,uBAAgD,CAAC;IAEvD,IAAIH,AAA2BH,WAA3BG,WAAW,WAAW,EACxBG,qBAAqB,WAAW,GAAGH,WAAW,WAAW;IAK3D,IAAIJ,eAAeD,MAAM,MAAM,GAC7BQ,qBAAqB,eAAe,GAAG;QAAE,MAAM;IAAc;IAG/D,MAAMC,2BACJH,AAAqB,SAArBA,mBAA6BC,mBAAmB,WAAY;IAE9D,OAAO;QACL,QAAQ;YACN,GAAGH,gBAAgB;YACnB,GAAGI,oBAAoB;YACvB,kBAAkBC;QACpB;IACF;AACF;AAEO,MAAMC,cAAc;IACzB,SAAS;QACP,gBAAgB;YACd,uBAAuB;gBAAC;aAAkB;YAC1C,2BAA2BP;YAC3B,oBAAoBJ;QACtB;QACA,QAAQ;YACN,eAAe;gBACb,aAAa;oBAAE,OAAO;oBAAQ,OAAO;gBAAK;YAC5C;QACF;IACF;AACF"}

package/dist/es/ai-model/models/kimi.mjs

@@ -1,4 +1,5 @@
 import { createLocateResultValue, parseCoordinateList } from "../shared/model-locate-result/index.mjs";
+import { isLocateIntent } from "./utils/intent.mjs";
 const kimiNormalizedPointCoordinatesMeta = {
     shape: 'point',
     order: 'xy',
@@ -24,6 +25,9 @@ const buildKimiChatCompletionParams = (input)=>{
     const effectiveReasoningEnabled = reasoningEnabled ?? false;
     const commonOverrideConfig = {};
     commonOverrideConfig.temperature = void 0;
+    if (isLocateIntent(input.intent)) commonOverrideConfig.response_format = {
+        type: 'json_object'
+    };
     const modelSpecificConfig = {
         thinking: {
             type: effectiveReasoningEnabled ? 'enabled' : 'disabled'

package/dist/es/ai-model/models/kimi.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/models/kimi.mjs","sources":["../../../../src/ai-model/models/kimi.ts"],"sourcesContent":["import type { TModelFamily } from '@midscene/shared/env';\nimport type {\n  ChatCompletionCallContext,\n  ChatCompletionParamsResult,\n  ModelAdapterDefinition,\n} from '../model-adapter/types';\nimport {\n  type LocateResultValue,\n  createLocateResultValue,\n  parseCoordinateList,\n} from '../shared/model-locate-result';\n\nconst kimiNormalizedPointCoordinatesMeta = {\n  shape: 'point',\n  order: 'xy',\n  normalizedBy: 1,\n} as const;\nconst kimiPixelPointCoordinatesMeta = {\n  shape: 'point',\n  order: 'xy',\n} as const;\n\nfunction parseKimiRawLocateValue(input: unknown): LocateResultValue {\n  const point = parseCoordinateList(input, 'point');\n  if (point.length < 2) {\n    throw new Error(`invalid point data: ${JSON.stringify(input)} `);\n  }\n  const [x, y] = point;\n  // Keep this compatible with OSWorld's Kimi adapter: values <= 1 are\n  // normalized coordinates, otherwise they are treated as screenshot pixels.\n  const coordinatesMeta =\n    x <= 1 && y <= 1\n      ? kimiNormalizedPointCoordinatesMeta\n      : kimiPixelPointCoordinatesMeta;\n  return createLocateResultValue(coordinatesMeta, [x, y]);\n}\n\nconst buildKimiChatCompletionParams = (\n  input: ChatCompletionCallContext,\n): ChatCompletionParamsResult => {\n  const { midsceneDefaults, userConfig } = input;\n  const { reasoningEnabled } = userConfig;\n  const effectiveReasoningEnabled = reasoningEnabled ?? false;\n  const commonOverrideConfig: Record<string, unknown> = {};\n\n  // kimi disallow custom temperature\n  commonOverrideConfig.temperature = undefined;\n\n  const modelSpecificConfig: Record<string, unknown> = {\n    thinking: {\n      type: effectiveReasoningEnabled ? 'enabled' : 'disabled',\n    },\n  };\n\n  return {\n    config: {\n      ...midsceneDefaults,\n      ...commonOverrideConfig,\n      ...modelSpecificConfig,\n    },\n  };\n};\n\nexport const kimiAdapters = {\n  kimi: {\n    chatCompletion: {\n      unsupportedUserConfig: ['reasoningEffort', 'reasoningBudget'],\n      buildChatCompletionParams: buildKimiChatCompletionParams,\n      useReasoningAsContentFallback: true,\n    },\n    locate: {\n      resultAdapter: {\n        coordinates: kimiNormalizedPointCoordinatesMeta,\n        parseRawLocateValue: parseKimiRawLocateValue,\n      },\n    },\n  },\n} satisfies Pick<Record<TModelFamily, ModelAdapterDefinition>, 'kimi'>;\n"],"names":["kimiNormalizedPointCoordinatesMeta","kimiPixelPointCoordinatesMeta","parseKimiRawLocateValue","input","point","parseCoordinateList","Error","JSON","x","y","coordinatesMeta","createLocateResultValue","buildKimiChatCompletionParams","midsceneDefaults","userConfig","reasoningEnabled","effectiveReasoningEnabled","commonOverrideConfig","undefined","modelSpecificConfig","kimiAdapters"],"mappings":";AAYA,MAAMA,qCAAqC;IACzC,OAAO;IACP,OAAO;IACP,cAAc;AAChB;AACA,MAAMC,gCAAgC;IACpC,OAAO;IACP,OAAO;AACT;AAEA,SAASC,wBAAwBC,KAAc;IAC7C,MAAMC,QAAQC,oBAAoBF,OAAO;IACzC,IAAIC,MAAM,MAAM,GAAG,GACjB,MAAM,IAAIE,MAAM,CAAC,oBAAoB,EAAEC,KAAK,SAAS,CAACJ,OAAO,CAAC,CAAC;IAEjE,MAAM,CAACK,GAAGC,EAAE,GAAGL;IAGf,MAAMM,kBACJF,KAAK,KAAKC,KAAK,IACXT,qCACAC;IACN,OAAOU,wBAAwBD,iBAAiB;QAACF;QAAGC;KAAE;AACxD;AAEA,MAAMG,gCAAgC,CACpCT;IAEA,MAAM,EAAEU,gBAAgB,EAAEC,UAAU,EAAE,GAAGX;IACzC,MAAM,EAAEY,gBAAgB,EAAE,GAAGD;IAC7B,MAAME,4BAA4BD,oBAAoB;IACtD,MAAME,uBAAgD,CAAC;IAGvDA,qBAAqB,WAAW,GAAGC;IAEnC,MAAMC,sBAA+C;QACnD,UAAU;YACR,MAAMH,4BAA4B,YAAY;QAChD;IACF;IAEA,OAAO;QACL,QAAQ;YACN,GAAGH,gBAAgB;YACnB,GAAGI,oBAAoB;YACvB,GAAGE,mBAAmB;QACxB;IACF;AACF;AAEO,MAAMC,eAAe;IAC1B,MAAM;QACJ,gBAAgB;YACd,uBAAuB;gBAAC;gBAAmB;aAAkB;YAC7D,2BAA2BR;YAC3B,+BAA+B;QACjC;QACA,QAAQ;YACN,eAAe;gBACb,aAAaZ;gBACb,qBAAqBE;YACvB;QACF;IACF;AACF"}
+{"version":3,"file":"ai-model/models/kimi.mjs","sources":["../../../../src/ai-model/models/kimi.ts"],"sourcesContent":["import type { TModelFamily } from '@midscene/shared/env';\nimport type {\n  ChatCompletionCallContext,\n  ChatCompletionParamsResult,\n  ModelAdapterDefinition,\n} from '../model-adapter/types';\nimport {\n  type LocateResultValue,\n  createLocateResultValue,\n  parseCoordinateList,\n} from '../shared/model-locate-result';\nimport { isLocateIntent } from './utils/intent';\n\nconst kimiNormalizedPointCoordinatesMeta = {\n  shape: 'point',\n  order: 'xy',\n  normalizedBy: 1,\n} as const;\nconst kimiPixelPointCoordinatesMeta = {\n  shape: 'point',\n  order: 'xy',\n} as const;\n\nfunction parseKimiRawLocateValue(input: unknown): LocateResultValue {\n  const point = parseCoordinateList(input, 'point');\n  if (point.length < 2) {\n    throw new Error(`invalid point data: ${JSON.stringify(input)} `);\n  }\n  const [x, y] = point;\n  // Keep this compatible with OSWorld's Kimi adapter: values <= 1 are\n  // normalized coordinates, otherwise they are treated as screenshot pixels.\n  const coordinatesMeta =\n    x <= 1 && y <= 1\n      ? kimiNormalizedPointCoordinatesMeta\n      : kimiPixelPointCoordinatesMeta;\n  return createLocateResultValue(coordinatesMeta, [x, y]);\n}\n\nconst buildKimiChatCompletionParams = (\n  input: ChatCompletionCallContext,\n): ChatCompletionParamsResult => {\n  const { midsceneDefaults, userConfig } = input;\n  const { reasoningEnabled } = userConfig;\n  const effectiveReasoningEnabled = reasoningEnabled ?? false;\n  const commonOverrideConfig: Record<string, unknown> = {};\n\n  // kimi disallow custom temperature\n  commonOverrideConfig.temperature = undefined;\n\n  // Kimi Chat Completions response_format:\n  // https://platform.kimi.com/docs/api/chat\n  if (isLocateIntent(input.intent)) {\n    commonOverrideConfig.response_format = { type: 'json_object' };\n  }\n\n  const modelSpecificConfig: Record<string, unknown> = {\n    thinking: {\n      type: effectiveReasoningEnabled ? 'enabled' : 'disabled',\n    },\n  };\n\n  return {\n    config: {\n      ...midsceneDefaults,\n      ...commonOverrideConfig,\n      ...modelSpecificConfig,\n    },\n  };\n};\n\nexport const kimiAdapters = {\n  kimi: {\n    chatCompletion: {\n      unsupportedUserConfig: ['reasoningEffort', 'reasoningBudget'],\n      buildChatCompletionParams: buildKimiChatCompletionParams,\n      useReasoningAsContentFallback: true,\n    },\n    locate: {\n      resultAdapter: {\n        coordinates: kimiNormalizedPointCoordinatesMeta,\n        parseRawLocateValue: parseKimiRawLocateValue,\n      },\n    },\n  },\n} satisfies Pick<Record<TModelFamily, ModelAdapterDefinition>, 'kimi'>;\n"],"names":["kimiNormalizedPointCoordinatesMeta","kimiPixelPointCoordinatesMeta","parseKimiRawLocateValue","input","point","parseCoordinateList","Error","JSON","x","y","coordinatesMeta","createLocateResultValue","buildKimiChatCompletionParams","midsceneDefaults","userConfig","reasoningEnabled","effectiveReasoningEnabled","commonOverrideConfig","undefined","isLocateIntent","modelSpecificConfig","kimiAdapters"],"mappings":";;AAaA,MAAMA,qCAAqC;IACzC,OAAO;IACP,OAAO;IACP,cAAc;AAChB;AACA,MAAMC,gCAAgC;IACpC,OAAO;IACP,OAAO;AACT;AAEA,SAASC,wBAAwBC,KAAc;IAC7C,MAAMC,QAAQC,oBAAoBF,OAAO;IACzC,IAAIC,MAAM,MAAM,GAAG,GACjB,MAAM,IAAIE,MAAM,CAAC,oBAAoB,EAAEC,KAAK,SAAS,CAACJ,OAAO,CAAC,CAAC;IAEjE,MAAM,CAACK,GAAGC,EAAE,GAAGL;IAGf,MAAMM,kBACJF,KAAK,KAAKC,KAAK,IACXT,qCACAC;IACN,OAAOU,wBAAwBD,iBAAiB;QAACF;QAAGC;KAAE;AACxD;AAEA,MAAMG,gCAAgC,CACpCT;IAEA,MAAM,EAAEU,gBAAgB,EAAEC,UAAU,EAAE,GAAGX;IACzC,MAAM,EAAEY,gBAAgB,EAAE,GAAGD;IAC7B,MAAME,4BAA4BD,oBAAoB;IACtD,MAAME,uBAAgD,CAAC;IAGvDA,qBAAqB,WAAW,GAAGC;IAInC,IAAIC,eAAehB,MAAM,MAAM,GAC7Bc,qBAAqB,eAAe,GAAG;QAAE,MAAM;IAAc;IAG/D,MAAMG,sBAA+C;QACnD,UAAU;YACR,MAAMJ,4BAA4B,YAAY;QAChD;IACF;IAEA,OAAO;QACL,QAAQ;YACN,GAAGH,gBAAgB;YACnB,GAAGI,oBAAoB;YACvB,GAAGG,mBAAmB;QACxB;IACF;AACF;AAEO,MAAMC,eAAe;IAC1B,MAAM;QACJ,gBAAgB;YACd,uBAAuB;gBAAC;gBAAmB;aAAkB;YAC7D,2BAA2BT;YAC3B,+BAA+B;QACjC;QACA,QAAQ;YACN,eAAe;gBACb,aAAaZ;gBACb,qBAAqBE;YACvB;QACF;IACF;AACF"}

package/dist/es/ai-model/models/mimo.mjs

@@ -1,9 +1,10 @@
+import { isLocateIntent } from "./utils/intent.mjs";
 const buildMimoChatCompletionParams = (input)=>{
     const { intent, midsceneDefaults, userConfig } = input;
     const { reasoningEnabled } = userConfig;
     const commonOverrideConfig = {};
-    commonOverrideConfig.response_format = {
-        type: 'default' === intent ? 'json_object' : 'text'
+    if (isLocateIntent(intent)) commonOverrideConfig.response_format = {
+        type: 'json_object'
     };
     if (void 0 !== userConfig.temperature) commonOverrideConfig.temperature = userConfig.temperature;
     const modelSpecificConfig = {

package/dist/es/ai-model/models/mimo.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/models/mimo.mjs","sources":["../../../../src/ai-model/models/mimo.ts"],"sourcesContent":["import type { TModelFamily } from '@midscene/shared/env';\nimport type {\n  ChatCompletionCallContext,\n  ChatCompletionParamsResult,\n  ModelAdapterDefinition,\n} from '../model-adapter/types';\n\nconst buildMimoChatCompletionParams = (\n  input: ChatCompletionCallContext,\n): ChatCompletionParamsResult => {\n  const { intent, midsceneDefaults, userConfig } = input;\n  const { reasoningEnabled } = userConfig;\n  const commonOverrideConfig: Record<string, unknown> = {};\n\n  // https://platform.xiaomimimo.com/docs/zh-CN/api/chat/openai-api\n  // Observed with thinking disabled: Mimo needs json_object to return JSON.\n  commonOverrideConfig.response_format = {\n    type: intent === 'default' ? 'json_object' : 'text',\n  };\n\n  if (userConfig.temperature !== undefined) {\n    commonOverrideConfig.temperature = userConfig.temperature;\n  }\n\n  const modelSpecificConfig: Record<string, unknown> = {\n    thinking: {\n      type: (reasoningEnabled ?? false) ? 'enabled' : 'disabled',\n    },\n  };\n\n  return {\n    config: {\n      ...midsceneDefaults,\n      ...commonOverrideConfig,\n      ...modelSpecificConfig,\n    },\n  };\n};\n\nexport const mimoAdapters = {\n  'xiaomi-mimo': {\n    chatCompletion: {\n      unsupportedUserConfig: ['reasoningEffort', 'reasoningBudget'],\n      buildChatCompletionParams: buildMimoChatCompletionParams,\n      useReasoningAsContentFallback: true,\n    },\n  },\n} satisfies Pick<Record<TModelFamily, ModelAdapterDefinition>, 'xiaomi-mimo'>;\n"],"names":["buildMimoChatCompletionParams","input","intent","midsceneDefaults","userConfig","reasoningEnabled","commonOverrideConfig","undefined","modelSpecificConfig","mimoAdapters"],"mappings":"AAOA,MAAMA,gCAAgC,CACpCC;IAEA,MAAM,EAAEC,MAAM,EAAEC,gBAAgB,EAAEC,UAAU,EAAE,GAAGH;IACjD,MAAM,EAAEI,gBAAgB,EAAE,GAAGD;IAC7B,MAAME,uBAAgD,CAAC;IAIvDA,qBAAqB,eAAe,GAAG;QACrC,MAAMJ,AAAW,cAAXA,SAAuB,gBAAgB;IAC/C;IAEA,IAAIE,AAA2BG,WAA3BH,WAAW,WAAW,EACxBE,qBAAqB,WAAW,GAAGF,WAAW,WAAW;IAG3D,MAAMI,sBAA+C;QACnD,UAAU;YACR,MAAOH,oBAAoB,QAAS,YAAY;QAClD;IACF;IAEA,OAAO;QACL,QAAQ;YACN,GAAGF,gBAAgB;YACnB,GAAGG,oBAAoB;YACvB,GAAGE,mBAAmB;QACxB;IACF;AACF;AAEO,MAAMC,eAAe;IAC1B,eAAe;QACb,gBAAgB;YACd,uBAAuB;gBAAC;gBAAmB;aAAkB;YAC7D,2BAA2BT;YAC3B,+BAA+B;QACjC;IACF;AACF"}
+{"version":3,"file":"ai-model/models/mimo.mjs","sources":["../../../../src/ai-model/models/mimo.ts"],"sourcesContent":["import type { TModelFamily } from '@midscene/shared/env';\nimport type {\n  ChatCompletionCallContext,\n  ChatCompletionParamsResult,\n  ModelAdapterDefinition,\n} from '../model-adapter/types';\nimport { isLocateIntent } from './utils/intent';\n\nconst buildMimoChatCompletionParams = (\n  input: ChatCompletionCallContext,\n): ChatCompletionParamsResult => {\n  const { intent, midsceneDefaults, userConfig } = input;\n  const { reasoningEnabled } = userConfig;\n  const commonOverrideConfig: Record<string, unknown> = {};\n\n  // https://platform.xiaomimimo.com/docs/zh-CN/api/chat/openai-api\n  // Observed with thinking disabled: Mimo needs json_object to return JSON.\n  if (isLocateIntent(intent)) {\n    commonOverrideConfig.response_format = { type: 'json_object' };\n  }\n\n  if (userConfig.temperature !== undefined) {\n    commonOverrideConfig.temperature = userConfig.temperature;\n  }\n\n  const modelSpecificConfig: Record<string, unknown> = {\n    thinking: {\n      type: (reasoningEnabled ?? false) ? 'enabled' : 'disabled',\n    },\n  };\n\n  return {\n    config: {\n      ...midsceneDefaults,\n      ...commonOverrideConfig,\n      ...modelSpecificConfig,\n    },\n  };\n};\n\nexport const mimoAdapters = {\n  'xiaomi-mimo': {\n    chatCompletion: {\n      unsupportedUserConfig: ['reasoningEffort', 'reasoningBudget'],\n      buildChatCompletionParams: buildMimoChatCompletionParams,\n      useReasoningAsContentFallback: true,\n    },\n  },\n} satisfies Pick<Record<TModelFamily, ModelAdapterDefinition>, 'xiaomi-mimo'>;\n"],"names":["buildMimoChatCompletionParams","input","intent","midsceneDefaults","userConfig","reasoningEnabled","commonOverrideConfig","isLocateIntent","undefined","modelSpecificConfig","mimoAdapters"],"mappings":";AAQA,MAAMA,gCAAgC,CACpCC;IAEA,MAAM,EAAEC,MAAM,EAAEC,gBAAgB,EAAEC,UAAU,EAAE,GAAGH;IACjD,MAAM,EAAEI,gBAAgB,EAAE,GAAGD;IAC7B,MAAME,uBAAgD,CAAC;IAIvD,IAAIC,eAAeL,SACjBI,qBAAqB,eAAe,GAAG;QAAE,MAAM;IAAc;IAG/D,IAAIF,AAA2BI,WAA3BJ,WAAW,WAAW,EACxBE,qBAAqB,WAAW,GAAGF,WAAW,WAAW;IAG3D,MAAMK,sBAA+C;QACnD,UAAU;YACR,MAAOJ,oBAAoB,QAAS,YAAY;QAClD;IACF;IAEA,OAAO;QACL,QAAQ;YACN,GAAGF,gBAAgB;YACnB,GAAGG,oBAAoB;YACvB,GAAGG,mBAAmB;QACxB;IACF;AACF;AAEO,MAAMC,eAAe;IAC1B,eAAe;QACb,gBAAgB;YACd,uBAAuB;gBAAC;gBAAmB;aAAkB;YAC7D,2BAA2BV;YAC3B,+BAA+B;QACjC;IACF;AACF"}

package/dist/es/ai-model/models/qwen.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/models/qwen.mjs","sources":["../../../../src/ai-model/models/qwen.ts"],"sourcesContent":["import type { TModelFamily } from '@midscene/shared/env';\nimport type {\n  ChatCompletionCallContext,\n  ChatCompletionParamsResult,\n  ModelAdapterDefinition,\n} from '../model-adapter/types';\nimport {\n  type LocateResultValue,\n  type PixelBbox,\n  createLocateResultValue,\n  isBboxLocateResultValue,\n  unwrapCoordinateListLikeInput,\n} from '../shared/model-locate-result';\n\nconst defaultBboxSize = 20;\nconst qwen25BboxCoordinatesMeta = {\n  shape: 'bbox',\n  order: 'xy',\n} as const;\nconst qwen25PointCoordinatesMeta = {\n  shape: 'point',\n  order: 'xy',\n} as const;\nconst qwen3BboxCoordinatesMeta = {\n  shape: 'bbox',\n  order: 'xy',\n  normalizedBy: 1000,\n} as const;\n\nfunction topLeftPointToPixelBbox(x: number, y: number): PixelBbox {\n  return [\n    Math.round(x),\n    Math.round(y),\n    Math.round(x + defaultBboxSize),\n    Math.round(y + defaultBboxSize),\n  ];\n}\n\nfunction parseQwen25RawLocateValue(input: unknown): LocateResultValue {\n  const bbox = unwrapCoordinateListLikeInput(input as any) as number[];\n  if (bbox.length < 2) {\n    const msg = `invalid bbox data for qwen-vl mode: ${JSON.stringify(bbox)} `;\n    throw new Error(msg);\n  }\n\n  if (typeof bbox[2] === 'number' && typeof bbox[3] === 'number') {\n    return createLocateResultValue(qwen25BboxCoordinatesMeta, [\n      bbox[0],\n      bbox[1],\n      bbox[2],\n      bbox[3],\n    ]);\n  }\n\n  return createLocateResultValue(qwen25PointCoordinatesMeta, [\n    bbox[0],\n    bbox[1],\n  ]);\n}\n\nfunction normalizeQwen25ResultToPixelBbox(\n  result: LocateResultValue,\n): PixelBbox {\n  if (isBboxLocateResultValue(result)) {\n    const { coordinates } = result;\n    return [\n      Math.round(coordinates[0]),\n      Math.round(coordinates[1]),\n      Math.round(coordinates[2]),\n      Math.round(coordinates[3]),\n    ];\n  }\n\n  const { coordinates } = result;\n  return topLeftPointToPixelBbox(coordinates[0], coordinates[1]);\n}\n\nconst buildQwenChatCompletionParams = (\n  input: ChatCompletionCallContext,\n): ChatCompletionParamsResult => {\n  const { midsceneDefaults, userConfig } = input;\n  const { reasoningEnabled, reasoningBudget } = userConfig;\n\n  const commonOverrideConfig: Record<string, unknown> = {};\n\n  if (userConfig.temperature !== undefined) {\n    commonOverrideConfig.temperature = userConfig.temperature;\n  }\n\n  const modelSpecificConfig: Record<string, unknown> = {};\n\n  if (reasoningEnabled !== 'default') {\n    modelSpecificConfig.enable_thinking = reasoningEnabled ?? false;\n    if (reasoningBudget !== undefined) {\n      modelSpecificConfig.thinking_budget = reasoningBudget;\n    }\n  }\n\n  return {\n    config: {\n      ...midsceneDefaults,\n      ...commonOverrideConfig,\n      ...modelSpecificConfig,\n    },\n  };\n};\n\nconst buildQwen25ChatCompletionParams = (\n  input: ChatCompletionCallContext,\n): ChatCompletionParamsResult => {\n  const { midsceneDefaults, userConfig } = input;\n  const commonOverrideConfig: Record<string, unknown> = {};\n\n  if (userConfig.temperature !== undefined) {\n    commonOverrideConfig.temperature = userConfig.temperature;\n  }\n\n  return {\n    config: {\n      ...midsceneDefaults,\n      ...commonOverrideConfig,\n      vl_high_resolution_images: true,\n    },\n  };\n};\n\nconst qwen3Adapter: ModelAdapterDefinition = {\n  chatCompletion: {\n    unsupportedUserConfig: ['reasoningEffort'],\n    buildChatCompletionParams: buildQwenChatCompletionParams,\n    messageExtraction: {\n      kind: 'default',\n      // DashScope returns Qwen thinking as `reasoning_content`. vLLM's\n      // OpenAI-compatible server is migrating the recommended field to\n      // `reasoning`, so Qwen adapters accept both serving conventions:\n      // https://github.com/vllm-project/vllm/issues/27755\n      reasoningContentKeys: ['reasoning_content', 'reasoning'],\n    },\n    useReasoningAsContentFallback: true,\n  },\n  locate: {\n    resultAdapter: {\n      coordinates: qwen3BboxCoordinatesMeta,\n    },\n  },\n};\n\nexport const qwenAdapters = {\n  'qwen2.5-vl': {\n    chatCompletion: {\n      unsupportedUserConfig: [\n        'reasoningEnabled',\n        'reasoningEffort',\n        'reasoningBudget',\n      ],\n      buildChatCompletionParams: buildQwen25ChatCompletionParams,\n    },\n    imagePreprocess: {\n      padBlockSize: 28,\n    },\n    locate: {\n      resultAdapter: {\n        coordinates: qwen25BboxCoordinatesMeta,\n        parseRawLocateValue: parseQwen25RawLocateValue,\n        mapLocateResultToPixelBbox: normalizeQwen25ResultToPixelBbox,\n      },\n    },\n  },\n  'qwen3-vl': qwen3Adapter,\n  qwen3: qwen3Adapter,\n  'qwen3.5': qwen3Adapter,\n  'qwen3.6': qwen3Adapter,\n} satisfies Pick<\n  Record<TModelFamily, ModelAdapterDefinition>,\n  'qwen2.5-vl' | 'qwen3-vl' | 'qwen3' | 'qwen3.5' | 'qwen3.6'\n>;\n"],"names":["defaultBboxSize","qwen25BboxCoordinatesMeta","qwen25PointCoordinatesMeta","qwen3BboxCoordinatesMeta","topLeftPointToPixelBbox","x","y","Math","parseQwen25RawLocateValue","input","bbox","unwrapCoordinateListLikeInput","msg","JSON","Error","createLocateResultValue","normalizeQwen25ResultToPixelBbox","result","isBboxLocateResultValue","coordinates","buildQwenChatCompletionParams","midsceneDefaults","userConfig","reasoningEnabled","reasoningBudget","commonOverrideConfig","undefined","modelSpecificConfig","buildQwen25ChatCompletionParams","qwen3Adapter","qwenAdapters"],"mappings":";AAcA,MAAMA,kBAAkB;AACxB,MAAMC,4BAA4B;IAChC,OAAO;IACP,OAAO;AACT;AACA,MAAMC,6BAA6B;IACjC,OAAO;IACP,OAAO;AACT;AACA,MAAMC,2BAA2B;IAC/B,OAAO;IACP,OAAO;IACP,cAAc;AAChB;AAEA,SAASC,wBAAwBC,CAAS,EAAEC,CAAS;IACnD,OAAO;QACLC,KAAK,KAAK,CAACF;QACXE,KAAK,KAAK,CAACD;QACXC,KAAK,KAAK,CAACF,IAAIL;QACfO,KAAK,KAAK,CAACD,IAAIN;KAChB;AACH;AAEA,SAASQ,0BAA0BC,KAAc;IAC/C,MAAMC,OAAOC,8BAA8BF;IAC3C,IAAIC,KAAK,MAAM,GAAG,GAAG;QACnB,MAAME,MAAM,CAAC,oCAAoC,EAAEC,KAAK,SAAS,CAACH,MAAM,CAAC,CAAC;QAC1E,MAAM,IAAII,MAAMF;IAClB;IAEA,IAAI,AAAmB,YAAnB,OAAOF,IAAI,CAAC,EAAE,IAAiB,AAAmB,YAAnB,OAAOA,IAAI,CAAC,EAAE,EAC/C,OAAOK,wBAAwBd,2BAA2B;QACxDS,IAAI,CAAC,EAAE;QACPA,IAAI,CAAC,EAAE;QACPA,IAAI,CAAC,EAAE;QACPA,IAAI,CAAC,EAAE;KACR;IAGH,OAAOK,wBAAwBb,4BAA4B;QACzDQ,IAAI,CAAC,EAAE;QACPA,IAAI,CAAC,EAAE;KACR;AACH;AAEA,SAASM,iCACPC,MAAyB;IAEzB,IAAIC,wBAAwBD,SAAS;QACnC,MAAM,EAAEE,WAAW,EAAE,GAAGF;QACxB,OAAO;YACLV,KAAK,KAAK,CAACY,WAAW,CAAC,EAAE;YACzBZ,KAAK,KAAK,CAACY,WAAW,CAAC,EAAE;YACzBZ,KAAK,KAAK,CAACY,WAAW,CAAC,EAAE;YACzBZ,KAAK,KAAK,CAACY,WAAW,CAAC,EAAE;SAC1B;IACH;IAEA,MAAM,EAAEA,WAAW,EAAE,GAAGF;IACxB,OAAOb,wBAAwBe,WAAW,CAAC,EAAE,EAAEA,WAAW,CAAC,EAAE;AAC/D;AAEA,MAAMC,gCAAgC,CACpCX;IAEA,MAAM,EAAEY,gBAAgB,EAAEC,UAAU,EAAE,GAAGb;IACzC,MAAM,EAAEc,gBAAgB,EAAEC,eAAe,EAAE,GAAGF;IAE9C,MAAMG,uBAAgD,CAAC;IAEvD,IAAIH,AAA2BI,WAA3BJ,WAAW,WAAW,EACxBG,qBAAqB,WAAW,GAAGH,WAAW,WAAW;IAG3D,MAAMK,sBAA+C,CAAC;IAEtD,IAAIJ,AAAqB,cAArBA,kBAAgC;QAClCI,oBAAoB,eAAe,GAAGJ,oBAAoB;QAC1D,IAAIC,AAAoBE,WAApBF,iBACFG,oBAAoB,eAAe,GAAGH;IAE1C;IAEA,OAAO;QACL,QAAQ;YACN,GAAGH,gBAAgB;YACnB,GAAGI,oBAAoB;YACvB,GAAGE,mBAAmB;QACxB;IACF;AACF;AAEA,MAAMC,kCAAkC,CACtCnB;IAEA,MAAM,EAAEY,gBAAgB,EAAEC,UAAU,EAAE,GAAGb;IACzC,MAAMgB,uBAAgD,CAAC;IAEvD,IAAIH,AAA2BI,WAA3BJ,WAAW,WAAW,EACxBG,qBAAqB,WAAW,GAAGH,WAAW,WAAW;IAG3D,OAAO;QACL,QAAQ;YACN,GAAGD,gBAAgB;YACnB,GAAGI,oBAAoB;YACvB,2BAA2B;QAC7B;IACF;AACF;AAEA,MAAMI,eAAuC;IAC3C,gBAAgB;QACd,uBAAuB;YAAC;SAAkB;QAC1C,2BAA2BT;QAC3B,mBAAmB;YACjB,MAAM;YAKN,sBAAsB;gBAAC;gBAAqB;aAAY;QAC1D;QACA,+BAA+B;IACjC;IACA,QAAQ;QACN,eAAe;YACb,aAAajB;QACf;IACF;AACF;AAEO,MAAM2B,eAAe;IAC1B,cAAc;QACZ,gBAAgB;YACd,uBAAuB;gBACrB;gBACA;gBACA;aACD;YACD,2BAA2BF;QAC7B;QACA,iBAAiB;YACf,cAAc;QAChB;QACA,QAAQ;YACN,eAAe;gBACb,aAAa3B;gBACb,qBAAqBO;gBACrB,4BAA4BQ;YAC9B;QACF;IACF;IACA,YAAYa;IACZ,OAAOA;IACP,WAAWA;IACX,WAAWA;AACb"}
+{"version":3,"file":"ai-model/models/qwen.mjs","sources":["../../../../src/ai-model/models/qwen.ts"],"sourcesContent":["import type { TModelFamily } from '@midscene/shared/env';\nimport type {\n  ChatCompletionCallContext,\n  ChatCompletionParamsResult,\n  ModelAdapterDefinition,\n} from '../model-adapter/types';\nimport {\n  type LocateResultValue,\n  type PixelBbox,\n  createLocateResultValue,\n  isBboxLocateResultValue,\n  unwrapCoordinateListLikeInput,\n} from '../shared/model-locate-result';\n\nconst defaultBboxSize = 20;\nconst qwen25BboxCoordinatesMeta = {\n  shape: 'bbox',\n  order: 'xy',\n} as const;\nconst qwen25PointCoordinatesMeta = {\n  shape: 'point',\n  order: 'xy',\n} as const;\nconst qwen3BboxCoordinatesMeta = {\n  shape: 'bbox',\n  order: 'xy',\n  normalizedBy: 1000,\n} as const;\n\nfunction topLeftPointToPixelBbox(x: number, y: number): PixelBbox {\n  return [\n    Math.round(x),\n    Math.round(y),\n    Math.round(x + defaultBboxSize),\n    Math.round(y + defaultBboxSize),\n  ];\n}\n\nfunction parseQwen25RawLocateValue(input: unknown): LocateResultValue {\n  const bbox = unwrapCoordinateListLikeInput(input as any) as number[];\n  if (bbox.length < 2) {\n    const msg = `invalid bbox data for qwen-vl mode: ${JSON.stringify(bbox)} `;\n    throw new Error(msg);\n  }\n\n  if (typeof bbox[2] === 'number' && typeof bbox[3] === 'number') {\n    return createLocateResultValue(qwen25BboxCoordinatesMeta, [\n      bbox[0],\n      bbox[1],\n      bbox[2],\n      bbox[3],\n    ]);\n  }\n\n  return createLocateResultValue(qwen25PointCoordinatesMeta, [\n    bbox[0],\n    bbox[1],\n  ]);\n}\n\nfunction normalizeQwen25ResultToPixelBbox(\n  result: LocateResultValue,\n): PixelBbox {\n  if (isBboxLocateResultValue(result)) {\n    const { coordinates } = result;\n    return [\n      Math.round(coordinates[0]),\n      Math.round(coordinates[1]),\n      Math.round(coordinates[2]),\n      Math.round(coordinates[3]),\n    ];\n  }\n\n  const { coordinates } = result;\n  return topLeftPointToPixelBbox(coordinates[0], coordinates[1]);\n}\n\nconst buildQwenChatCompletionParams = (\n  input: ChatCompletionCallContext,\n): ChatCompletionParamsResult => {\n  const { midsceneDefaults, userConfig } = input;\n  const { reasoningEnabled, reasoningBudget } = userConfig;\n\n  const commonOverrideConfig: Record<string, unknown> = {};\n\n  if (userConfig.temperature !== undefined) {\n    commonOverrideConfig.temperature = userConfig.temperature;\n  }\n\n  // Alibaba Cloud Model Studio JSON mode:\n  // https://help.aliyun.com/zh/model-studio/json-mode\n  // Observed in qwen3.6 grounding runs: enabling this can make the model\n  // return only [\"bbox_2d\"] without coordinates.\n  // if (isLocateIntent(input.intent)) {\n  //   commonOverrideConfig.response_format = { type: 'json_object' };\n  // }\n\n  const modelSpecificConfig: Record<string, unknown> = {};\n\n  if (reasoningEnabled !== 'default') {\n    modelSpecificConfig.enable_thinking = reasoningEnabled ?? false;\n    if (reasoningBudget !== undefined) {\n      modelSpecificConfig.thinking_budget = reasoningBudget;\n    }\n  }\n\n  return {\n    config: {\n      ...midsceneDefaults,\n      ...commonOverrideConfig,\n      ...modelSpecificConfig,\n    },\n  };\n};\n\nconst buildQwen25ChatCompletionParams = (\n  input: ChatCompletionCallContext,\n): ChatCompletionParamsResult => {\n  const { midsceneDefaults, userConfig } = input;\n  const commonOverrideConfig: Record<string, unknown> = {};\n\n  if (userConfig.temperature !== undefined) {\n    commonOverrideConfig.temperature = userConfig.temperature;\n  }\n\n  return {\n    config: {\n      ...midsceneDefaults,\n      ...commonOverrideConfig,\n      vl_high_resolution_images: true,\n    },\n  };\n};\n\nconst qwen3Adapter: ModelAdapterDefinition = {\n  chatCompletion: {\n    unsupportedUserConfig: ['reasoningEffort'],\n    buildChatCompletionParams: buildQwenChatCompletionParams,\n    messageExtraction: {\n      kind: 'default',\n      // DashScope returns Qwen thinking as `reasoning_content`. vLLM's\n      // OpenAI-compatible server is migrating the recommended field to\n      // `reasoning`, so Qwen adapters accept both serving conventions:\n      // https://github.com/vllm-project/vllm/issues/27755\n      reasoningContentKeys: ['reasoning_content', 'reasoning'],\n    },\n    useReasoningAsContentFallback: true,\n  },\n  locate: {\n    resultAdapter: {\n      coordinates: qwen3BboxCoordinatesMeta,\n    },\n  },\n};\n\nexport const qwenAdapters = {\n  'qwen2.5-vl': {\n    chatCompletion: {\n      unsupportedUserConfig: [\n        'reasoningEnabled',\n        'reasoningEffort',\n        'reasoningBudget',\n      ],\n      buildChatCompletionParams: buildQwen25ChatCompletionParams,\n    },\n    imagePreprocess: {\n      padBlockSize: 28,\n    },\n    locate: {\n      resultAdapter: {\n        coordinates: qwen25BboxCoordinatesMeta,\n        parseRawLocateValue: parseQwen25RawLocateValue,\n        mapLocateResultToPixelBbox: normalizeQwen25ResultToPixelBbox,\n      },\n    },\n  },\n  'qwen3-vl': qwen3Adapter,\n  qwen3: qwen3Adapter,\n  'qwen3.5': qwen3Adapter,\n  'qwen3.6': qwen3Adapter,\n} satisfies Pick<\n  Record<TModelFamily, ModelAdapterDefinition>,\n  'qwen2.5-vl' | 'qwen3-vl' | 'qwen3' | 'qwen3.5' | 'qwen3.6'\n>;\n"],"names":["defaultBboxSize","qwen25BboxCoordinatesMeta","qwen25PointCoordinatesMeta","qwen3BboxCoordinatesMeta","topLeftPointToPixelBbox","x","y","Math","parseQwen25RawLocateValue","input","bbox","unwrapCoordinateListLikeInput","msg","JSON","Error","createLocateResultValue","normalizeQwen25ResultToPixelBbox","result","isBboxLocateResultValue","coordinates","buildQwenChatCompletionParams","midsceneDefaults","userConfig","reasoningEnabled","reasoningBudget","commonOverrideConfig","undefined","modelSpecificConfig","buildQwen25ChatCompletionParams","qwen3Adapter","qwenAdapters"],"mappings":";AAcA,MAAMA,kBAAkB;AACxB,MAAMC,4BAA4B;IAChC,OAAO;IACP,OAAO;AACT;AACA,MAAMC,6BAA6B;IACjC,OAAO;IACP,OAAO;AACT;AACA,MAAMC,2BAA2B;IAC/B,OAAO;IACP,OAAO;IACP,cAAc;AAChB;AAEA,SAASC,wBAAwBC,CAAS,EAAEC,CAAS;IACnD,OAAO;QACLC,KAAK,KAAK,CAACF;QACXE,KAAK,KAAK,CAACD;QACXC,KAAK,KAAK,CAACF,IAAIL;QACfO,KAAK,KAAK,CAACD,IAAIN;KAChB;AACH;AAEA,SAASQ,0BAA0BC,KAAc;IAC/C,MAAMC,OAAOC,8BAA8BF;IAC3C,IAAIC,KAAK,MAAM,GAAG,GAAG;QACnB,MAAME,MAAM,CAAC,oCAAoC,EAAEC,KAAK,SAAS,CAACH,MAAM,CAAC,CAAC;QAC1E,MAAM,IAAII,MAAMF;IAClB;IAEA,IAAI,AAAmB,YAAnB,OAAOF,IAAI,CAAC,EAAE,IAAiB,AAAmB,YAAnB,OAAOA,IAAI,CAAC,EAAE,EAC/C,OAAOK,wBAAwBd,2BAA2B;QACxDS,IAAI,CAAC,EAAE;QACPA,IAAI,CAAC,EAAE;QACPA,IAAI,CAAC,EAAE;QACPA,IAAI,CAAC,EAAE;KACR;IAGH,OAAOK,wBAAwBb,4BAA4B;QACzDQ,IAAI,CAAC,EAAE;QACPA,IAAI,CAAC,EAAE;KACR;AACH;AAEA,SAASM,iCACPC,MAAyB;IAEzB,IAAIC,wBAAwBD,SAAS;QACnC,MAAM,EAAEE,WAAW,EAAE,GAAGF;QACxB,OAAO;YACLV,KAAK,KAAK,CAACY,WAAW,CAAC,EAAE;YACzBZ,KAAK,KAAK,CAACY,WAAW,CAAC,EAAE;YACzBZ,KAAK,KAAK,CAACY,WAAW,CAAC,EAAE;YACzBZ,KAAK,KAAK,CAACY,WAAW,CAAC,EAAE;SAC1B;IACH;IAEA,MAAM,EAAEA,WAAW,EAAE,GAAGF;IACxB,OAAOb,wBAAwBe,WAAW,CAAC,EAAE,EAAEA,WAAW,CAAC,EAAE;AAC/D;AAEA,MAAMC,gCAAgC,CACpCX;IAEA,MAAM,EAAEY,gBAAgB,EAAEC,UAAU,EAAE,GAAGb;IACzC,MAAM,EAAEc,gBAAgB,EAAEC,eAAe,EAAE,GAAGF;IAE9C,MAAMG,uBAAgD,CAAC;IAEvD,IAAIH,AAA2BI,WAA3BJ,WAAW,WAAW,EACxBG,qBAAqB,WAAW,GAAGH,WAAW,WAAW;IAW3D,MAAMK,sBAA+C,CAAC;IAEtD,IAAIJ,AAAqB,cAArBA,kBAAgC;QAClCI,oBAAoB,eAAe,GAAGJ,oBAAoB;QAC1D,IAAIC,AAAoBE,WAApBF,iBACFG,oBAAoB,eAAe,GAAGH;IAE1C;IAEA,OAAO;QACL,QAAQ;YACN,GAAGH,gBAAgB;YACnB,GAAGI,oBAAoB;YACvB,GAAGE,mBAAmB;QACxB;IACF;AACF;AAEA,MAAMC,kCAAkC,CACtCnB;IAEA,MAAM,EAAEY,gBAAgB,EAAEC,UAAU,EAAE,GAAGb;IACzC,MAAMgB,uBAAgD,CAAC;IAEvD,IAAIH,AAA2BI,WAA3BJ,WAAW,WAAW,EACxBG,qBAAqB,WAAW,GAAGH,WAAW,WAAW;IAG3D,OAAO;QACL,QAAQ;YACN,GAAGD,gBAAgB;YACnB,GAAGI,oBAAoB;YACvB,2BAA2B;QAC7B;IACF;AACF;AAEA,MAAMI,eAAuC;IAC3C,gBAAgB;QACd,uBAAuB;YAAC;SAAkB;QAC1C,2BAA2BT;QAC3B,mBAAmB;YACjB,MAAM;YAKN,sBAAsB;gBAAC;gBAAqB;aAAY;QAC1D;QACA,+BAA+B;IACjC;IACA,QAAQ;QACN,eAAe;YACb,aAAajB;QACf;IACF;AACF;AAEO,MAAM2B,eAAe;IAC1B,cAAc;QACZ,gBAAgB;YACd,uBAAuB;gBACrB;gBACA;gBACA;aACD;YACD,2BAA2BF;QAC7B;QACA,iBAAiB;YACf,cAAc;QAChB;QACA,QAAQ;YACN,eAAe;gBACb,aAAa3B;gBACb,qBAAqBO;gBACrB,4BAA4BQ;YAC9B;QACF;IACF;IACA,YAAYa;IACZ,OAAOA;IACP,WAAWA;IACX,WAAWA;AACb"}

package/dist/es/ai-model/models/ui-tars/adapter.mjs

@@ -1,7 +1,6 @@
 import { UITarsModelVersion } from "@midscene/shared/env";
 import { assert } from "@midscene/shared/utils";
-import { jsonrepair } from "jsonrepair";
-import { extractJSONFromCodeBlock, safeParseJson } from "../../service-caller/json.mjs";
+import { parseModelResponseJson } from "../../service-caller/json.mjs";
 import { createLocateResultValue, unwrapCoordinateListLikeInput } from "../../shared/model-locate-result/index.mjs";
 import { createUiTarsPlanner } from "./planning.mjs";
 const defaultVlmUiTarsReplanningCycleLimit = 40;
@@ -15,44 +14,6 @@ const uiTarsPointCoordinatesMeta = {
     order: 'xy',
     normalizedBy: 1000
 };
-function normalizeJsonObject(obj, context = {}) {
-    if (null == obj) return obj;
-    if (Array.isArray(obj)) return obj.map((item)=>normalizeJsonObject(item, context));
-    if ('object' == typeof obj) {
-        const normalized = {};
-        for (const [key, value] of Object.entries(obj)){
-            const trimmedKey = key.trim();
-            const preserveStringValue = context.preserveStringValueKeys?.includes(trimmedKey) ?? false;
-            const normalizedValue = 'string' == typeof value ? preserveStringValue ? value : value.trim() : normalizeJsonObject(value, context);
-            normalized[trimmedKey] = normalizedValue;
-        }
-        return normalized;
-    }
-    return 'string' == typeof obj ? obj.trim() : obj;
-}
-function shouldRepairUiTarsLocateJson(source) {
-    return 'locate' === source || 'section-locator' === source || 'planning-action-param' === source;
-}
-function preprocessUiTarsLocateJson(input) {
-    if (input.includes('bbox')) while(/\d+\s+\d+/.test(input))input = input.replace(/(\d+)\s+(\d+)/g, '$1,$2');
-    return input;
-}
-const uiTarsJsonParser = (raw, context = {
-    source: 'generic-object'
-})=>{
-    const { source } = context;
-    try {
-        return safeParseJson(raw, context);
-    } catch (firstError) {
-        if (!shouldRepairUiTarsLocateJson(source)) throw firstError;
-        const jsonString = preprocessUiTarsLocateJson(extractJSONFromCodeBlock(raw));
-        try {
-            return normalizeJsonObject(JSON.parse(jsonrepair(jsonString)), context);
-        } catch (error) {
-            throw Error(`failed to parse LLM response into JSON. Error - ${String(error ?? firstError ?? 'unknown error')}. Response - \n ${raw}`);
-        }
-    }
-};
 function parseUiTarsRawLocateValue(input) {
     const bbox = unwrapCoordinateListLikeInput(input);
     if ('string' == typeof bbox) {
@@ -98,7 +59,7 @@ function parseUiTarsRawLocateValue(input) {
 }
 function createUiTarsAdapter(uiTarsModelVersion) {
     return {
-        jsonParser: uiTarsJsonParser,
+        jsonParser: parseModelResponseJson,
         chatCompletion: {
             unsupportedUserConfig: [
                 'reasoningEnabled',

package/dist/es/ai-model/models/ui-tars/adapter.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/models/ui-tars/adapter.mjs","sources":["../../../../../src/ai-model/models/ui-tars/adapter.ts"],"sourcesContent":["import { type TModelFamily, UITarsModelVersion } from '@midscene/shared/env';\nimport { assert } from '@midscene/shared/utils';\nimport { jsonrepair } from 'jsonrepair';\nimport type {\n  JsonParserContext,\n  JsonParserSource,\n  ModelAdapterDefinition,\n} from '../../model-adapter/types';\nimport {\n  extractJSONFromCodeBlock,\n  safeParseJson,\n} from '../../service-caller/json';\nimport {\n  type LocateResultValue,\n  createLocateResultValue,\n  unwrapCoordinateListLikeInput,\n} from '../../shared/model-locate-result';\nimport { createUiTarsPlanner } from './planning';\n\nconst defaultVlmUiTarsReplanningCycleLimit = 40;\nconst uiTarsBboxCoordinatesMeta = {\n  shape: 'bbox',\n  order: 'xy',\n  normalizedBy: 1000,\n} as const;\nconst uiTarsPointCoordinatesMeta = {\n  shape: 'point',\n  order: 'xy',\n  normalizedBy: 1000,\n} as const;\n\nfunction normalizeJsonObject(\n  obj: any,\n  context: Pick<JsonParserContext, 'preserveStringValueKeys'> = {},\n): any {\n  if (obj === null || obj === undefined) {\n    return obj;\n  }\n\n  if (Array.isArray(obj)) {\n    return obj.map((item) => normalizeJsonObject(item, context));\n  }\n\n  if (typeof obj === 'object') {\n    const normalized: any = {};\n    for (const [key, value] of Object.entries(obj)) {\n      const trimmedKey = key.trim();\n      const preserveStringValue =\n        context.preserveStringValueKeys?.includes(trimmedKey) ?? false;\n      const normalizedValue =\n        typeof value === 'string'\n          ? preserveStringValue\n            ? value\n            : value.trim()\n          : normalizeJsonObject(value, context);\n      normalized[trimmedKey] = normalizedValue;\n    }\n    return normalized;\n  }\n\n  return typeof obj === 'string' ? obj.trim() : obj;\n}\n\nfunction shouldRepairUiTarsLocateJson(source: JsonParserSource) {\n  return (\n    source === 'locate' ||\n    source === 'section-locator' ||\n    source === 'planning-action-param'\n  );\n}\n\nfunction preprocessUiTarsLocateJson(input: string) {\n  if (input.includes('bbox')) {\n    while (/\\d+\\s+\\d+/.test(input)) {\n      input = input.replace(/(\\d+)\\s+(\\d+)/g, '$1,$2');\n    }\n  }\n  return input;\n}\n\nconst uiTarsJsonParser: ModelAdapterDefinition['jsonParser'] = (\n  raw,\n  context = { source: 'generic-object' },\n) => {\n  const { source } = context;\n  try {\n    return safeParseJson(raw, context);\n  } catch (firstError) {\n    if (!shouldRepairUiTarsLocateJson(source)) {\n      throw firstError;\n    }\n\n    const jsonString = preprocessUiTarsLocateJson(\n      extractJSONFromCodeBlock(raw),\n    );\n    try {\n      return normalizeJsonObject(JSON.parse(jsonrepair(jsonString)), context);\n    } catch (error) {\n      throw Error(\n        `failed to parse LLM response into JSON. Error - ${String(\n          error ?? firstError ?? 'unknown error',\n        )}. Response - \\n ${raw}`,\n      );\n    }\n  }\n};\n\n// UI-TARS has not received active updates for a long time, so this parser is\n// intentionally kept separate from Doubao even though the current logic is the\n// same. This avoids coupling UI-TARS behavior to future Doubao adapter changes.\nfunction parseUiTarsRawLocateValue(input: unknown): LocateResultValue {\n  const bbox = unwrapCoordinateListLikeInput(input as any);\n  if (typeof bbox === 'string') {\n    assert(\n      /^(\\d+)\\s(\\d+)\\s(\\d+)\\s(\\d+)$/.test(bbox.trim()),\n      `invalid bbox data string for ui-tars mode: ${bbox}`,\n    );\n    const splitted = bbox.split(' ');\n    if (splitted.length === 4) {\n      return createLocateResultValue(uiTarsBboxCoordinatesMeta, [\n        Number(splitted[0]),\n        Number(splitted[1]),\n        Number(splitted[2]),\n        Number(splitted[3]),\n      ]);\n    }\n    throw new Error(`invalid bbox data string for ui-tars mode: ${bbox}`);\n  }\n\n  let bboxList: number[] = [];\n  if (Array.isArray(bbox) && typeof bbox[0] === 'string') {\n    bbox.forEach((item) => {\n      if (typeof item === 'string' && item.includes(',')) {\n        const [x, y] = item.split(',');\n        bboxList.push(Number(x.trim()), Number(y.trim()));\n      } else if (typeof item === 'string' && item.includes(' ')) {\n        const [x, y] = item.split(' ');\n        bboxList.push(Number(x.trim()), Number(y.trim()));\n      } else {\n        bboxList.push(Number(item));\n      }\n    });\n  } else {\n    bboxList = bbox as number[];\n  }\n\n  if (bboxList.length === 4 || bboxList.length === 5) {\n    return createLocateResultValue(uiTarsBboxCoordinatesMeta, [\n      bboxList[0],\n      bboxList[1],\n      bboxList[2],\n      bboxList[3],\n    ]);\n  }\n\n  if (\n    bboxList.length === 6 ||\n    bboxList.length === 2 ||\n    bboxList.length === 3 ||\n    bboxList.length === 7\n  ) {\n    return createLocateResultValue(uiTarsPointCoordinatesMeta, [\n      bboxList[0],\n      bboxList[1],\n    ]);\n  }\n\n  if (bbox.length === 8) {\n    return createLocateResultValue(uiTarsBboxCoordinatesMeta, [\n      bboxList[0],\n      bboxList[1],\n      bboxList[4],\n      bboxList[5],\n    ]);\n  }\n\n  const msg = `invalid bbox data for ui-tars mode: ${JSON.stringify(bbox)} `;\n  throw new Error(msg);\n}\n\nfunction createUiTarsAdapter(\n  uiTarsModelVersion: UITarsModelVersion,\n): ModelAdapterDefinition {\n  return {\n    jsonParser: uiTarsJsonParser,\n    chatCompletion: {\n      unsupportedUserConfig: [\n        'reasoningEnabled',\n        'reasoningEffort',\n        'reasoningBudget',\n      ],\n      buildChatCompletionParams: ({ midsceneDefaults, userConfig }) => {\n        const commonOverrideConfig: Record<string, unknown> = {};\n\n        if (userConfig.temperature !== undefined) {\n          commonOverrideConfig.temperature = userConfig.temperature;\n        }\n\n        return {\n          config: {\n            ...midsceneDefaults,\n            ...commonOverrideConfig,\n          },\n        };\n      },\n    },\n    planning: {\n      kind: 'custom',\n      cacheEnabled: false,\n      defaultReplanningCycleLimit: defaultVlmUiTarsReplanningCycleLimit,\n      planner: createUiTarsPlanner(uiTarsModelVersion),\n    },\n    locate: {\n      resultAdapter: {\n        coordinates: uiTarsBboxCoordinatesMeta,\n        parseRawLocateValue: parseUiTarsRawLocateValue,\n      },\n    },\n  };\n}\n\nconst uiTarsDoubao15Adapter = createUiTarsAdapter(\n  UITarsModelVersion.DOUBAO_1_5_20B,\n);\n\nexport const uiTarsAdapters = {\n  'vlm-ui-tars': createUiTarsAdapter(UITarsModelVersion.V1_0),\n  'vlm-ui-tars-doubao': uiTarsDoubao15Adapter,\n  'vlm-ui-tars-doubao-1.5': uiTarsDoubao15Adapter,\n} satisfies Pick<\n  Record<TModelFamily, ModelAdapterDefinition>,\n  'vlm-ui-tars' | 'vlm-ui-tars-doubao' | 'vlm-ui-tars-doubao-1.5'\n>;\n"],"names":["defaultVlmUiTarsReplanningCycleLimit","uiTarsBboxCoordinatesMeta","uiTarsPointCoordinatesMeta","normalizeJsonObject","obj","context","Array","item","normalized","key","value","Object","trimmedKey","preserveStringValue","normalizedValue","shouldRepairUiTarsLocateJson","source","preprocessUiTarsLocateJson","input","uiTarsJsonParser","raw","safeParseJson","firstError","jsonString","extractJSONFromCodeBlock","JSON","jsonrepair","error","Error","String","parseUiTarsRawLocateValue","bbox","unwrapCoordinateListLikeInput","assert","splitted","createLocateResultValue","Number","bboxList","x","y","msg","createUiTarsAdapter","uiTarsModelVersion","midsceneDefaults","userConfig","commonOverrideConfig","undefined","createUiTarsPlanner","uiTarsDoubao15Adapter","UITarsModelVersion","uiTarsAdapters"],"mappings":";;;;;;AAmBA,MAAMA,uCAAuC;AAC7C,MAAMC,4BAA4B;IAChC,OAAO;IACP,OAAO;IACP,cAAc;AAChB;AACA,MAAMC,6BAA6B;IACjC,OAAO;IACP,OAAO;IACP,cAAc;AAChB;AAEA,SAASC,oBACPC,GAAQ,EACRC,UAA8D,CAAC,CAAC;IAEhE,IAAID,QAAAA,KACF,OAAOA;IAGT,IAAIE,MAAM,OAAO,CAACF,MAChB,OAAOA,IAAI,GAAG,CAAC,CAACG,OAASJ,oBAAoBI,MAAMF;IAGrD,IAAI,AAAe,YAAf,OAAOD,KAAkB;QAC3B,MAAMI,aAAkB,CAAC;QACzB,KAAK,MAAM,CAACC,KAAKC,MAAM,IAAIC,OAAO,OAAO,CAACP,KAAM;YAC9C,MAAMQ,aAAaH,IAAI,IAAI;YAC3B,MAAMI,sBACJR,QAAQ,uBAAuB,EAAE,SAASO,eAAe;YAC3D,MAAME,kBACJ,AAAiB,YAAjB,OAAOJ,QACHG,sBACEH,QACAA,MAAM,IAAI,KACZP,oBAAoBO,OAAOL;YACjCG,UAAU,CAACI,WAAW,GAAGE;QAC3B;QACA,OAAON;IACT;IAEA,OAAO,AAAe,YAAf,OAAOJ,MAAmBA,IAAI,IAAI,KAAKA;AAChD;AAEA,SAASW,6BAA6BC,MAAwB;IAC5D,OACEA,AAAW,aAAXA,UACAA,AAAW,sBAAXA,UACAA,AAAW,4BAAXA;AAEJ;AAEA,SAASC,2BAA2BC,KAAa;IAC/C,IAAIA,MAAM,QAAQ,CAAC,SACjB,MAAO,YAAY,IAAI,CAACA,OACtBA,QAAQA,MAAM,OAAO,CAAC,kBAAkB;IAG5C,OAAOA;AACT;AAEA,MAAMC,mBAAyD,CAC7DC,KACAf,UAAU;IAAE,QAAQ;AAAiB,CAAC;IAEtC,MAAM,EAAEW,MAAM,EAAE,GAAGX;IACnB,IAAI;QACF,OAAOgB,cAAcD,KAAKf;IAC5B,EAAE,OAAOiB,YAAY;QACnB,IAAI,CAACP,6BAA6BC,SAChC,MAAMM;QAGR,MAAMC,aAAaN,2BACjBO,yBAAyBJ;QAE3B,IAAI;YACF,OAAOjB,oBAAoBsB,KAAK,KAAK,CAACC,WAAWH,cAAclB;QACjE,EAAE,OAAOsB,OAAO;YACd,MAAMC,MACJ,CAAC,gDAAgD,EAAEC,OACjDF,SAASL,cAAc,iBACvB,gBAAgB,EAAEF,KAAK;QAE7B;IACF;AACF;AAKA,SAASU,0BAA0BZ,KAAc;IAC/C,MAAMa,OAAOC,8BAA8Bd;IAC3C,IAAI,AAAgB,YAAhB,OAAOa,MAAmB;QAC5BE,OACE,+BAA+B,IAAI,CAACF,KAAK,IAAI,KAC7C,CAAC,2CAA2C,EAAEA,MAAM;QAEtD,MAAMG,WAAWH,KAAK,KAAK,CAAC;QAC5B,IAAIG,AAAoB,MAApBA,SAAS,MAAM,EACjB,OAAOC,wBAAwBlC,2BAA2B;YACxDmC,OAAOF,QAAQ,CAAC,EAAE;YAClBE,OAAOF,QAAQ,CAAC,EAAE;YAClBE,OAAOF,QAAQ,CAAC,EAAE;YAClBE,OAAOF,QAAQ,CAAC,EAAE;SACnB;QAEH,MAAM,IAAIN,MAAM,CAAC,2CAA2C,EAAEG,MAAM;IACtE;IAEA,IAAIM,WAAqB,EAAE;IAC3B,IAAI/B,MAAM,OAAO,CAACyB,SAAS,AAAmB,YAAnB,OAAOA,IAAI,CAAC,EAAE,EACvCA,KAAK,OAAO,CAAC,CAACxB;QACZ,IAAI,AAAgB,YAAhB,OAAOA,QAAqBA,KAAK,QAAQ,CAAC,MAAM;YAClD,MAAM,CAAC+B,GAAGC,EAAE,GAAGhC,KAAK,KAAK,CAAC;YAC1B8B,SAAS,IAAI,CAACD,OAAOE,EAAE,IAAI,KAAKF,OAAOG,EAAE,IAAI;QAC/C,OAAO,IAAI,AAAgB,YAAhB,OAAOhC,QAAqBA,KAAK,QAAQ,CAAC,MAAM;YACzD,MAAM,CAAC+B,GAAGC,EAAE,GAAGhC,KAAK,KAAK,CAAC;YAC1B8B,SAAS,IAAI,CAACD,OAAOE,EAAE,IAAI,KAAKF,OAAOG,EAAE,IAAI;QAC/C,OACEF,SAAS,IAAI,CAACD,OAAO7B;IAEzB;SAEA8B,WAAWN;IAGb,IAAIM,AAAoB,MAApBA,SAAS,MAAM,IAAUA,AAAoB,MAApBA,SAAS,MAAM,EAC1C,OAAOF,wBAAwBlC,2BAA2B;QACxDoC,QAAQ,CAAC,EAAE;QACXA,QAAQ,CAAC,EAAE;QACXA,QAAQ,CAAC,EAAE;QACXA,QAAQ,CAAC,EAAE;KACZ;IAGH,IACEA,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,EAEf,OAAOF,wBAAwBjC,4BAA4B;QACzDmC,QAAQ,CAAC,EAAE;QACXA,QAAQ,CAAC,EAAE;KACZ;IAGH,IAAIN,AAAgB,MAAhBA,KAAK,MAAM,EACb,OAAOI,wBAAwBlC,2BAA2B;QACxDoC,QAAQ,CAAC,EAAE;QACXA,QAAQ,CAAC,EAAE;QACXA,QAAQ,CAAC,EAAE;QACXA,QAAQ,CAAC,EAAE;KACZ;IAGH,MAAMG,MAAM,CAAC,oCAAoC,EAAEf,KAAK,SAAS,CAACM,MAAM,CAAC,CAAC;IAC1E,MAAM,IAAIH,MAAMY;AAClB;AAEA,SAASC,oBACPC,kBAAsC;IAEtC,OAAO;QACL,YAAYvB;QACZ,gBAAgB;YACd,uBAAuB;gBACrB;gBACA;gBACA;aACD;YACD,2BAA2B,CAAC,EAAEwB,gBAAgB,EAAEC,UAAU,EAAE;gBAC1D,MAAMC,uBAAgD,CAAC;gBAEvD,IAAID,AAA2BE,WAA3BF,WAAW,WAAW,EACxBC,qBAAqB,WAAW,GAAGD,WAAW,WAAW;gBAG3D,OAAO;oBACL,QAAQ;wBACN,GAAGD,gBAAgB;wBACnB,GAAGE,oBAAoB;oBACzB;gBACF;YACF;QACF;QACA,UAAU;YACR,MAAM;YACN,cAAc;YACd,6BAA6B7C;YAC7B,SAAS+C,oBAAoBL;QAC/B;QACA,QAAQ;YACN,eAAe;gBACb,aAAazC;gBACb,qBAAqB6B;YACvB;QACF;IACF;AACF;AAEA,MAAMkB,wBAAwBP,oBAC5BQ,mBAAmB,cAAc;AAG5B,MAAMC,iBAAiB;IAC5B,eAAeT,oBAAoBQ,mBAAmB,IAAI;IAC1D,sBAAsBD;IACtB,0BAA0BA;AAC5B"}
+{"version":3,"file":"ai-model/models/ui-tars/adapter.mjs","sources":["../../../../../src/ai-model/models/ui-tars/adapter.ts"],"sourcesContent":["import { type TModelFamily, UITarsModelVersion } from '@midscene/shared/env';\nimport { assert } from '@midscene/shared/utils';\nimport type { ModelAdapterDefinition } from '../../model-adapter/types';\nimport { parseModelResponseJson } from '../../service-caller/json';\nimport {\n  type LocateResultValue,\n  createLocateResultValue,\n  unwrapCoordinateListLikeInput,\n} from '../../shared/model-locate-result';\nimport { createUiTarsPlanner } from './planning';\n\nconst defaultVlmUiTarsReplanningCycleLimit = 40;\nconst uiTarsBboxCoordinatesMeta = {\n  shape: 'bbox',\n  order: 'xy',\n  normalizedBy: 1000,\n} as const;\nconst uiTarsPointCoordinatesMeta = {\n  shape: 'point',\n  order: 'xy',\n  normalizedBy: 1000,\n} as const;\n\n// UI-TARS has not received active updates for a long time, so this parser is\n// intentionally kept separate from Doubao even though the current logic is the\n// same. This avoids coupling UI-TARS behavior to future Doubao adapter changes.\nfunction parseUiTarsRawLocateValue(input: unknown): LocateResultValue {\n  const bbox = unwrapCoordinateListLikeInput(input as any);\n  if (typeof bbox === 'string') {\n    assert(\n      /^(\\d+)\\s(\\d+)\\s(\\d+)\\s(\\d+)$/.test(bbox.trim()),\n      `invalid bbox data string for ui-tars mode: ${bbox}`,\n    );\n    const splitted = bbox.split(' ');\n    if (splitted.length === 4) {\n      return createLocateResultValue(uiTarsBboxCoordinatesMeta, [\n        Number(splitted[0]),\n        Number(splitted[1]),\n        Number(splitted[2]),\n        Number(splitted[3]),\n      ]);\n    }\n    throw new Error(`invalid bbox data string for ui-tars mode: ${bbox}`);\n  }\n\n  let bboxList: number[] = [];\n  if (Array.isArray(bbox) && typeof bbox[0] === 'string') {\n    bbox.forEach((item) => {\n      if (typeof item === 'string' && item.includes(',')) {\n        const [x, y] = item.split(',');\n        bboxList.push(Number(x.trim()), Number(y.trim()));\n      } else if (typeof item === 'string' && item.includes(' ')) {\n        const [x, y] = item.split(' ');\n        bboxList.push(Number(x.trim()), Number(y.trim()));\n      } else {\n        bboxList.push(Number(item));\n      }\n    });\n  } else {\n    bboxList = bbox as number[];\n  }\n\n  if (bboxList.length === 4 || bboxList.length === 5) {\n    return createLocateResultValue(uiTarsBboxCoordinatesMeta, [\n      bboxList[0],\n      bboxList[1],\n      bboxList[2],\n      bboxList[3],\n    ]);\n  }\n\n  if (\n    bboxList.length === 6 ||\n    bboxList.length === 2 ||\n    bboxList.length === 3 ||\n    bboxList.length === 7\n  ) {\n    return createLocateResultValue(uiTarsPointCoordinatesMeta, [\n      bboxList[0],\n      bboxList[1],\n    ]);\n  }\n\n  if (bbox.length === 8) {\n    return createLocateResultValue(uiTarsBboxCoordinatesMeta, [\n      bboxList[0],\n      bboxList[1],\n      bboxList[4],\n      bboxList[5],\n    ]);\n  }\n\n  const msg = `invalid bbox data for ui-tars mode: ${JSON.stringify(bbox)} `;\n  throw new Error(msg);\n}\n\nfunction createUiTarsAdapter(\n  uiTarsModelVersion: UITarsModelVersion,\n): ModelAdapterDefinition {\n  return {\n    jsonParser: parseModelResponseJson,\n    chatCompletion: {\n      unsupportedUserConfig: [\n        'reasoningEnabled',\n        'reasoningEffort',\n        'reasoningBudget',\n      ],\n      buildChatCompletionParams: ({ midsceneDefaults, userConfig }) => {\n        const commonOverrideConfig: Record<string, unknown> = {};\n\n        if (userConfig.temperature !== undefined) {\n          commonOverrideConfig.temperature = userConfig.temperature;\n        }\n\n        return {\n          config: {\n            ...midsceneDefaults,\n            ...commonOverrideConfig,\n          },\n        };\n      },\n    },\n    planning: {\n      kind: 'custom',\n      cacheEnabled: false,\n      defaultReplanningCycleLimit: defaultVlmUiTarsReplanningCycleLimit,\n      planner: createUiTarsPlanner(uiTarsModelVersion),\n    },\n    locate: {\n      resultAdapter: {\n        coordinates: uiTarsBboxCoordinatesMeta,\n        parseRawLocateValue: parseUiTarsRawLocateValue,\n      },\n    },\n  };\n}\n\nconst uiTarsDoubao15Adapter = createUiTarsAdapter(\n  UITarsModelVersion.DOUBAO_1_5_20B,\n);\n\nexport const uiTarsAdapters = {\n  'vlm-ui-tars': createUiTarsAdapter(UITarsModelVersion.V1_0),\n  'vlm-ui-tars-doubao': uiTarsDoubao15Adapter,\n  'vlm-ui-tars-doubao-1.5': uiTarsDoubao15Adapter,\n} satisfies Pick<\n  Record<TModelFamily, ModelAdapterDefinition>,\n  'vlm-ui-tars' | 'vlm-ui-tars-doubao' | 'vlm-ui-tars-doubao-1.5'\n>;\n"],"names":["defaultVlmUiTarsReplanningCycleLimit","uiTarsBboxCoordinatesMeta","uiTarsPointCoordinatesMeta","parseUiTarsRawLocateValue","input","bbox","unwrapCoordinateListLikeInput","assert","splitted","createLocateResultValue","Number","Error","bboxList","Array","item","x","y","msg","JSON","createUiTarsAdapter","uiTarsModelVersion","parseModelResponseJson","midsceneDefaults","userConfig","commonOverrideConfig","undefined","createUiTarsPlanner","uiTarsDoubao15Adapter","UITarsModelVersion","uiTarsAdapters"],"mappings":";;;;;AAWA,MAAMA,uCAAuC;AAC7C,MAAMC,4BAA4B;IAChC,OAAO;IACP,OAAO;IACP,cAAc;AAChB;AACA,MAAMC,6BAA6B;IACjC,OAAO;IACP,OAAO;IACP,cAAc;AAChB;AAKA,SAASC,0BAA0BC,KAAc;IAC/C,MAAMC,OAAOC,8BAA8BF;IAC3C,IAAI,AAAgB,YAAhB,OAAOC,MAAmB;QAC5BE,OACE,+BAA+B,IAAI,CAACF,KAAK,IAAI,KAC7C,CAAC,2CAA2C,EAAEA,MAAM;QAEtD,MAAMG,WAAWH,KAAK,KAAK,CAAC;QAC5B,IAAIG,AAAoB,MAApBA,SAAS,MAAM,EACjB,OAAOC,wBAAwBR,2BAA2B;YACxDS,OAAOF,QAAQ,CAAC,EAAE;YAClBE,OAAOF,QAAQ,CAAC,EAAE;YAClBE,OAAOF,QAAQ,CAAC,EAAE;YAClBE,OAAOF,QAAQ,CAAC,EAAE;SACnB;QAEH,MAAM,IAAIG,MAAM,CAAC,2CAA2C,EAAEN,MAAM;IACtE;IAEA,IAAIO,WAAqB,EAAE;IAC3B,IAAIC,MAAM,OAAO,CAACR,SAAS,AAAmB,YAAnB,OAAOA,IAAI,CAAC,EAAE,EACvCA,KAAK,OAAO,CAAC,CAACS;QACZ,IAAI,AAAgB,YAAhB,OAAOA,QAAqBA,KAAK,QAAQ,CAAC,MAAM;YAClD,MAAM,CAACC,GAAGC,EAAE,GAAGF,KAAK,KAAK,CAAC;YAC1BF,SAAS,IAAI,CAACF,OAAOK,EAAE,IAAI,KAAKL,OAAOM,EAAE,IAAI;QAC/C,OAAO,IAAI,AAAgB,YAAhB,OAAOF,QAAqBA,KAAK,QAAQ,CAAC,MAAM;YACzD,MAAM,CAACC,GAAGC,EAAE,GAAGF,KAAK,KAAK,CAAC;YAC1BF,SAAS,IAAI,CAACF,OAAOK,EAAE,IAAI,KAAKL,OAAOM,EAAE,IAAI;QAC/C,OACEJ,SAAS,IAAI,CAACF,OAAOI;IAEzB;SAEAF,WAAWP;IAGb,IAAIO,AAAoB,MAApBA,SAAS,MAAM,IAAUA,AAAoB,MAApBA,SAAS,MAAM,EAC1C,OAAOH,wBAAwBR,2BAA2B;QACxDW,QAAQ,CAAC,EAAE;QACXA,QAAQ,CAAC,EAAE;QACXA,QAAQ,CAAC,EAAE;QACXA,QAAQ,CAAC,EAAE;KACZ;IAGH,IACEA,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,EAEf,OAAOH,wBAAwBP,4BAA4B;QACzDU,QAAQ,CAAC,EAAE;QACXA,QAAQ,CAAC,EAAE;KACZ;IAGH,IAAIP,AAAgB,MAAhBA,KAAK,MAAM,EACb,OAAOI,wBAAwBR,2BAA2B;QACxDW,QAAQ,CAAC,EAAE;QACXA,QAAQ,CAAC,EAAE;QACXA,QAAQ,CAAC,EAAE;QACXA,QAAQ,CAAC,EAAE;KACZ;IAGH,MAAMK,MAAM,CAAC,oCAAoC,EAAEC,KAAK,SAAS,CAACb,MAAM,CAAC,CAAC;IAC1E,MAAM,IAAIM,MAAMM;AAClB;AAEA,SAASE,oBACPC,kBAAsC;IAEtC,OAAO;QACL,YAAYC;QACZ,gBAAgB;YACd,uBAAuB;gBACrB;gBACA;gBACA;aACD;YACD,2BAA2B,CAAC,EAAEC,gBAAgB,EAAEC,UAAU,EAAE;gBAC1D,MAAMC,uBAAgD,CAAC;gBAEvD,IAAID,AAA2BE,WAA3BF,WAAW,WAAW,EACxBC,qBAAqB,WAAW,GAAGD,WAAW,WAAW;gBAG3D,OAAO;oBACL,QAAQ;wBACN,GAAGD,gBAAgB;wBACnB,GAAGE,oBAAoB;oBACzB;gBACF;YACF;QACF;QACA,UAAU;YACR,MAAM;YACN,cAAc;YACd,6BAA6BxB;YAC7B,SAAS0B,oBAAoBN;QAC/B;QACA,QAAQ;YACN,eAAe;gBACb,aAAanB;gBACb,qBAAqBE;YACvB;QACF;IACF;AACF;AAEA,MAAMwB,wBAAwBR,oBAC5BS,mBAAmB,cAAc;AAG5B,MAAMC,iBAAiB;IAC5B,eAAeV,oBAAoBS,mBAAmB,IAAI;IAC1D,sBAAsBD;IACtB,0BAA0BA;AAC5B"}

package/dist/es/ai-model/models/utils/intent.mjs

@@ -0,0 +1,6 @@
+function isLocateIntent(intent) {
+    return 'default' === intent;
+}
+export { isLocateIntent };
+
+//# sourceMappingURL=intent.mjs.map

package/dist/es/ai-model/models/utils/intent.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/models/utils/intent.mjs","sources":["../../../../../src/ai-model/models/utils/intent.ts"],"sourcesContent":["import type { TIntent } from '@midscene/shared/env';\n\nexport function isLocateIntent(intent?: TIntent): boolean {\n  return intent === 'default';\n}\n"],"names":["isLocateIntent","intent"],"mappings":"AAEO,SAASA,eAAeC,MAAgB;IAC7C,OAAOA,AAAW,cAAXA;AACT"}

package/dist/es/ai-model/prompt/extraction.mjs

@@ -1,5 +1,5 @@
 import { getPreferredLanguage } from "@midscene/shared/env";
-import { safeParseJson } from "../service-caller/json.mjs";
+import { parseModelResponseJson } from "../service-caller/json.mjs";
 import { extractXMLTag } from "./util.mjs";
 function buildTypeQueryDemandValue(type, demand) {
     const currentScreenshotConstraint = 'based on the current screenshot and its contents if provided, unless the user explicitly asks to compare with reference images';
@@ -14,13 +14,19 @@ function parseXMLExtractionResponse(xmlString) {
     if (!dataJsonStr) throw new Error('Missing required field: data-json');
     let data;
     try {
-        data = safeParseJson(dataJsonStr);
+        data = parseModelResponseJson(dataJsonStr, {
+            source: 'generic-object',
+            requireObject: false
+        });
     } catch (e) {
         throw new Error(`Failed to parse data-json: ${e}`);
     }
     let errors;
     if (errorsStr) try {
-        const parsedErrors = safeParseJson(errorsStr);
+        const parsedErrors = parseModelResponseJson(errorsStr, {
+            source: 'generic-object',
+            requireObject: false
+        });
         if (Array.isArray(parsedErrors)) errors = parsedErrors;
     } catch (e) {}
     return {

package/dist/es/ai-model/prompt/extraction.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/prompt/extraction.mjs","sources":["../../../../src/ai-model/prompt/extraction.ts"],"sourcesContent":["import type { AIDataExtractionResponse, ServiceExtractParam } from '@/types';\nimport { getPreferredLanguage } from '@midscene/shared/env';\nimport { safeParseJson } from '../service-caller/json';\nimport { extractXMLTag } from './util';\n\nexport function buildTypeQueryDemandValue(\n  type: 'Boolean' | 'Number' | 'String' | 'Assert' | 'WaitFor',\n  demand: ServiceExtractParam,\n) {\n  const currentScreenshotConstraint =\n    'based on the current screenshot and its contents if provided, unless the user explicitly asks to compare with reference images';\n\n  if (type === 'Assert') {\n    return `Boolean, ${currentScreenshotConstraint}, whether the following statement is true: ${demand}`;\n  }\n\n  if (type === 'WaitFor') {\n    return `Boolean, the user wants to do some 'wait for' operation. ${currentScreenshotConstraint}, please check whether the following statement is true: ${demand}`;\n  }\n\n  return `${type}, ${currentScreenshotConstraint}, ${demand}`;\n}\n\n/**\n * Parse XML response from LLM and convert to AIDataExtractionResponse\n */\nexport function parseXMLExtractionResponse<T>(\n  xmlString: string,\n): AIDataExtractionResponse<T> {\n  // Keep the internal field named `thought`, but ask models to emit\n  // <observation>. Gemini may only return <thought>-named content when\n  // thinking summaries are enabled.\n  const thought = extractXMLTag(xmlString, 'observation');\n  const dataJsonStr = extractXMLTag(xmlString, 'data-json');\n  const errorsStr = extractXMLTag(xmlString, 'errors');\n\n  // Parse data-json (required)\n  if (!dataJsonStr) {\n    throw new Error('Missing required field: data-json');\n  }\n\n  let data: T;\n  try {\n    data = safeParseJson(dataJsonStr) as T;\n  } catch (e) {\n    throw new Error(`Failed to parse data-json: ${e}`);\n  }\n\n  // Parse errors (optional)\n  let errors: string[] | undefined;\n  if (errorsStr) {\n    try {\n      const parsedErrors = safeParseJson(errorsStr);\n      if (Array.isArray(parsedErrors)) {\n        errors = parsedErrors;\n      }\n    } catch (e) {\n      // If errors parsing fails, just ignore it\n    }\n  }\n\n  return {\n    ...(thought ? { thought } : {}),\n    data,\n    ...(errors && errors.length > 0 ? { errors } : {}),\n  };\n}\n\nexport function systemPromptToExtract(options?: {\n  screenshotIncluded?: boolean;\n  referenceImagesIncluded?: boolean;\n}) {\n  const preferredLanguage = getPreferredLanguage();\n  const screenshotIncluded = options?.screenshotIncluded ?? true;\n  const referenceImagesIncluded = options?.referenceImagesIncluded ?? false;\n\n  const contextPrompts = [\n    \"The user will give you data requirements in <DATA_DEMAND>. You need to understand the user's requirements and extract the data satisfying the <DATA_DEMAND>.\",\n  ];\n\n  if (screenshotIncluded) {\n    contextPrompts.push(\n      'The user will provide a current screenshot to evaluate, and may provide its contents. Base your answer on the current screenshot and its contents when provided. Treat them as the primary source of truth for what is currently visible or true.',\n    );\n  } else {\n    contextPrompts.push(\n      'The user will not provide a current screenshot. Use only the supplied page contents and other inputs, and do not infer unsupported visual details.',\n    );\n  }\n\n  if (referenceImagesIncluded) {\n    const referenceImagesPrompt =\n      'Reference images are supporting context only unless <DATA_DEMAND> explicitly asks for comparison, matching, or reasoning about them.';\n    contextPrompts.push(\n      screenshotIncluded\n        ? `${referenceImagesPrompt} Do not conclude that something exists in the current screenshot solely because it appears in a reference image; when they conflict, trust the current screenshot and its contents.`\n        : `${referenceImagesPrompt} Do not treat reference images as direct evidence of the current state unless the demand explicitly asks you to use them that way.`,\n    );\n  }\n  const contextPrompt = contextPrompts.join('\\n\\n');\n\n  return `\nYou are a versatile professional in software UI design and testing. Your outstanding contributions will impact the user experience of billions of users.\n\n${contextPrompt}\n\nIf a key specifies a JSON data type (such as Number, String, Boolean, Object, Array), ensure the returned value strictly matches that data type.\n\nWhen DATA_DEMAND is a JSON object, the keys in your response must exactly match the keys in DATA_DEMAND. Do not rename, translate, or substitute any key.\n\n\nReturn in the following XML format:\n<observation>brief evidence observed for the extraction, less than 300 words. Use ${preferredLanguage} in this field.</observation>\n<data-json>the extracted data as JSON. Make sure both the value and scheme meet the DATA_DEMAND. If you want to write some description in this field, use the same language as the DATA_DEMAND.</data-json>\n<errors>optional error messages as JSON array, e.g., [\"error1\", \"error2\"]</errors>\n\n# Example 1\nFor example, if the DATA_DEMAND is:\n\n<DATA_DEMAND>\n{\n  \"name\": \"name shows on the left panel, string\",\n  \"age\": \"age shows on the right panel, number\",\n  \"isAdmin\": \"if the user is admin, boolean\"\n}\n</DATA_DEMAND>\n\nBy viewing the screenshot and page contents, you can extract the following data:\n\n<observation>According to the screenshot, i can see ...</observation>\n<data-json>\n{\n  \"name\": \"John\",\n  \"age\": 30,\n  \"isAdmin\": true\n}\n</data-json>\n\n# Example 2\nIf the DATA_DEMAND is:\n\n<DATA_DEMAND>\nthe todo items list, string[]\n</DATA_DEMAND>\n\nBy viewing the screenshot and page contents, you can extract the following data:\n\n<observation>According to the screenshot, i can see ...</observation>\n<data-json>\n[\"todo 1\", \"todo 2\", \"todo 3\"]\n</data-json>\n\n# Example 3\nIf the DATA_DEMAND is:\n\n<DATA_DEMAND>\nthe page title, string\n</DATA_DEMAND>\n\nBy viewing the screenshot and page contents, you can extract the following data:\n\n<observation>According to the screenshot, i can see ...</observation>\n<data-json>\n\"todo list\"\n</data-json>\n\n# Example 4\nIf the DATA_DEMAND is:\n\n<DATA_DEMAND>\n{\n  \"StatementIsTruthy\": \"Boolean, is it currently the SMS page?\"\n}\n</DATA_DEMAND>\n\nBy viewing the screenshot and page contents, you can extract the following data:\n\n<observation>According to the screenshot, i can see ...</observation>\n<data-json>\n{ \"StatementIsTruthy\": true }\n</data-json>\n`;\n}\n\nexport const extractDataQueryPrompt = (\n  pageDescription: string,\n  dataQuery: string | Record<string, string>,\n) => {\n  let dataQueryText = '';\n  if (typeof dataQuery === 'string') {\n    dataQueryText = dataQuery;\n  } else {\n    dataQueryText = JSON.stringify(dataQuery, null, 2);\n  }\n\n  return `\n<PageDescription>\n${pageDescription}\n</PageDescription>\n\n<DATA_DEMAND>\n${dataQueryText}\n</DATA_DEMAND>\n  `;\n};\n"],"names":["buildTypeQueryDemandValue","type","demand","currentScreenshotConstraint","parseXMLExtractionResponse","xmlString","thought","extractXMLTag","dataJsonStr","errorsStr","Error","data","safeParseJson","e","errors","parsedErrors","Array","systemPromptToExtract","options","preferredLanguage","getPreferredLanguage","screenshotIncluded","referenceImagesIncluded","contextPrompts","referenceImagesPrompt","contextPrompt","extractDataQueryPrompt","pageDescription","dataQuery","dataQueryText","JSON"],"mappings":";;;AAKO,SAASA,0BACdC,IAA4D,EAC5DC,MAA2B;IAE3B,MAAMC,8BACJ;IAEF,IAAIF,AAAS,aAATA,MACF,OAAO,CAAC,SAAS,EAAEE,4BAA4B,2CAA2C,EAAED,QAAQ;IAGtG,IAAID,AAAS,cAATA,MACF,OAAO,CAAC,yDAAyD,EAAEE,4BAA4B,wDAAwD,EAAED,QAAQ;IAGnK,OAAO,GAAGD,KAAK,EAAE,EAAEE,4BAA4B,EAAE,EAAED,QAAQ;AAC7D;AAKO,SAASE,2BACdC,SAAiB;IAKjB,MAAMC,UAAUC,cAAcF,WAAW;IACzC,MAAMG,cAAcD,cAAcF,WAAW;IAC7C,MAAMI,YAAYF,cAAcF,WAAW;IAG3C,IAAI,CAACG,aACH,MAAM,IAAIE,MAAM;IAGlB,IAAIC;IACJ,IAAI;QACFA,OAAOC,cAAcJ;IACvB,EAAE,OAAOK,GAAG;QACV,MAAM,IAAIH,MAAM,CAAC,2BAA2B,EAAEG,GAAG;IACnD;IAGA,IAAIC;IACJ,IAAIL,WACF,IAAI;QACF,MAAMM,eAAeH,cAAcH;QACnC,IAAIO,MAAM,OAAO,CAACD,eAChBD,SAASC;IAEb,EAAE,OAAOF,GAAG,CAEZ;IAGF,OAAO;QACL,GAAIP,UAAU;YAAEA;QAAQ,IAAI,CAAC,CAAC;QAC9BK;QACA,GAAIG,UAAUA,OAAO,MAAM,GAAG,IAAI;YAAEA;QAAO,IAAI,CAAC,CAAC;IACnD;AACF;AAEO,SAASG,sBAAsBC,OAGrC;IACC,MAAMC,oBAAoBC;IAC1B,MAAMC,qBAAqBH,SAAS,sBAAsB;IAC1D,MAAMI,0BAA0BJ,SAAS,2BAA2B;IAEpE,MAAMK,iBAAiB;QACrB;KACD;IAED,IAAIF,oBACFE,eAAe,IAAI,CACjB;SAGFA,eAAe,IAAI,CACjB;IAIJ,IAAID,yBAAyB;QAC3B,MAAME,wBACJ;QACFD,eAAe,IAAI,CACjBF,qBACI,GAAGG,sBAAsB,mLAAmL,CAAC,GAC7M,GAAGA,sBAAsB,kIAAkI,CAAC;IAEpK;IACA,MAAMC,gBAAgBF,eAAe,IAAI,CAAC;IAE1C,OAAO,CAAC;;;AAGV,EAAEE,cAAc;;;;;;;;kFAQkE,EAAEN,kBAAkB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAqEtG,CAAC;AACD;AAEO,MAAMO,yBAAyB,CACpCC,iBACAC;IAEA,IAAIC,gBAAgB;IAElBA,gBADE,AAAqB,YAArB,OAAOD,YACOA,YAEAE,KAAK,SAAS,CAACF,WAAW,MAAM;IAGlD,OAAO,CAAC;;AAEV,EAAED,gBAAgB;;;;AAIlB,EAAEE,cAAc;;EAEd,CAAC;AACH"}
+{"version":3,"file":"ai-model/prompt/extraction.mjs","sources":["../../../../src/ai-model/prompt/extraction.ts"],"sourcesContent":["import type { AIDataExtractionResponse, ServiceExtractParam } from '@/types';\nimport { getPreferredLanguage } from '@midscene/shared/env';\nimport { parseModelResponseJson } from '../service-caller/json';\nimport { extractXMLTag } from './util';\n\nexport function buildTypeQueryDemandValue(\n  type: 'Boolean' | 'Number' | 'String' | 'Assert' | 'WaitFor',\n  demand: ServiceExtractParam,\n) {\n  const currentScreenshotConstraint =\n    'based on the current screenshot and its contents if provided, unless the user explicitly asks to compare with reference images';\n\n  if (type === 'Assert') {\n    return `Boolean, ${currentScreenshotConstraint}, whether the following statement is true: ${demand}`;\n  }\n\n  if (type === 'WaitFor') {\n    return `Boolean, the user wants to do some 'wait for' operation. ${currentScreenshotConstraint}, please check whether the following statement is true: ${demand}`;\n  }\n\n  return `${type}, ${currentScreenshotConstraint}, ${demand}`;\n}\n\n/**\n * Parse XML response from LLM and convert to AIDataExtractionResponse\n */\nexport function parseXMLExtractionResponse<T>(\n  xmlString: string,\n): AIDataExtractionResponse<T> {\n  // Keep the internal field named `thought`, but ask models to emit\n  // <observation>. Gemini may only return <thought>-named content when\n  // thinking summaries are enabled.\n  const thought = extractXMLTag(xmlString, 'observation');\n  const dataJsonStr = extractXMLTag(xmlString, 'data-json');\n  const errorsStr = extractXMLTag(xmlString, 'errors');\n\n  // Parse data-json (required)\n  if (!dataJsonStr) {\n    throw new Error('Missing required field: data-json');\n  }\n\n  let data: T;\n  try {\n    data = parseModelResponseJson(dataJsonStr, {\n      source: 'generic-object',\n      requireObject: false,\n    }) as T;\n  } catch (e) {\n    throw new Error(`Failed to parse data-json: ${e}`);\n  }\n\n  // Parse errors (optional)\n  let errors: string[] | undefined;\n  if (errorsStr) {\n    try {\n      const parsedErrors = parseModelResponseJson(errorsStr, {\n        source: 'generic-object',\n        requireObject: false,\n      });\n      if (Array.isArray(parsedErrors)) {\n        errors = parsedErrors;\n      }\n    } catch (e) {\n      // If errors parsing fails, just ignore it\n    }\n  }\n\n  return {\n    ...(thought ? { thought } : {}),\n    data,\n    ...(errors && errors.length > 0 ? { errors } : {}),\n  };\n}\n\nexport function systemPromptToExtract(options?: {\n  screenshotIncluded?: boolean;\n  referenceImagesIncluded?: boolean;\n}) {\n  const preferredLanguage = getPreferredLanguage();\n  const screenshotIncluded = options?.screenshotIncluded ?? true;\n  const referenceImagesIncluded = options?.referenceImagesIncluded ?? false;\n\n  const contextPrompts = [\n    \"The user will give you data requirements in <DATA_DEMAND>. You need to understand the user's requirements and extract the data satisfying the <DATA_DEMAND>.\",\n  ];\n\n  if (screenshotIncluded) {\n    contextPrompts.push(\n      'The user will provide a current screenshot to evaluate, and may provide its contents. Base your answer on the current screenshot and its contents when provided. Treat them as the primary source of truth for what is currently visible or true.',\n    );\n  } else {\n    contextPrompts.push(\n      'The user will not provide a current screenshot. Use only the supplied page contents and other inputs, and do not infer unsupported visual details.',\n    );\n  }\n\n  if (referenceImagesIncluded) {\n    const referenceImagesPrompt =\n      'Reference images are supporting context only unless <DATA_DEMAND> explicitly asks for comparison, matching, or reasoning about them.';\n    contextPrompts.push(\n      screenshotIncluded\n        ? `${referenceImagesPrompt} Do not conclude that something exists in the current screenshot solely because it appears in a reference image; when they conflict, trust the current screenshot and its contents.`\n        : `${referenceImagesPrompt} Do not treat reference images as direct evidence of the current state unless the demand explicitly asks you to use them that way.`,\n    );\n  }\n  const contextPrompt = contextPrompts.join('\\n\\n');\n\n  return `\nYou are a versatile professional in software UI design and testing. Your outstanding contributions will impact the user experience of billions of users.\n\n${contextPrompt}\n\nIf a key specifies a JSON data type (such as Number, String, Boolean, Object, Array), ensure the returned value strictly matches that data type.\n\nWhen DATA_DEMAND is a JSON object, the keys in your response must exactly match the keys in DATA_DEMAND. Do not rename, translate, or substitute any key.\n\n\nReturn in the following XML format:\n<observation>brief evidence observed for the extraction, less than 300 words. Use ${preferredLanguage} in this field.</observation>\n<data-json>the extracted data as JSON. Make sure both the value and scheme meet the DATA_DEMAND. If you want to write some description in this field, use the same language as the DATA_DEMAND.</data-json>\n<errors>optional error messages as JSON array, e.g., [\"error1\", \"error2\"]</errors>\n\n# Example 1\nFor example, if the DATA_DEMAND is:\n\n<DATA_DEMAND>\n{\n  \"name\": \"name shows on the left panel, string\",\n  \"age\": \"age shows on the right panel, number\",\n  \"isAdmin\": \"if the user is admin, boolean\"\n}\n</DATA_DEMAND>\n\nBy viewing the screenshot and page contents, you can extract the following data:\n\n<observation>According to the screenshot, i can see ...</observation>\n<data-json>\n{\n  \"name\": \"John\",\n  \"age\": 30,\n  \"isAdmin\": true\n}\n</data-json>\n\n# Example 2\nIf the DATA_DEMAND is:\n\n<DATA_DEMAND>\nthe todo items list, string[]\n</DATA_DEMAND>\n\nBy viewing the screenshot and page contents, you can extract the following data:\n\n<observation>According to the screenshot, i can see ...</observation>\n<data-json>\n[\"todo 1\", \"todo 2\", \"todo 3\"]\n</data-json>\n\n# Example 3\nIf the DATA_DEMAND is:\n\n<DATA_DEMAND>\nthe page title, string\n</DATA_DEMAND>\n\nBy viewing the screenshot and page contents, you can extract the following data:\n\n<observation>According to the screenshot, i can see ...</observation>\n<data-json>\n\"todo list\"\n</data-json>\n\n# Example 4\nIf the DATA_DEMAND is:\n\n<DATA_DEMAND>\n{\n  \"StatementIsTruthy\": \"Boolean, is it currently the SMS page?\"\n}\n</DATA_DEMAND>\n\nBy viewing the screenshot and page contents, you can extract the following data:\n\n<observation>According to the screenshot, i can see ...</observation>\n<data-json>\n{ \"StatementIsTruthy\": true }\n</data-json>\n`;\n}\n\nexport const extractDataQueryPrompt = (\n  pageDescription: string,\n  dataQuery: string | Record<string, string>,\n) => {\n  let dataQueryText = '';\n  if (typeof dataQuery === 'string') {\n    dataQueryText = dataQuery;\n  } else {\n    dataQueryText = JSON.stringify(dataQuery, null, 2);\n  }\n\n  return `\n<PageDescription>\n${pageDescription}\n</PageDescription>\n\n<DATA_DEMAND>\n${dataQueryText}\n</DATA_DEMAND>\n  `;\n};\n"],"names":["buildTypeQueryDemandValue","type","demand","currentScreenshotConstraint","parseXMLExtractionResponse","xmlString","thought","extractXMLTag","dataJsonStr","errorsStr","Error","data","parseModelResponseJson","e","errors","parsedErrors","Array","systemPromptToExtract","options","preferredLanguage","getPreferredLanguage","screenshotIncluded","referenceImagesIncluded","contextPrompts","referenceImagesPrompt","contextPrompt","extractDataQueryPrompt","pageDescription","dataQuery","dataQueryText","JSON"],"mappings":";;;AAKO,SAASA,0BACdC,IAA4D,EAC5DC,MAA2B;IAE3B,MAAMC,8BACJ;IAEF,IAAIF,AAAS,aAATA,MACF,OAAO,CAAC,SAAS,EAAEE,4BAA4B,2CAA2C,EAAED,QAAQ;IAGtG,IAAID,AAAS,cAATA,MACF,OAAO,CAAC,yDAAyD,EAAEE,4BAA4B,wDAAwD,EAAED,QAAQ;IAGnK,OAAO,GAAGD,KAAK,EAAE,EAAEE,4BAA4B,EAAE,EAAED,QAAQ;AAC7D;AAKO,SAASE,2BACdC,SAAiB;IAKjB,MAAMC,UAAUC,cAAcF,WAAW;IACzC,MAAMG,cAAcD,cAAcF,WAAW;IAC7C,MAAMI,YAAYF,cAAcF,WAAW;IAG3C,IAAI,CAACG,aACH,MAAM,IAAIE,MAAM;IAGlB,IAAIC;IACJ,IAAI;QACFA,OAAOC,uBAAuBJ,aAAa;YACzC,QAAQ;YACR,eAAe;QACjB;IACF,EAAE,OAAOK,GAAG;QACV,MAAM,IAAIH,MAAM,CAAC,2BAA2B,EAAEG,GAAG;IACnD;IAGA,IAAIC;IACJ,IAAIL,WACF,IAAI;QACF,MAAMM,eAAeH,uBAAuBH,WAAW;YACrD,QAAQ;YACR,eAAe;QACjB;QACA,IAAIO,MAAM,OAAO,CAACD,eAChBD,SAASC;IAEb,EAAE,OAAOF,GAAG,CAEZ;IAGF,OAAO;QACL,GAAIP,UAAU;YAAEA;QAAQ,IAAI,CAAC,CAAC;QAC9BK;QACA,GAAIG,UAAUA,OAAO,MAAM,GAAG,IAAI;YAAEA;QAAO,IAAI,CAAC,CAAC;IACnD;AACF;AAEO,SAASG,sBAAsBC,OAGrC;IACC,MAAMC,oBAAoBC;IAC1B,MAAMC,qBAAqBH,SAAS,sBAAsB;IAC1D,MAAMI,0BAA0BJ,SAAS,2BAA2B;IAEpE,MAAMK,iBAAiB;QACrB;KACD;IAED,IAAIF,oBACFE,eAAe,IAAI,CACjB;SAGFA,eAAe,IAAI,CACjB;IAIJ,IAAID,yBAAyB;QAC3B,MAAME,wBACJ;QACFD,eAAe,IAAI,CACjBF,qBACI,GAAGG,sBAAsB,mLAAmL,CAAC,GAC7M,GAAGA,sBAAsB,kIAAkI,CAAC;IAEpK;IACA,MAAMC,gBAAgBF,eAAe,IAAI,CAAC;IAE1C,OAAO,CAAC;;;AAGV,EAAEE,cAAc;;;;;;;;kFAQkE,EAAEN,kBAAkB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAqEtG,CAAC;AACD;AAEO,MAAMO,yBAAyB,CACpCC,iBACAC;IAEA,IAAIC,gBAAgB;IAElBA,gBADE,AAAqB,YAArB,OAAOD,YACOA,YAEAE,KAAK,SAAS,CAACF,WAAW,MAAM;IAGlD,OAAO,CAAC;;AAEV,EAAED,gBAAgB;;;;AAIlB,EAAEE,cAAc;;EAEd,CAAC;AACH"}

package/dist/es/ai-model/prompt/recorder-metadata-generator.mjs

@@ -1,4 +1,5 @@
 import { getMidsceneRecorderEventDescription, getMidsceneRecorderScreenshotsForLLM, getMidsceneRecorderSemantic } from "@midscene/shared/recorder";
+import { getModelRuntime } from "../models/index.mjs";
 import { callAIWithObjectResponse } from "../service-caller/index.mjs";
 import { compactRecorderSemanticForGeneration } from "./recorder-generation-common.mjs";
 function summarizeRecorderEvents(input) {
@@ -71,7 +72,7 @@ Respond with a JSON object containing exactly "title" and "description".`
             role: 'user',
             content: messageContent
         }
-    ], modelConfig);
+    ], getModelRuntime(modelConfig));
     return {
         title: normalizeMetadataValue(response.content.title),
         description: normalizeMetadataValue(response.content.description)

package/dist/es/ai-model/prompt/recorder-metadata-generator.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/prompt/recorder-metadata-generator.mjs","sources":["../../../../src/ai-model/prompt/recorder-metadata-generator.ts"],"sourcesContent":["import type { IModelConfig } from '@midscene/shared/env';\nimport {\n  type MidsceneRecorderEvent,\n  type MidsceneRecorderTarget,\n  getMidsceneRecorderEventDescription,\n  getMidsceneRecorderScreenshotsForLLM,\n  getMidsceneRecorderSemantic,\n} from '@midscene/shared/recorder';\nimport { callAIWithObjectResponse } from '../service-caller/index';\nimport { compactRecorderSemanticForGeneration } from './recorder-generation-common';\n\nexport interface RecorderMetadataGenerationInput {\n  target: MidsceneRecorderTarget;\n  events: MidsceneRecorderEvent[];\n  fallbackName?: string;\n  maxScreenshots?: number;\n}\n\nexport interface RecorderGeneratedMetadata {\n  title?: string;\n  description?: string;\n}\n\nfunction summarizeRecorderEvents(input: RecorderMetadataGenerationInput) {\n  const events = input.events;\n  const navigationEvents = events.filter(\n    (event) => event.type === 'navigation',\n  );\n  const clickEvents = events.filter((event) => event.type === 'click');\n  const inputEvents = events.filter((event) => event.type === 'input');\n  const scrollEvents = events.filter((event) => event.type === 'scroll');\n  const urls = navigationEvents\n    .map((event) => event.url)\n    .filter((url): url is string => Boolean(url));\n  const titles = navigationEvents\n    .map((event) => event.title)\n    .filter((title): title is string => Boolean(title));\n\n  return {\n    platform: input.target.platformId,\n    target: input.target,\n    fallbackName: input.fallbackName,\n    pageCount: navigationEvents.length,\n    pageTitles: titles.slice(0, 5),\n    urls: urls.slice(0, 5),\n    clickCount: clickEvents.length,\n    inputCount: inputEvents.length,\n    scrollCount: scrollEvents.length,\n    totalActions: events.length,\n    firstUrl: urls[0] || input.target.values.url || '',\n    lastUrl: urls[urls.length - 1] || '',\n    events: events.slice(0, 20).map((event) => ({\n      type: event.type,\n      actionType: event.actionType,\n      url: event.url,\n      title: event.title,\n      value: event.value,\n      description: getMidsceneRecorderEventDescription(event),\n      semantic: compactRecorderSemanticForGeneration(\n        getMidsceneRecorderSemantic(event),\n      ),\n    })),\n  };\n}\n\nfunction normalizeMetadataValue(value: unknown) {\n  return typeof value === 'string' ? value.trim() : '';\n}\n\nexport async function generateRecorderSessionMetadata(\n  input: RecorderMetadataGenerationInput,\n  modelConfig: IModelConfig,\n): Promise<RecorderGeneratedMetadata> {\n  if (!input?.events?.length) {\n    throw new Error('generateRecorderSessionMetadata: events are required.');\n  }\n  if (!modelConfig?.modelName) {\n    throw new Error(\n      'generateRecorderSessionMetadata: modelConfig.modelName is required.',\n    );\n  }\n\n  const summary = summarizeRecorderEvents(input);\n  const screenshots = getMidsceneRecorderScreenshotsForLLM(\n    input.events,\n    input.maxScreenshots ?? 1,\n  );\n  const messageContent: any[] = [\n    {\n      type: 'text',\n      text: `Generate a concise title (5-7 words) and brief description (1-2 sentences) for a Studio recording of user actions.\n\nThe recording can target Web, Android, iOS, HarmonyOS, or Computer. Do not assume it is a browser session unless the platform is web.\nDescribe what the user did or accomplished. The description should use the user as the subject, preferably starting with \"The user ...\". Do not start the description with \"The session ...\".\nThe title should be action-oriented and highlight the main task accomplished.\n\nSummary:\n${JSON.stringify(summary, null, 2)}\n\nRespond with a JSON object containing exactly \"title\" and \"description\".`,\n    },\n  ];\n\n  for (const screenshot of screenshots) {\n    messageContent.push({\n      type: 'image_url',\n      image_url: { url: screenshot },\n    });\n  }\n\n  const response = await callAIWithObjectResponse<{\n    title?: string;\n    description?: string;\n  }>(\n    [\n      {\n        role: 'system',\n        content:\n          'You generate clear, task-oriented titles and descriptions for recorded automation sessions.',\n      },\n      {\n        role: 'user',\n        content: messageContent,\n      },\n    ],\n    modelConfig,\n  );\n\n  return {\n    title: normalizeMetadataValue(response.content.title),\n    description: normalizeMetadataValue(response.content.description),\n  };\n}\n"],"names":["summarizeRecorderEvents","input","events","navigationEvents","event","clickEvents","inputEvents","scrollEvents","urls","url","Boolean","titles","title","getMidsceneRecorderEventDescription","compactRecorderSemanticForGeneration","getMidsceneRecorderSemantic","normalizeMetadataValue","value","generateRecorderSessionMetadata","modelConfig","Error","summary","screenshots","getMidsceneRecorderScreenshotsForLLM","messageContent","JSON","screenshot","response","callAIWithObjectResponse"],"mappings":";;;AAuBA,SAASA,wBAAwBC,KAAsC;IACrE,MAAMC,SAASD,MAAM,MAAM;IAC3B,MAAME,mBAAmBD,OAAO,MAAM,CACpC,CAACE,QAAUA,AAAe,iBAAfA,MAAM,IAAI;IAEvB,MAAMC,cAAcH,OAAO,MAAM,CAAC,CAACE,QAAUA,AAAe,YAAfA,MAAM,IAAI;IACvD,MAAME,cAAcJ,OAAO,MAAM,CAAC,CAACE,QAAUA,AAAe,YAAfA,MAAM,IAAI;IACvD,MAAMG,eAAeL,OAAO,MAAM,CAAC,CAACE,QAAUA,AAAe,aAAfA,MAAM,IAAI;IACxD,MAAMI,OAAOL,iBACV,GAAG,CAAC,CAACC,QAAUA,MAAM,GAAG,EACxB,MAAM,CAAC,CAACK,MAAuBC,QAAQD;IAC1C,MAAME,SAASR,iBACZ,GAAG,CAAC,CAACC,QAAUA,MAAM,KAAK,EAC1B,MAAM,CAAC,CAACQ,QAA2BF,QAAQE;IAE9C,OAAO;QACL,UAAUX,MAAM,MAAM,CAAC,UAAU;QACjC,QAAQA,MAAM,MAAM;QACpB,cAAcA,MAAM,YAAY;QAChC,WAAWE,iBAAiB,MAAM;QAClC,YAAYQ,OAAO,KAAK,CAAC,GAAG;QAC5B,MAAMH,KAAK,KAAK,CAAC,GAAG;QACpB,YAAYH,YAAY,MAAM;QAC9B,YAAYC,YAAY,MAAM;QAC9B,aAAaC,aAAa,MAAM;QAChC,cAAcL,OAAO,MAAM;QAC3B,UAAUM,IAAI,CAAC,EAAE,IAAIP,MAAM,MAAM,CAAC,MAAM,CAAC,GAAG,IAAI;QAChD,SAASO,IAAI,CAACA,KAAK,MAAM,GAAG,EAAE,IAAI;QAClC,QAAQN,OAAO,KAAK,CAAC,GAAG,IAAI,GAAG,CAAC,CAACE,QAAW;gBAC1C,MAAMA,MAAM,IAAI;gBAChB,YAAYA,MAAM,UAAU;gBAC5B,KAAKA,MAAM,GAAG;gBACd,OAAOA,MAAM,KAAK;gBAClB,OAAOA,MAAM,KAAK;gBAClB,aAAaS,oCAAoCT;gBACjD,UAAUU,qCACRC,4BAA4BX;YAEhC;IACF;AACF;AAEA,SAASY,uBAAuBC,KAAc;IAC5C,OAAO,AAAiB,YAAjB,OAAOA,QAAqBA,MAAM,IAAI,KAAK;AACpD;AAEO,eAAeC,gCACpBjB,KAAsC,EACtCkB,WAAyB;IAEzB,IAAI,CAAClB,OAAO,QAAQ,QAClB,MAAM,IAAImB,MAAM;IAElB,IAAI,CAACD,aAAa,WAChB,MAAM,IAAIC,MACR;IAIJ,MAAMC,UAAUrB,wBAAwBC;IACxC,MAAMqB,cAAcC,qCAClBtB,MAAM,MAAM,EACZA,MAAM,cAAc,IAAI;IAE1B,MAAMuB,iBAAwB;QAC5B;YACE,MAAM;YACN,MAAM,CAAC;;;;;;;AAOb,EAAEC,KAAK,SAAS,CAACJ,SAAS,MAAM,GAAG;;wEAEqC,CAAC;QACrE;KACD;IAED,KAAK,MAAMK,cAAcJ,YACvBE,eAAe,IAAI,CAAC;QAClB,MAAM;QACN,WAAW;YAAE,KAAKE;QAAW;IAC/B;IAGF,MAAMC,WAAW,MAAMC,yBAIrB;QACE;YACE,MAAM;YACN,SACE;QACJ;QACA;YACE,MAAM;YACN,SAASJ;QACX;KACD,EACDL;IAGF,OAAO;QACL,OAAOH,uBAAuBW,SAAS,OAAO,CAAC,KAAK;QACpD,aAAaX,uBAAuBW,SAAS,OAAO,CAAC,WAAW;IAClE;AACF"}
+{"version":3,"file":"ai-model/prompt/recorder-metadata-generator.mjs","sources":["../../../../src/ai-model/prompt/recorder-metadata-generator.ts"],"sourcesContent":["import type { IModelConfig } from '@midscene/shared/env';\nimport {\n  type MidsceneRecorderEvent,\n  type MidsceneRecorderTarget,\n  getMidsceneRecorderEventDescription,\n  getMidsceneRecorderScreenshotsForLLM,\n  getMidsceneRecorderSemantic,\n} from '@midscene/shared/recorder';\nimport { getModelRuntime } from '../models';\nimport { callAIWithObjectResponse } from '../service-caller/index';\nimport { compactRecorderSemanticForGeneration } from './recorder-generation-common';\n\nexport interface RecorderMetadataGenerationInput {\n  target: MidsceneRecorderTarget;\n  events: MidsceneRecorderEvent[];\n  fallbackName?: string;\n  maxScreenshots?: number;\n}\n\nexport interface RecorderGeneratedMetadata {\n  title?: string;\n  description?: string;\n}\n\nfunction summarizeRecorderEvents(input: RecorderMetadataGenerationInput) {\n  const events = input.events;\n  const navigationEvents = events.filter(\n    (event) => event.type === 'navigation',\n  );\n  const clickEvents = events.filter((event) => event.type === 'click');\n  const inputEvents = events.filter((event) => event.type === 'input');\n  const scrollEvents = events.filter((event) => event.type === 'scroll');\n  const urls = navigationEvents\n    .map((event) => event.url)\n    .filter((url): url is string => Boolean(url));\n  const titles = navigationEvents\n    .map((event) => event.title)\n    .filter((title): title is string => Boolean(title));\n\n  return {\n    platform: input.target.platformId,\n    target: input.target,\n    fallbackName: input.fallbackName,\n    pageCount: navigationEvents.length,\n    pageTitles: titles.slice(0, 5),\n    urls: urls.slice(0, 5),\n    clickCount: clickEvents.length,\n    inputCount: inputEvents.length,\n    scrollCount: scrollEvents.length,\n    totalActions: events.length,\n    firstUrl: urls[0] || input.target.values.url || '',\n    lastUrl: urls[urls.length - 1] || '',\n    events: events.slice(0, 20).map((event) => ({\n      type: event.type,\n      actionType: event.actionType,\n      url: event.url,\n      title: event.title,\n      value: event.value,\n      description: getMidsceneRecorderEventDescription(event),\n      semantic: compactRecorderSemanticForGeneration(\n        getMidsceneRecorderSemantic(event),\n      ),\n    })),\n  };\n}\n\nfunction normalizeMetadataValue(value: unknown) {\n  return typeof value === 'string' ? value.trim() : '';\n}\n\nexport async function generateRecorderSessionMetadata(\n  input: RecorderMetadataGenerationInput,\n  modelConfig: IModelConfig,\n): Promise<RecorderGeneratedMetadata> {\n  if (!input?.events?.length) {\n    throw new Error('generateRecorderSessionMetadata: events are required.');\n  }\n  if (!modelConfig?.modelName) {\n    throw new Error(\n      'generateRecorderSessionMetadata: modelConfig.modelName is required.',\n    );\n  }\n\n  const summary = summarizeRecorderEvents(input);\n  const screenshots = getMidsceneRecorderScreenshotsForLLM(\n    input.events,\n    input.maxScreenshots ?? 1,\n  );\n  const messageContent: any[] = [\n    {\n      type: 'text',\n      text: `Generate a concise title (5-7 words) and brief description (1-2 sentences) for a Studio recording of user actions.\n\nThe recording can target Web, Android, iOS, HarmonyOS, or Computer. Do not assume it is a browser session unless the platform is web.\nDescribe what the user did or accomplished. The description should use the user as the subject, preferably starting with \"The user ...\". Do not start the description with \"The session ...\".\nThe title should be action-oriented and highlight the main task accomplished.\n\nSummary:\n${JSON.stringify(summary, null, 2)}\n\nRespond with a JSON object containing exactly \"title\" and \"description\".`,\n    },\n  ];\n\n  for (const screenshot of screenshots) {\n    messageContent.push({\n      type: 'image_url',\n      image_url: { url: screenshot },\n    });\n  }\n\n  const response = await callAIWithObjectResponse<{\n    title?: string;\n    description?: string;\n  }>(\n    [\n      {\n        role: 'system',\n        content:\n          'You generate clear, task-oriented titles and descriptions for recorded automation sessions.',\n      },\n      {\n        role: 'user',\n        content: messageContent,\n      },\n    ],\n    getModelRuntime(modelConfig),\n  );\n\n  return {\n    title: normalizeMetadataValue(response.content.title),\n    description: normalizeMetadataValue(response.content.description),\n  };\n}\n"],"names":["summarizeRecorderEvents","input","events","navigationEvents","event","clickEvents","inputEvents","scrollEvents","urls","url","Boolean","titles","title","getMidsceneRecorderEventDescription","compactRecorderSemanticForGeneration","getMidsceneRecorderSemantic","normalizeMetadataValue","value","generateRecorderSessionMetadata","modelConfig","Error","summary","screenshots","getMidsceneRecorderScreenshotsForLLM","messageContent","JSON","screenshot","response","callAIWithObjectResponse","getModelRuntime"],"mappings":";;;;AAwBA,SAASA,wBAAwBC,KAAsC;IACrE,MAAMC,SAASD,MAAM,MAAM;IAC3B,MAAME,mBAAmBD,OAAO,MAAM,CACpC,CAACE,QAAUA,AAAe,iBAAfA,MAAM,IAAI;IAEvB,MAAMC,cAAcH,OAAO,MAAM,CAAC,CAACE,QAAUA,AAAe,YAAfA,MAAM,IAAI;IACvD,MAAME,cAAcJ,OAAO,MAAM,CAAC,CAACE,QAAUA,AAAe,YAAfA,MAAM,IAAI;IACvD,MAAMG,eAAeL,OAAO,MAAM,CAAC,CAACE,QAAUA,AAAe,aAAfA,MAAM,IAAI;IACxD,MAAMI,OAAOL,iBACV,GAAG,CAAC,CAACC,QAAUA,MAAM,GAAG,EACxB,MAAM,CAAC,CAACK,MAAuBC,QAAQD;IAC1C,MAAME,SAASR,iBACZ,GAAG,CAAC,CAACC,QAAUA,MAAM,KAAK,EAC1B,MAAM,CAAC,CAACQ,QAA2BF,QAAQE;IAE9C,OAAO;QACL,UAAUX,MAAM,MAAM,CAAC,UAAU;QACjC,QAAQA,MAAM,MAAM;QACpB,cAAcA,MAAM,YAAY;QAChC,WAAWE,iBAAiB,MAAM;QAClC,YAAYQ,OAAO,KAAK,CAAC,GAAG;QAC5B,MAAMH,KAAK,KAAK,CAAC,GAAG;QACpB,YAAYH,YAAY,MAAM;QAC9B,YAAYC,YAAY,MAAM;QAC9B,aAAaC,aAAa,MAAM;QAChC,cAAcL,OAAO,MAAM;QAC3B,UAAUM,IAAI,CAAC,EAAE,IAAIP,MAAM,MAAM,CAAC,MAAM,CAAC,GAAG,IAAI;QAChD,SAASO,IAAI,CAACA,KAAK,MAAM,GAAG,EAAE,IAAI;QAClC,QAAQN,OAAO,KAAK,CAAC,GAAG,IAAI,GAAG,CAAC,CAACE,QAAW;gBAC1C,MAAMA,MAAM,IAAI;gBAChB,YAAYA,MAAM,UAAU;gBAC5B,KAAKA,MAAM,GAAG;gBACd,OAAOA,MAAM,KAAK;gBAClB,OAAOA,MAAM,KAAK;gBAClB,aAAaS,oCAAoCT;gBACjD,UAAUU,qCACRC,4BAA4BX;YAEhC;IACF;AACF;AAEA,SAASY,uBAAuBC,KAAc;IAC5C,OAAO,AAAiB,YAAjB,OAAOA,QAAqBA,MAAM,IAAI,KAAK;AACpD;AAEO,eAAeC,gCACpBjB,KAAsC,EACtCkB,WAAyB;IAEzB,IAAI,CAAClB,OAAO,QAAQ,QAClB,MAAM,IAAImB,MAAM;IAElB,IAAI,CAACD,aAAa,WAChB,MAAM,IAAIC,MACR;IAIJ,MAAMC,UAAUrB,wBAAwBC;IACxC,MAAMqB,cAAcC,qCAClBtB,MAAM,MAAM,EACZA,MAAM,cAAc,IAAI;IAE1B,MAAMuB,iBAAwB;QAC5B;YACE,MAAM;YACN,MAAM,CAAC;;;;;;;AAOb,EAAEC,KAAK,SAAS,CAACJ,SAAS,MAAM,GAAG;;wEAEqC,CAAC;QACrE;KACD;IAED,KAAK,MAAMK,cAAcJ,YACvBE,eAAe,IAAI,CAAC;QAClB,MAAM;QACN,WAAW;YAAE,KAAKE;QAAW;IAC/B;IAGF,MAAMC,WAAW,MAAMC,yBAIrB;QACE;YACE,MAAM;YACN,SACE;QACJ;QACA;YACE,MAAM;YACN,SAASJ;QACX;KACD,EACDK,gBAAgBV;IAGlB,OAAO;QACL,OAAOH,uBAAuBW,SAAS,OAAO,CAAC,KAAK;QACpD,aAAaX,uBAAuBW,SAAS,OAAO,CAAC,WAAW;IAClE;AACF"}