@midscene/core 1.13.4-beta-20260928072823.0 → 1.14.0

package/dist/es/ai-model/models/qwen.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/models/qwen.mjs","sources":["../../../../src/ai-model/models/qwen.ts"],"sourcesContent":["import type { TModelFamily } from '@midscene/shared/env';\nimport type {\n  ChatCompletionCallContext,\n  ChatCompletionParamsResult,\n  ModelAdapterDefinition,\n} from '../model-adapter/types';\nimport {\n  type LocateResultValue,\n  createLocateResultValue,\n  unwrapCoordinateListLikeInput,\n} from '../shared/model-locate-result';\n\nconst qwen25BboxCoordinatesMeta = {\n  shape: 'bbox',\n  order: 'xy',\n  rounding: 'round',\n} as const;\nconst qwen25PointCoordinatesMeta = {\n  shape: 'point',\n  order: 'xy',\n  rounding: 'round',\n} as const;\nconst qwen3BboxCoordinatesMeta = {\n  shape: 'bbox',\n  order: 'xy',\n  normalizedBy: 1000,\n  rounding: 'round',\n} as const;\n\nfunction parseQwen25RawLocateValue(input: unknown): LocateResultValue {\n  const bbox = unwrapCoordinateListLikeInput(input as any) as number[];\n  if (bbox.length < 2) {\n    const msg = `invalid bbox data for qwen-vl mode: ${JSON.stringify(bbox)} `;\n    throw new Error(msg);\n  }\n\n  if (typeof bbox[2] === 'number' && typeof bbox[3] === 'number') {\n    return createLocateResultValue(qwen25BboxCoordinatesMeta, [\n      bbox[0],\n      bbox[1],\n      bbox[2],\n      bbox[3],\n    ]);\n  }\n\n  return createLocateResultValue(qwen25PointCoordinatesMeta, [\n    bbox[0],\n    bbox[1],\n  ]);\n}\n\nconst buildQwenChatCompletionParams = (\n  input: ChatCompletionCallContext,\n): ChatCompletionParamsResult => {\n  const { midsceneDefaults, userConfig } = input;\n  const { reasoningEnabled, reasoningBudget } = userConfig;\n\n  const commonOverrideConfig: Record<string, unknown> = {};\n\n  if (userConfig.temperature !== undefined) {\n    commonOverrideConfig.temperature = userConfig.temperature;\n  }\n\n  // Alibaba Cloud Model Studio JSON mode:\n  // https://help.aliyun.com/zh/model-studio/json-mode\n  // Observed in qwen3.6 grounding runs: enabling this can make the model\n  // return only [\"bbox_2d\"] without coordinates.\n  // if (isLocateIntent(input.intent)) {\n  //   commonOverrideConfig.response_format = { type: 'json_object' };\n  // }\n\n  const modelSpecificConfig: Record<string, unknown> = {};\n\n  if (reasoningEnabled !== 'default') {\n    modelSpecificConfig.enable_thinking = reasoningEnabled ?? false;\n    if (reasoningBudget !== undefined) {\n      modelSpecificConfig.thinking_budget = reasoningBudget;\n    }\n  }\n\n  return {\n    config: {\n      ...midsceneDefaults,\n      ...commonOverrideConfig,\n      ...modelSpecificConfig,\n    },\n  };\n};\n\nconst buildQwen25ChatCompletionParams = (\n  input: ChatCompletionCallContext,\n): ChatCompletionParamsResult => {\n  const { midsceneDefaults, userConfig } = input;\n  const commonOverrideConfig: Record<string, unknown> = {};\n\n  if (userConfig.temperature !== undefined) {\n    commonOverrideConfig.temperature = userConfig.temperature;\n  }\n\n  return {\n    config: {\n      ...midsceneDefaults,\n      ...commonOverrideConfig,\n      vl_high_resolution_images: true,\n    },\n  };\n};\n\nconst qwen3Adapter: ModelAdapterDefinition = {\n  acceptBbox2dAlias: true,\n  chatCompletion: {\n    unsupportedUserConfig: ['reasoningEffort'],\n    buildChatCompletionParams: buildQwenChatCompletionParams,\n    messageExtraction: {\n      kind: 'default',\n      // DashScope returns Qwen thinking as `reasoning_content`. vLLM's\n      // OpenAI-compatible server is migrating the recommended field to\n      // `reasoning`, so Qwen adapters accept both serving conventions:\n      // https://github.com/vllm-project/vllm/issues/27755\n      reasoningContentKeys: ['reasoning_content', 'reasoning'],\n    },\n    useReasoningAsContentFallback: true,\n  },\n  locate: {\n    element: {\n      resultFormat: {\n        coordinates: qwen3BboxCoordinatesMeta,\n      },\n    },\n  },\n};\n\nexport const qwenAdapters = {\n  'qwen2.5-vl': {\n    acceptBbox2dAlias: true,\n    chatCompletion: {\n      unsupportedUserConfig: [\n        'reasoningEnabled',\n        'reasoningEffort',\n        'reasoningBudget',\n      ],\n      buildChatCompletionParams: buildQwen25ChatCompletionParams,\n    },\n    imagePreprocess: {\n      padBlockSize: 28,\n    },\n    locate: {\n      element: {\n        resultFormat: {\n          coordinates: qwen25BboxCoordinatesMeta,\n          parseRawLocateValue: parseQwen25RawLocateValue,\n        },\n      },\n    },\n  },\n  'qwen3-vl': qwen3Adapter,\n  qwen3: qwen3Adapter,\n  'qwen3.5': qwen3Adapter,\n  'qwen3.6': qwen3Adapter,\n} satisfies Pick<\n  Record<TModelFamily, ModelAdapterDefinition>,\n  'qwen2.5-vl' | 'qwen3-vl' | 'qwen3' | 'qwen3.5' | 'qwen3.6'\n>;\n"],"names":["qwen25BboxCoordinatesMeta","qwen25PointCoordinatesMeta","qwen3BboxCoordinatesMeta","parseQwen25RawLocateValue","input","bbox","unwrapCoordinateListLikeInput","msg","JSON","Error","createLocateResultValue","buildQwenChatCompletionParams","midsceneDefaults","userConfig","reasoningEnabled","reasoningBudget","commonOverrideConfig","undefined","modelSpecificConfig","buildQwen25ChatCompletionParams","qwen3Adapter","qwenAdapters"],"mappings":";AAYA,MAAMA,4BAA4B;IAChC,OAAO;IACP,OAAO;IACP,UAAU;AACZ;AACA,MAAMC,6BAA6B;IACjC,OAAO;IACP,OAAO;IACP,UAAU;AACZ;AACA,MAAMC,2BAA2B;IAC/B,OAAO;IACP,OAAO;IACP,cAAc;IACd,UAAU;AACZ;AAEA,SAASC,0BAA0BC,KAAc;IAC/C,MAAMC,OAAOC,8BAA8BF;IAC3C,IAAIC,KAAK,MAAM,GAAG,GAAG;QACnB,MAAME,MAAM,CAAC,oCAAoC,EAAEC,KAAK,SAAS,CAACH,MAAM,CAAC,CAAC;QAC1E,MAAM,IAAII,MAAMF;IAClB;IAEA,IAAI,AAAmB,YAAnB,OAAOF,IAAI,CAAC,EAAE,IAAiB,AAAmB,YAAnB,OAAOA,IAAI,CAAC,EAAE,EAC/C,OAAOK,wBAAwBV,2BAA2B;QACxDK,IAAI,CAAC,EAAE;QACPA,IAAI,CAAC,EAAE;QACPA,IAAI,CAAC,EAAE;QACPA,IAAI,CAAC,EAAE;KACR;IAGH,OAAOK,wBAAwBT,4BAA4B;QACzDI,IAAI,CAAC,EAAE;QACPA,IAAI,CAAC,EAAE;KACR;AACH;AAEA,MAAMM,gCAAgC,CACpCP;IAEA,MAAM,EAAEQ,gBAAgB,EAAEC,UAAU,EAAE,GAAGT;IACzC,MAAM,EAAEU,gBAAgB,EAAEC,eAAe,EAAE,GAAGF;IAE9C,MAAMG,uBAAgD,CAAC;IAEvD,IAAIH,AAA2BI,WAA3BJ,WAAW,WAAW,EACxBG,qBAAqB,WAAW,GAAGH,WAAW,WAAW;IAW3D,MAAMK,sBAA+C,CAAC;IAEtD,IAAIJ,AAAqB,cAArBA,kBAAgC;QAClCI,oBAAoB,eAAe,GAAGJ,oBAAoB;QAC1D,IAAIC,AAAoBE,WAApBF,iBACFG,oBAAoB,eAAe,GAAGH;IAE1C;IAEA,OAAO;QACL,QAAQ;YACN,GAAGH,gBAAgB;YACnB,GAAGI,oBAAoB;YACvB,GAAGE,mBAAmB;QACxB;IACF;AACF;AAEA,MAAMC,kCAAkC,CACtCf;IAEA,MAAM,EAAEQ,gBAAgB,EAAEC,UAAU,EAAE,GAAGT;IACzC,MAAMY,uBAAgD,CAAC;IAEvD,IAAIH,AAA2BI,WAA3BJ,WAAW,WAAW,EACxBG,qBAAqB,WAAW,GAAGH,WAAW,WAAW;IAG3D,OAAO;QACL,QAAQ;YACN,GAAGD,gBAAgB;YACnB,GAAGI,oBAAoB;YACvB,2BAA2B;QAC7B;IACF;AACF;AAEA,MAAMI,eAAuC;IAC3C,mBAAmB;IACnB,gBAAgB;QACd,uBAAuB;YAAC;SAAkB;QAC1C,2BAA2BT;QAC3B,mBAAmB;YACjB,MAAM;YAKN,sBAAsB;gBAAC;gBAAqB;aAAY;QAC1D;QACA,+BAA+B;IACjC;IACA,QAAQ;QACN,SAAS;YACP,cAAc;gBACZ,aAAaT;YACf;QACF;IACF;AACF;AAEO,MAAMmB,eAAe;IAC1B,cAAc;QACZ,mBAAmB;QACnB,gBAAgB;YACd,uBAAuB;gBACrB;gBACA;gBACA;aACD;YACD,2BAA2BF;QAC7B;QACA,iBAAiB;YACf,cAAc;QAChB;QACA,QAAQ;YACN,SAAS;gBACP,cAAc;oBACZ,aAAanB;oBACb,qBAAqBG;gBACvB;YACF;QACF;IACF;IACA,YAAYiB;IACZ,OAAOA;IACP,WAAWA;IACX,WAAWA;AACb"}
+{"version":3,"file":"ai-model/models/qwen.mjs","sources":["../../../../src/ai-model/models/qwen.ts"],"sourcesContent":["import type { TModelFamily } from '@midscene/shared/env';\nimport type {\n  ChatCompletionCallContext,\n  ChatCompletionParamsResult,\n  ModelAdapterDefinition,\n} from '../model-adapter/types';\nimport {\n  type LocateResultValue,\n  createLocateResultValue,\n  unwrapCoordinateListLikeInput,\n} from '../shared/model-locate-result';\n\nconst qwen25BboxCoordinatesMeta = {\n  shape: 'bbox',\n  order: 'xy',\n  rounding: 'round',\n} as const;\nconst qwen25PointCoordinatesMeta = {\n  shape: 'point',\n  order: 'xy',\n  rounding: 'round',\n} as const;\nconst qwen3BboxCoordinatesMeta = {\n  shape: 'bbox',\n  order: 'xy',\n  normalizedBy: 1000,\n  rounding: 'round',\n} as const;\n\nfunction parseQwen25RawLocateValue(input: unknown): LocateResultValue {\n  const bbox = unwrapCoordinateListLikeInput(input as any) as number[];\n  if (bbox.length < 2) {\n    const msg = `invalid bbox data for qwen-vl mode: ${JSON.stringify(bbox)} `;\n    throw new Error(msg);\n  }\n\n  if (typeof bbox[2] === 'number' && typeof bbox[3] === 'number') {\n    return createLocateResultValue(qwen25BboxCoordinatesMeta, [\n      bbox[0],\n      bbox[1],\n      bbox[2],\n      bbox[3],\n    ]);\n  }\n\n  return createLocateResultValue(qwen25PointCoordinatesMeta, [\n    bbox[0],\n    bbox[1],\n  ]);\n}\n\nconst buildQwenChatCompletionParams = (\n  input: ChatCompletionCallContext,\n): ChatCompletionParamsResult => {\n  const { midsceneDefaults, userConfig } = input;\n  const { reasoningEnabled, reasoningBudget } = userConfig;\n\n  const commonOverrideConfig: Record<string, unknown> = {};\n\n  if (userConfig.temperature !== undefined) {\n    commonOverrideConfig.temperature = userConfig.temperature;\n  }\n\n  // Alibaba Cloud Model Studio JSON mode:\n  // https://help.aliyun.com/zh/model-studio/json-mode\n  // Observed in qwen3.6 grounding runs: enabling this can make the model\n  // return only [\"bbox_2d\"] without coordinates.\n  // if (isLocateIntent(input.intent)) {\n  //   commonOverrideConfig.response_format = { type: 'json_object' };\n  // }\n\n  const modelSpecificConfig: Record<string, unknown> = {};\n\n  if (reasoningEnabled !== 'default') {\n    modelSpecificConfig.enable_thinking = reasoningEnabled ?? false;\n    if (reasoningBudget !== undefined) {\n      modelSpecificConfig.thinking_budget = reasoningBudget;\n    }\n  }\n\n  return {\n    config: {\n      ...midsceneDefaults,\n      ...commonOverrideConfig,\n      ...modelSpecificConfig,\n    },\n  };\n};\n\nconst buildQwen25ChatCompletionParams = (\n  input: ChatCompletionCallContext,\n): ChatCompletionParamsResult => {\n  const { midsceneDefaults, userConfig } = input;\n  const commonOverrideConfig: Record<string, unknown> = {};\n\n  if (userConfig.temperature !== undefined) {\n    commonOverrideConfig.temperature = userConfig.temperature;\n  }\n\n  return {\n    config: {\n      ...midsceneDefaults,\n      ...commonOverrideConfig,\n      vl_high_resolution_images: true,\n    },\n  };\n};\n\nconst qwen3Adapter: ModelAdapterDefinition = {\n  chatCompletion: {\n    unsupportedUserConfig: ['reasoningEffort'],\n    buildChatCompletionParams: buildQwenChatCompletionParams,\n    messageExtraction: {\n      kind: 'default',\n      // DashScope returns Qwen thinking as `reasoning_content`. vLLM's\n      // OpenAI-compatible server is migrating the recommended field to\n      // `reasoning`, so Qwen adapters accept both serving conventions:\n      // https://github.com/vllm-project/vllm/issues/27755\n      reasoningContentKeys: ['reasoning_content', 'reasoning'],\n    },\n    useReasoningAsContentFallback: true,\n  },\n  locate: {\n    element: {\n      resultFormat: {\n        resultKey: 'bbox_2d',\n        resultKeyAliases: ['bbox'],\n        coordinates: qwen3BboxCoordinatesMeta,\n      },\n    },\n  },\n};\n\nexport const qwenAdapters = {\n  'qwen2.5-vl': {\n    chatCompletion: {\n      unsupportedUserConfig: [\n        'reasoningEnabled',\n        'reasoningEffort',\n        'reasoningBudget',\n      ],\n      buildChatCompletionParams: buildQwen25ChatCompletionParams,\n    },\n    imagePreprocess: {\n      padBlockSize: 28,\n    },\n    locate: {\n      element: {\n        resultFormat: {\n          resultKey: 'bbox_2d',\n          resultKeyAliases: ['bbox'],\n          coordinates: qwen25BboxCoordinatesMeta,\n          parseRawLocateValue: parseQwen25RawLocateValue,\n        },\n      },\n    },\n  },\n  'qwen3-vl': qwen3Adapter,\n  qwen3: qwen3Adapter,\n  'qwen3.5': qwen3Adapter,\n  'qwen3.6': qwen3Adapter,\n} satisfies Pick<\n  Record<TModelFamily, ModelAdapterDefinition>,\n  'qwen2.5-vl' | 'qwen3-vl' | 'qwen3' | 'qwen3.5' | 'qwen3.6'\n>;\n"],"names":["qwen25BboxCoordinatesMeta","qwen25PointCoordinatesMeta","qwen3BboxCoordinatesMeta","parseQwen25RawLocateValue","input","bbox","unwrapCoordinateListLikeInput","msg","JSON","Error","createLocateResultValue","buildQwenChatCompletionParams","midsceneDefaults","userConfig","reasoningEnabled","reasoningBudget","commonOverrideConfig","undefined","modelSpecificConfig","buildQwen25ChatCompletionParams","qwen3Adapter","qwenAdapters"],"mappings":";AAYA,MAAMA,4BAA4B;IAChC,OAAO;IACP,OAAO;IACP,UAAU;AACZ;AACA,MAAMC,6BAA6B;IACjC,OAAO;IACP,OAAO;IACP,UAAU;AACZ;AACA,MAAMC,2BAA2B;IAC/B,OAAO;IACP,OAAO;IACP,cAAc;IACd,UAAU;AACZ;AAEA,SAASC,0BAA0BC,KAAc;IAC/C,MAAMC,OAAOC,8BAA8BF;IAC3C,IAAIC,KAAK,MAAM,GAAG,GAAG;QACnB,MAAME,MAAM,CAAC,oCAAoC,EAAEC,KAAK,SAAS,CAACH,MAAM,CAAC,CAAC;QAC1E,MAAM,IAAII,MAAMF;IAClB;IAEA,IAAI,AAAmB,YAAnB,OAAOF,IAAI,CAAC,EAAE,IAAiB,AAAmB,YAAnB,OAAOA,IAAI,CAAC,EAAE,EAC/C,OAAOK,wBAAwBV,2BAA2B;QACxDK,IAAI,CAAC,EAAE;QACPA,IAAI,CAAC,EAAE;QACPA,IAAI,CAAC,EAAE;QACPA,IAAI,CAAC,EAAE;KACR;IAGH,OAAOK,wBAAwBT,4BAA4B;QACzDI,IAAI,CAAC,EAAE;QACPA,IAAI,CAAC,EAAE;KACR;AACH;AAEA,MAAMM,gCAAgC,CACpCP;IAEA,MAAM,EAAEQ,gBAAgB,EAAEC,UAAU,EAAE,GAAGT;IACzC,MAAM,EAAEU,gBAAgB,EAAEC,eAAe,EAAE,GAAGF;IAE9C,MAAMG,uBAAgD,CAAC;IAEvD,IAAIH,AAA2BI,WAA3BJ,WAAW,WAAW,EACxBG,qBAAqB,WAAW,GAAGH,WAAW,WAAW;IAW3D,MAAMK,sBAA+C,CAAC;IAEtD,IAAIJ,AAAqB,cAArBA,kBAAgC;QAClCI,oBAAoB,eAAe,GAAGJ,oBAAoB;QAC1D,IAAIC,AAAoBE,WAApBF,iBACFG,oBAAoB,eAAe,GAAGH;IAE1C;IAEA,OAAO;QACL,QAAQ;YACN,GAAGH,gBAAgB;YACnB,GAAGI,oBAAoB;YACvB,GAAGE,mBAAmB;QACxB;IACF;AACF;AAEA,MAAMC,kCAAkC,CACtCf;IAEA,MAAM,EAAEQ,gBAAgB,EAAEC,UAAU,EAAE,GAAGT;IACzC,MAAMY,uBAAgD,CAAC;IAEvD,IAAIH,AAA2BI,WAA3BJ,WAAW,WAAW,EACxBG,qBAAqB,WAAW,GAAGH,WAAW,WAAW;IAG3D,OAAO;QACL,QAAQ;YACN,GAAGD,gBAAgB;YACnB,GAAGI,oBAAoB;YACvB,2BAA2B;QAC7B;IACF;AACF;AAEA,MAAMI,eAAuC;IAC3C,gBAAgB;QACd,uBAAuB;YAAC;SAAkB;QAC1C,2BAA2BT;QAC3B,mBAAmB;YACjB,MAAM;YAKN,sBAAsB;gBAAC;gBAAqB;aAAY;QAC1D;QACA,+BAA+B;IACjC;IACA,QAAQ;QACN,SAAS;YACP,cAAc;gBACZ,WAAW;gBACX,kBAAkB;oBAAC;iBAAO;gBAC1B,aAAaT;YACf;QACF;IACF;AACF;AAEO,MAAMmB,eAAe;IAC1B,cAAc;QACZ,gBAAgB;YACd,uBAAuB;gBACrB;gBACA;gBACA;aACD;YACD,2BAA2BF;QAC7B;QACA,iBAAiB;YACf,cAAc;QAChB;QACA,QAAQ;YACN,SAAS;gBACP,cAAc;oBACZ,WAAW;oBACX,kBAAkB;wBAAC;qBAAO;oBAC1B,aAAanB;oBACb,qBAAqBG;gBACvB;YACF;QACF;IACF;IACA,YAAYiB;IACZ,OAAOA;IACP,WAAWA;IACX,WAAWA;AACb"}

package/dist/es/ai-model/models/registry.mjs

@@ -26,8 +26,8 @@ const MODEL_ADAPTER_CONFIGS = {
 const modelAdapterCache = new Map();
 const debugModelAdapter = getDebug('ai:model-adapter');
 function debugAdapterUnsupportedUserConfig(modelFamily, adapter) {
-    if (0 === adapter.chatCompletion.unsupportedUserConfig.length) return;
-    debugModelAdapter(`model adapter "${modelFamily}" unsupportedUserConfig: ${JSON.stringify(adapter.chatCompletion.unsupportedUserConfig)}`);
+    if (adapter.chatCompletion.unsupportedUserConfig.length > 0) debugModelAdapter(`model adapter "${modelFamily}" (chat-completion) unsupportedUserConfig: ${JSON.stringify(adapter.chatCompletion.unsupportedUserConfig)}`);
+    if (adapter.responses.unsupportedUserConfig.length > 0) debugModelAdapter(`model adapter "${modelFamily}" (responses) unsupportedUserConfig: ${JSON.stringify(adapter.responses.unsupportedUserConfig)}`);
 }
 function getModelAdapter(modelFamily) {
     const cacheKey = modelFamily ?? 'default';

package/dist/es/ai-model/models/registry.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/models/registry.mjs","sources":["../../../../src/ai-model/models/registry.ts"],"sourcesContent":["import type { IModelConfig, TModelFamily } from '@midscene/shared/env';\nimport { getDebug } from '@midscene/shared/logger';\nimport { ResolvedModelAdapter } from '../model-adapter/resolve';\nimport type {\n  ModelAdapter,\n  ModelAdapterDefinition,\n  ModelRuntime,\n} from '../model-adapter/types';\nimport { autoGlmAdapters } from './auto-glm/adapter';\nimport { deepSeekAdapters } from './deepseek/adapter';\nimport { defaultOpenAICompatibleAdapterConfig } from './default';\nimport { doubaoAdapters } from './doubao';\nimport { geminiAdapters } from './gemini';\nimport { glmAdapters } from './glm';\nimport { gptAdapters } from './gpt';\nimport { kimiAdapters } from './kimi';\nimport { mimoAdapters } from './mimo';\nimport { qwenAdapters } from './qwen';\nimport { uiTarsAdapters } from './ui-tars/adapter';\n\nexport const MODEL_ADAPTER_CONFIGS = {\n  ...qwenAdapters,\n  ...deepSeekAdapters,\n  ...doubaoAdapters,\n  ...geminiAdapters,\n  ...uiTarsAdapters,\n  ...glmAdapters,\n  ...autoGlmAdapters,\n  ...gptAdapters,\n  ...kimiAdapters,\n  ...mimoAdapters,\n} satisfies Record<TModelFamily, ModelAdapterDefinition>;\n\ntype ModelAdapterCacheKey = TModelFamily | 'default';\n\nconst modelAdapterCache = new Map<ModelAdapterCacheKey, ModelAdapter>();\nconst debugModelAdapter = getDebug('ai:model-adapter');\n\nfunction debugAdapterUnsupportedUserConfig(\n  modelFamily: ModelAdapterCacheKey,\n  adapter: ModelAdapter,\n): void {\n  if (adapter.chatCompletion.unsupportedUserConfig.length === 0) {\n    return;\n  }\n\n  debugModelAdapter(\n    `model adapter \"${modelFamily}\" unsupportedUserConfig: ${JSON.stringify(\n      adapter.chatCompletion.unsupportedUserConfig,\n    )}`,\n  );\n}\n\nexport function getModelAdapter(modelFamily?: TModelFamily): ModelAdapter {\n  const cacheKey: ModelAdapterCacheKey = modelFamily ?? 'default';\n  let adapter = modelAdapterCache.get(cacheKey);\n  if (adapter) {\n    return adapter;\n  }\n\n  const config = modelFamily\n    ? MODEL_ADAPTER_CONFIGS[modelFamily]\n    : defaultOpenAICompatibleAdapterConfig;\n  if (!config) {\n    throw new Error(\n      `No model adapter registered for modelFamily: ${modelFamily}`,\n    );\n  }\n\n  adapter = new ResolvedModelAdapter(config, cacheKey);\n  modelAdapterCache.set(cacheKey, adapter);\n  debugAdapterUnsupportedUserConfig(cacheKey, adapter);\n\n  return adapter;\n}\n\nexport function getModelRuntime(config: IModelConfig): ModelRuntime {\n  return {\n    config,\n    adapter: getModelAdapter(config.modelFamily),\n  };\n}\n"],"names":["MODEL_ADAPTER_CONFIGS","qwenAdapters","deepSeekAdapters","doubaoAdapters","geminiAdapters","uiTarsAdapters","glmAdapters","autoGlmAdapters","gptAdapters","kimiAdapters","mimoAdapters","modelAdapterCache","Map","debugModelAdapter","getDebug","debugAdapterUnsupportedUserConfig","modelFamily","adapter","JSON","getModelAdapter","cacheKey","config","defaultOpenAICompatibleAdapterConfig","Error","ResolvedModelAdapter","getModelRuntime"],"mappings":";;;;;;;;;;;;;AAoBO,MAAMA,wBAAwB;IACnC,GAAGC,YAAY;IACf,GAAGC,gBAAgB;IACnB,GAAGC,cAAc;IACjB,GAAGC,cAAc;IACjB,GAAGC,cAAc;IACjB,GAAGC,WAAW;IACd,GAAGC,eAAe;IAClB,GAAGC,WAAW;IACd,GAAGC,YAAY;IACf,GAAGC,YAAY;AACjB;AAIA,MAAMC,oBAAoB,IAAIC;AAC9B,MAAMC,oBAAoBC,SAAS;AAEnC,SAASC,kCACPC,WAAiC,EACjCC,OAAqB;IAErB,IAAIA,AAAwD,MAAxDA,QAAQ,cAAc,CAAC,qBAAqB,CAAC,MAAM,EACrD;IAGFJ,kBACE,CAAC,eAAe,EAAEG,YAAY,yBAAyB,EAAEE,KAAK,SAAS,CACrED,QAAQ,cAAc,CAAC,qBAAqB,GAC3C;AAEP;AAEO,SAASE,gBAAgBH,WAA0B;IACxD,MAAMI,WAAiCJ,eAAe;IACtD,IAAIC,UAAUN,kBAAkB,GAAG,CAACS;IACpC,IAAIH,SACF,OAAOA;IAGT,MAAMI,SAASL,cACXhB,qBAAqB,CAACgB,YAAY,GAClCM;IACJ,IAAI,CAACD,QACH,MAAM,IAAIE,MACR,CAAC,6CAA6C,EAAEP,aAAa;IAIjEC,UAAU,IAAIO,qBAAqBH,QAAQD;IAC3CT,kBAAkB,GAAG,CAACS,UAAUH;IAChCF,kCAAkCK,UAAUH;IAE5C,OAAOA;AACT;AAEO,SAASQ,gBAAgBJ,MAAoB;IAClD,OAAO;QACLA;QACA,SAASF,gBAAgBE,OAAO,WAAW;IAC7C;AACF"}
+{"version":3,"file":"ai-model/models/registry.mjs","sources":["../../../../src/ai-model/models/registry.ts"],"sourcesContent":["import type { IModelConfig, TModelFamily } from '@midscene/shared/env';\nimport { getDebug } from '@midscene/shared/logger';\nimport { ResolvedModelAdapter } from '../model-adapter/resolve';\nimport type {\n  ModelAdapter,\n  ModelAdapterDefinition,\n  ModelRuntime,\n} from '../model-adapter/types';\nimport { autoGlmAdapters } from './auto-glm/adapter';\nimport { deepSeekAdapters } from './deepseek/adapter';\nimport { defaultOpenAICompatibleAdapterConfig } from './default';\nimport { doubaoAdapters } from './doubao';\nimport { geminiAdapters } from './gemini';\nimport { glmAdapters } from './glm';\nimport { gptAdapters } from './gpt';\nimport { kimiAdapters } from './kimi';\nimport { mimoAdapters } from './mimo';\nimport { qwenAdapters } from './qwen';\nimport { uiTarsAdapters } from './ui-tars/adapter';\n\nexport const MODEL_ADAPTER_CONFIGS = {\n  ...qwenAdapters,\n  ...deepSeekAdapters,\n  ...doubaoAdapters,\n  ...geminiAdapters,\n  ...uiTarsAdapters,\n  ...glmAdapters,\n  ...autoGlmAdapters,\n  ...gptAdapters,\n  ...kimiAdapters,\n  ...mimoAdapters,\n} satisfies Record<TModelFamily, ModelAdapterDefinition>;\n\ntype ModelAdapterCacheKey = TModelFamily | 'default';\n\nconst modelAdapterCache = new Map<ModelAdapterCacheKey, ModelAdapter>();\nconst debugModelAdapter = getDebug('ai:model-adapter');\n\nfunction debugAdapterUnsupportedUserConfig(\n  modelFamily: ModelAdapterCacheKey,\n  adapter: ModelAdapter,\n): void {\n  if (adapter.chatCompletion.unsupportedUserConfig.length > 0) {\n    debugModelAdapter(\n      `model adapter \"${modelFamily}\" (chat-completion) unsupportedUserConfig: ${JSON.stringify(adapter.chatCompletion.unsupportedUserConfig)}`,\n    );\n  }\n  if (adapter.responses.unsupportedUserConfig.length > 0) {\n    debugModelAdapter(\n      `model adapter \"${modelFamily}\" (responses) unsupportedUserConfig: ${JSON.stringify(adapter.responses.unsupportedUserConfig)}`,\n    );\n  }\n}\n\nexport function getModelAdapter(modelFamily?: TModelFamily): ModelAdapter {\n  const cacheKey: ModelAdapterCacheKey = modelFamily ?? 'default';\n  let adapter = modelAdapterCache.get(cacheKey);\n  if (adapter) {\n    return adapter;\n  }\n\n  const config = modelFamily\n    ? MODEL_ADAPTER_CONFIGS[modelFamily]\n    : defaultOpenAICompatibleAdapterConfig;\n  if (!config) {\n    throw new Error(\n      `No model adapter registered for modelFamily: ${modelFamily}`,\n    );\n  }\n\n  adapter = new ResolvedModelAdapter(config, cacheKey);\n  modelAdapterCache.set(cacheKey, adapter);\n  debugAdapterUnsupportedUserConfig(cacheKey, adapter);\n\n  return adapter;\n}\n\nexport function getModelRuntime(config: IModelConfig): ModelRuntime {\n  return {\n    config,\n    adapter: getModelAdapter(config.modelFamily),\n  };\n}\n"],"names":["MODEL_ADAPTER_CONFIGS","qwenAdapters","deepSeekAdapters","doubaoAdapters","geminiAdapters","uiTarsAdapters","glmAdapters","autoGlmAdapters","gptAdapters","kimiAdapters","mimoAdapters","modelAdapterCache","Map","debugModelAdapter","getDebug","debugAdapterUnsupportedUserConfig","modelFamily","adapter","JSON","getModelAdapter","cacheKey","config","defaultOpenAICompatibleAdapterConfig","Error","ResolvedModelAdapter","getModelRuntime"],"mappings":";;;;;;;;;;;;;AAoBO,MAAMA,wBAAwB;IACnC,GAAGC,YAAY;IACf,GAAGC,gBAAgB;IACnB,GAAGC,cAAc;IACjB,GAAGC,cAAc;IACjB,GAAGC,cAAc;IACjB,GAAGC,WAAW;IACd,GAAGC,eAAe;IAClB,GAAGC,WAAW;IACd,GAAGC,YAAY;IACf,GAAGC,YAAY;AACjB;AAIA,MAAMC,oBAAoB,IAAIC;AAC9B,MAAMC,oBAAoBC,SAAS;AAEnC,SAASC,kCACPC,WAAiC,EACjCC,OAAqB;IAErB,IAAIA,QAAQ,cAAc,CAAC,qBAAqB,CAAC,MAAM,GAAG,GACxDJ,kBACE,CAAC,eAAe,EAAEG,YAAY,2CAA2C,EAAEE,KAAK,SAAS,CAACD,QAAQ,cAAc,CAAC,qBAAqB,GAAG;IAG7I,IAAIA,QAAQ,SAAS,CAAC,qBAAqB,CAAC,MAAM,GAAG,GACnDJ,kBACE,CAAC,eAAe,EAAEG,YAAY,qCAAqC,EAAEE,KAAK,SAAS,CAACD,QAAQ,SAAS,CAAC,qBAAqB,GAAG;AAGpI;AAEO,SAASE,gBAAgBH,WAA0B;IACxD,MAAMI,WAAiCJ,eAAe;IACtD,IAAIC,UAAUN,kBAAkB,GAAG,CAACS;IACpC,IAAIH,SACF,OAAOA;IAGT,MAAMI,SAASL,cACXhB,qBAAqB,CAACgB,YAAY,GAClCM;IACJ,IAAI,CAACD,QACH,MAAM,IAAIE,MACR,CAAC,6CAA6C,EAAEP,aAAa;IAIjEC,UAAU,IAAIO,qBAAqBH,QAAQD;IAC3CT,kBAAkB,GAAG,CAACS,UAAUH;IAChCF,kCAAkCK,UAAUH;IAE5C,OAAOA;AACT;AAEO,SAASQ,gBAAgBJ,MAAoB;IAClD,OAAO;QACLA;QACA,SAASF,gBAAgBE,OAAO,WAAW;IAC7C;AACF"}

package/dist/es/ai-model/service-caller/call-ai.mjs

@@ -5,6 +5,7 @@ import { isCodexAppServerProvider } from "./codex/codex-app-server.mjs";
 import { isModelCallRecordingEnabled, recordModelCallEvent } from "./model-call-recorder.mjs";
 import { callOpenAI } from "./openai/call-openai.mjs";
 import { prepareChatCompletion } from "./openai/chat-completion/chat-completion.mjs";
+import { prepareResponses } from "./openai/responses/responses.mjs";
 import { buildRequestAbortSignal, resolveEffectiveTimeoutMs, runWithAbortSignal, waitForRetry } from "./request-timeout.mjs";
 import { appendAIRequestFailureSummary, buildUsageInfo, nextInternalCallId, normalizeRetryCount, toError } from "./utils.mjs";
 function _define_property(obj, key, value) {
@@ -17,6 +18,7 @@ function _define_property(obj, key, value) {
     else obj[key] = value;
     return obj;
 }
+const DEFAULT_MODEL_PROTOCOL = 'openai-chat';
 async function callAI(messages, modelRuntime, options) {
     options?.abortSignal?.throwIfAborted();
     const isStreaming = options?.stream === true;
@@ -24,21 +26,27 @@ async function callAI(messages, modelRuntime, options) {
     const executionId = modelRuntime.executionId ?? `unscoped-${uuid()}`;
     const internalCallId = nextInternalCallId();
     const { config: modelConfig } = modelRuntime;
-    const prepare = async ()=>isCodexAppServerProvider(modelConfig.openaiBaseURL) ? {
+    const prepare = async ()=>{
+        const input = {
+            messages,
+            modelRuntime,
+            options
+        };
+        if (isCodexAppServerProvider(modelConfig.openaiBaseURL)) return {
             protocol: 'codex',
-            input: prepareCodexCall({
-                messages,
-                modelRuntime,
-                options
-            })
-        } : {
+            input: prepareCodexCall(input)
+        };
+        const protocol = modelConfig.protocol ?? DEFAULT_MODEL_PROTOCOL;
+        assert(modelRuntime.adapter.supportedProtocols.includes(protocol), `Model adapter "${modelConfig.modelFamily ?? 'default'}" does not support protocol "${protocol}"`);
+        if ('openai-responses' === protocol) return {
+            protocol: 'responses',
+            input: await prepareResponses(input)
+        };
+        return {
             protocol: 'chat-completion',
-            input: await prepareChatCompletion({
-                messages,
-                modelRuntime,
-                options
-            })
+            input: await prepareChatCompletion(input)
         };
+    };
     const prepared = options?.abortSignal ? await runWithAbortSignal(options.abortSignal, prepare) : await prepare();
     const result = await callModelWithRetry({
         messages,
@@ -50,6 +58,7 @@ async function callAI(messages, modelRuntime, options) {
     });
     const { rawUsage, timeCost, totalTimeCost, retryCount, requestId, responseModelName, ...response } = result;
     const usage = buildUsageInfo({
+        apiType: prepared.protocol,
         usageData: rawUsage,
         timeCost,
         totalTimeCost,
@@ -145,7 +154,7 @@ async function callModelOnce({ messages, modelRuntime, options, executionId, int
         } : options
     };
     try {
-        return await runWithAbortSignal(requestSignal, ()=>'codex' === prepared.protocol ? callCodex(context, prepared.input) : callOpenAI(context, prepared.input));
+        return await runWithAbortSignal(requestSignal, ()=>'codex' === prepared.protocol ? callCodex(context, prepared.input) : callOpenAI(context, prepared));
     } catch (error) {
         options?.abortSignal?.throwIfAborted();
         const lastError = toError(error);

package/dist/es/ai-model/service-caller/call-ai.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/service-caller/call-ai.mjs","sources":["../../../../src/ai-model/service-caller/call-ai.ts"],"sourcesContent":["import { getDebug } from '@midscene/shared/logger';\nimport { assert, uuid } from '@midscene/shared/utils';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport type { ModelRuntime } from '../models';\nimport { callCodex, prepareCodexCall } from './codex/call-codex';\nimport { isCodexAppServerProvider } from './codex/codex-app-server';\nimport {\n  isModelCallRecordingEnabled,\n  recordModelCallEvent,\n} from './model-call-recorder';\nimport { callOpenAI } from './openai/call-openai';\nimport { prepareChatCompletion } from './openai/chat-completion/chat-completion';\nimport {\n  buildRequestAbortSignal,\n  resolveEffectiveTimeoutMs,\n  runWithAbortSignal,\n  waitForRetry,\n} from './request-timeout';\nimport type {\n  AICallResult,\n  CallAIOptions,\n  ModelCallContext,\n  ModelCallResult,\n} from './types';\nimport {\n  appendAIRequestFailureSummary,\n  buildUsageInfo,\n  nextInternalCallId,\n  normalizeRetryCount,\n  toError,\n} from './utils';\n\nexport async function callAI(\n  messages: ChatCompletionMessageParam[],\n  modelRuntime: ModelRuntime,\n  options?: CallAIOptions,\n): Promise<AICallResult> {\n  options?.abortSignal?.throwIfAborted();\n  const isStreaming = options?.stream === true;\n  if (isStreaming) {\n    assert(\n      typeof options?.onChunk === 'function',\n      'onChunk is required when stream is true',\n    );\n  }\n\n  // Low-level callers without a TaskRunner still need a stable ID for the\n  // lifetime of this model call (including its network retries).\n  const executionId = modelRuntime.executionId ?? `unscoped-${uuid()}`;\n\n  // Stable internal ID for this call, used by the agent to deduplicate usage\n  // across the onUsage callback and the task-dump-based collectUsageMetrics()\n  // path when the provider does not return a request_id.\n  const internalCallId = nextInternalCallId();\n\n  const { config: modelConfig } = modelRuntime;\n\n  const prepare = async (): Promise<PreparedModelInput> =>\n    isCodexAppServerProvider(modelConfig.openaiBaseURL)\n      ? {\n          protocol: 'codex',\n          input: prepareCodexCall({ messages, modelRuntime, options }),\n        }\n      : {\n          protocol: 'chat-completion',\n          input: await prepareChatCompletion({\n            messages,\n            modelRuntime,\n            options,\n          }),\n        };\n\n  const prepared = options?.abortSignal\n    ? await runWithAbortSignal(options.abortSignal, prepare)\n    : await prepare();\n\n  const result = await callModelWithRetry({\n    messages,\n    modelRuntime,\n    options,\n    executionId,\n    internalCallId,\n    prepared,\n  });\n\n  const {\n    rawUsage,\n    timeCost,\n    totalTimeCost,\n    retryCount,\n    requestId,\n    responseModelName,\n    ...response\n  } = result;\n\n  const usage = buildUsageInfo({\n    usageData: rawUsage,\n    timeCost,\n    totalTimeCost,\n    retryCount,\n    requestId,\n    responseModelName,\n    modelName: modelConfig.modelName,\n    modelDescription: modelConfig.modelDescription,\n    slot: modelConfig.slot,\n    internalCallId,\n  });\n\n  const debugProfileStats = getDebug('ai:profile:stats');\n  const debugProfileDetail = getDebug('ai:profile:detail');\n  debugProfileStats(\n    `model, ${modelConfig.modelName}, mode, ${modelConfig.modelFamily || 'default'}, streaming, ${response.isStreamed}, prompt-tokens, ${usage?.prompt_tokens ?? ''}, completion-tokens, ${usage?.completion_tokens ?? ''}, total-tokens, ${usage?.total_tokens ?? ''}, cached-input, ${usage?.cached_input ?? ''}, cost-ms, ${timeCost ?? ''}, requestId, ${requestId ?? ''}, slot, ${modelConfig.slot}, configured-temperature, ${modelConfig.temperature ?? ''}`,\n  );\n  debugProfileDetail(`model usage detail: ${JSON.stringify(usage)}`);\n\n  if (usage && modelRuntime.onUsage) {\n    modelRuntime.onUsage(usage);\n  }\n  return { ...response, usage };\n}\n\ntype PreparedModelInput =\n  | { protocol: 'codex'; input: ReturnType<typeof prepareCodexCall> }\n  | {\n      protocol: 'chat-completion';\n      input: Awaited<ReturnType<typeof prepareChatCompletion>>;\n    };\n\ntype ModelCallInput = {\n  messages: ChatCompletionMessageParam[];\n  modelRuntime: ModelRuntime;\n  options?: CallAIOptions;\n  executionId: string;\n  internalCallId: string;\n  prepared: PreparedModelInput;\n};\n\n/** Carries the original failure across the retry boundary without replaying output. */\nclass NonRetryableModelCallError extends Error {\n  readonly cause: Error;\n\n  constructor(cause: Error) {\n    super(cause.message, { cause });\n    this.cause = cause;\n  }\n}\n\nasync function callModelWithRetry(\n  input: ModelCallInput,\n): Promise<ModelCallResult & { totalTimeCost: number; retryCount: number }> {\n  const startTime = Date.now();\n  const {\n    modelRuntime: { config },\n    options,\n  } = input;\n  const maxAttempts = normalizeRetryCount(config.retryCount) + 1;\n  const retryInterval = config.retryInterval ?? 2000;\n  const attemptErrors: Array<{ attempt: number; error: unknown }> = [];\n\n  for (let attempt = 1; ; attempt++) {\n    options?.abortSignal?.throwIfAborted();\n    try {\n      const result = await callModelOnce(input, attempt);\n      return {\n        ...result,\n        totalTimeCost: Date.now() - startTime,\n        retryCount: attempt - 1,\n      };\n    } catch (error) {\n      options?.abortSignal?.throwIfAborted();\n      if (error instanceof NonRetryableModelCallError) {\n        throw error.cause;\n      }\n      const lastError = toError(error);\n      attemptErrors.push({ attempt, error: lastError });\n      if (attempt === maxAttempts) {\n        throw appendAIRequestFailureSummary(\n          lastError,\n          attemptErrors,\n          maxAttempts,\n        );\n      }\n      getDebug('ai:call', { console: true })(\n        `AI call failed (attempt ${attempt}/${maxAttempts}), retrying in ${retryInterval}ms... Error: ${lastError.message}`,\n      );\n    }\n    await waitForRetry(retryInterval, options?.abortSignal);\n  }\n}\n\nasync function callModelOnce(\n  {\n    messages,\n    modelRuntime,\n    options,\n    executionId,\n    internalCallId,\n    prepared,\n  }: ModelCallInput,\n  attempt: number,\n): Promise<ModelCallResult> {\n  const { config: modelConfig } = modelRuntime;\n  const recordEvent = isModelCallRecordingEnabled()\n    ? (event: Record<string, unknown>) => {\n        void recordModelCallEvent({\n          executionId,\n          callId: internalCallId,\n          semanticRetryAttempt: options?.semanticRetryAttempt,\n          slot: modelConfig.slot,\n          intent: modelConfig.intent,\n          modelFamily: modelConfig.modelFamily,\n          ...event,\n          attempt,\n        });\n      }\n    : undefined;\n  const effectiveTimeoutMs = resolveEffectiveTimeoutMs(modelConfig.timeout);\n  const { signal: requestSignal, cleanup } = buildRequestAbortSignal(\n    effectiveTimeoutMs,\n    options?.abortSignal,\n  );\n  let deliveredChunk = false;\n  const onChunk = options?.onChunk;\n  const context: ModelCallContext = {\n    messages,\n    modelRuntime,\n    executionId,\n    recordEvent,\n    requestSignal,\n    effectiveTimeoutMs,\n    options: onChunk\n      ? {\n          ...options,\n          onChunk: (chunk) => {\n            requestSignal.throwIfAborted();\n            deliveredChunk = true;\n            onChunk(chunk);\n          },\n        }\n      : options,\n  };\n  try {\n    return await runWithAbortSignal(requestSignal, () =>\n      prepared.protocol === 'codex'\n        ? callCodex(context, prepared.input)\n        : callOpenAI(context, prepared.input),\n    );\n  } catch (error) {\n    options?.abortSignal?.throwIfAborted();\n    const lastError = toError(error);\n    if (deliveredChunk) {\n      throw new NonRetryableModelCallError(lastError);\n    }\n    throw lastError;\n  } finally {\n    cleanup();\n  }\n}\n"],"names":["callAI","messages","modelRuntime","options","isStreaming","assert","executionId","uuid","internalCallId","nextInternalCallId","modelConfig","prepare","isCodexAppServerProvider","prepareCodexCall","prepareChatCompletion","prepared","runWithAbortSignal","result","callModelWithRetry","rawUsage","timeCost","totalTimeCost","retryCount","requestId","responseModelName","response","usage","buildUsageInfo","debugProfileStats","getDebug","debugProfileDetail","JSON","NonRetryableModelCallError","Error","cause","input","startTime","Date","config","maxAttempts","normalizeRetryCount","retryInterval","attemptErrors","attempt","callModelOnce","error","lastError","toError","appendAIRequestFailureSummary","waitForRetry","recordEvent","isModelCallRecordingEnabled","event","recordModelCallEvent","undefined","effectiveTimeoutMs","resolveEffectiveTimeoutMs","requestSignal","cleanup","buildRequestAbortSignal","deliveredChunk","onChunk","context","chunk","callCodex","callOpenAI"],"mappings":";;;;;;;;;;;;;;;;;;;AAgCO,eAAeA,OACpBC,QAAsC,EACtCC,YAA0B,EAC1BC,OAAuB;IAEvBA,SAAS,aAAa;IACtB,MAAMC,cAAcD,SAAS,WAAW;IACxC,IAAIC,aACFC,OACE,AAA4B,cAA5B,OAAOF,SAAS,SAChB;IAMJ,MAAMG,cAAcJ,aAAa,WAAW,IAAI,CAAC,SAAS,EAAEK,QAAQ;IAKpE,MAAMC,iBAAiBC;IAEvB,MAAM,EAAE,QAAQC,WAAW,EAAE,GAAGR;IAEhC,MAAMS,UAAU,UACdC,yBAAyBF,YAAY,aAAa,IAC9C;YACE,UAAU;YACV,OAAOG,iBAAiB;gBAAEZ;gBAAUC;gBAAcC;YAAQ;QAC5D,IACA;YACE,UAAU;YACV,OAAO,MAAMW,sBAAsB;gBACjCb;gBACAC;gBACAC;YACF;QACF;IAEN,MAAMY,WAAWZ,SAAS,cACtB,MAAMa,mBAAmBb,QAAQ,WAAW,EAAEQ,WAC9C,MAAMA;IAEV,MAAMM,SAAS,MAAMC,mBAAmB;QACtCjB;QACAC;QACAC;QACAG;QACAE;QACAO;IACF;IAEA,MAAM,EACJI,QAAQ,EACRC,QAAQ,EACRC,aAAa,EACbC,UAAU,EACVC,SAAS,EACTC,iBAAiB,EACjB,GAAGC,UACJ,GAAGR;IAEJ,MAAMS,QAAQC,eAAe;QAC3B,WAAWR;QACXC;QACAC;QACAC;QACAC;QACAC;QACA,WAAWd,YAAY,SAAS;QAChC,kBAAkBA,YAAY,gBAAgB;QAC9C,MAAMA,YAAY,IAAI;QACtBF;IACF;IAEA,MAAMoB,oBAAoBC,SAAS;IACnC,MAAMC,qBAAqBD,SAAS;IACpCD,kBACE,CAAC,OAAO,EAAElB,YAAY,SAAS,CAAC,QAAQ,EAAEA,YAAY,WAAW,IAAI,UAAU,aAAa,EAAEe,SAAS,UAAU,CAAC,iBAAiB,EAAEC,OAAO,iBAAiB,GAAG,qBAAqB,EAAEA,OAAO,qBAAqB,GAAG,gBAAgB,EAAEA,OAAO,gBAAgB,GAAG,gBAAgB,EAAEA,OAAO,gBAAgB,GAAG,WAAW,EAAEN,YAAY,GAAG,aAAa,EAAEG,aAAa,GAAG,QAAQ,EAAEb,YAAY,IAAI,CAAC,0BAA0B,EAAEA,YAAY,WAAW,IAAI,IAAI;IAEjcoB,mBAAmB,CAAC,oBAAoB,EAAEC,KAAK,SAAS,CAACL,QAAQ;IAEjE,IAAIA,SAASxB,aAAa,OAAO,EAC/BA,aAAa,OAAO,CAACwB;IAEvB,OAAO;QAAE,GAAGD,QAAQ;QAAEC;IAAM;AAC9B;AAmBA,MAAMM,mCAAmCC;IAGvC,YAAYC,KAAY,CAAE;QACxB,KAAK,CAACA,MAAM,OAAO,EAAE;YAAEA;QAAM,IAH/B,uBAAS,SAAT;QAIE,IAAI,CAAC,KAAK,GAAGA;IACf;AACF;AAEA,eAAehB,mBACbiB,KAAqB;IAErB,MAAMC,YAAYC,KAAK,GAAG;IAC1B,MAAM,EACJ,cAAc,EAAEC,MAAM,EAAE,EACxBnC,OAAO,EACR,GAAGgC;IACJ,MAAMI,cAAcC,oBAAoBF,OAAO,UAAU,IAAI;IAC7D,MAAMG,gBAAgBH,OAAO,aAAa,IAAI;IAC9C,MAAMI,gBAA4D,EAAE;IAEpE,IAAK,IAAIC,UAAU,IAAKA,UAAW;QACjCxC,SAAS,aAAa;QACtB,IAAI;YACF,MAAMc,SAAS,MAAM2B,cAAcT,OAAOQ;YAC1C,OAAO;gBACL,GAAG1B,MAAM;gBACT,eAAeoB,KAAK,GAAG,KAAKD;gBAC5B,YAAYO,UAAU;YACxB;QACF,EAAE,OAAOE,OAAO;YACd1C,SAAS,aAAa;YACtB,IAAI0C,iBAAiBb,4BACnB,MAAMa,MAAM,KAAK;YAEnB,MAAMC,YAAYC,QAAQF;YAC1BH,cAAc,IAAI,CAAC;gBAAEC;gBAAS,OAAOG;YAAU;YAC/C,IAAIH,YAAYJ,aACd,MAAMS,8BACJF,WACAJ,eACAH;YAGJV,SAAS,WAAW;gBAAE,SAAS;YAAK,GAClC,CAAC,wBAAwB,EAAEc,QAAQ,CAAC,EAAEJ,YAAY,eAAe,EAAEE,cAAc,aAAa,EAAEK,UAAU,OAAO,EAAE;QAEvH;QACA,MAAMG,aAAaR,eAAetC,SAAS;IAC7C;AACF;AAEA,eAAeyC,cACb,EACE3C,QAAQ,EACRC,YAAY,EACZC,OAAO,EACPG,WAAW,EACXE,cAAc,EACdO,QAAQ,EACO,EACjB4B,OAAe;IAEf,MAAM,EAAE,QAAQjC,WAAW,EAAE,GAAGR;IAChC,MAAMgD,cAAcC,gCAChB,CAACC;QACMC,qBAAqB;YACxB/C;YACA,QAAQE;YACR,sBAAsBL,SAAS;YAC/B,MAAMO,YAAY,IAAI;YACtB,QAAQA,YAAY,MAAM;YAC1B,aAAaA,YAAY,WAAW;YACpC,GAAG0C,KAAK;YACRT;QACF;IACF,IACAW;IACJ,MAAMC,qBAAqBC,0BAA0B9C,YAAY,OAAO;IACxE,MAAM,EAAE,QAAQ+C,aAAa,EAAEC,OAAO,EAAE,GAAGC,wBACzCJ,oBACApD,SAAS;IAEX,IAAIyD,iBAAiB;IACrB,MAAMC,UAAU1D,SAAS;IACzB,MAAM2D,UAA4B;QAChC7D;QACAC;QACAI;QACA4C;QACAO;QACAF;QACA,SAASM,UACL;YACE,GAAG1D,OAAO;YACV,SAAS,CAAC4D;gBACRN,cAAc,cAAc;gBAC5BG,iBAAiB;gBACjBC,QAAQE;YACV;QACF,IACA5D;IACN;IACA,IAAI;QACF,OAAO,MAAMa,mBAAmByC,eAAe,IAC7C1C,AAAsB,YAAtBA,SAAS,QAAQ,GACbiD,UAAUF,SAAS/C,SAAS,KAAK,IACjCkD,WAAWH,SAAS/C,SAAS,KAAK;IAE1C,EAAE,OAAO8B,OAAO;QACd1C,SAAS,aAAa;QACtB,MAAM2C,YAAYC,QAAQF;QAC1B,IAAIe,gBACF,MAAM,IAAI5B,2BAA2Bc;QAEvC,MAAMA;IACR,SAAU;QACRY;IACF;AACF"}
+{"version":3,"file":"ai-model/service-caller/call-ai.mjs","sources":["../../../../src/ai-model/service-caller/call-ai.ts"],"sourcesContent":["import { getDebug } from '@midscene/shared/logger';\nimport { assert, uuid } from '@midscene/shared/utils';\nimport type { ModelRuntime } from '../models';\nimport { callCodex, prepareCodexCall } from './codex/call-codex';\nimport { isCodexAppServerProvider } from './codex/codex-app-server';\nimport {\n  isModelCallRecordingEnabled,\n  recordModelCallEvent,\n} from './model-call-recorder';\nimport { type PreparedOpenAIInput, callOpenAI } from './openai/call-openai';\nimport { prepareChatCompletion } from './openai/chat-completion/chat-completion';\nimport { prepareResponses } from './openai/responses/responses';\nimport {\n  buildRequestAbortSignal,\n  resolveEffectiveTimeoutMs,\n  runWithAbortSignal,\n  waitForRetry,\n} from './request-timeout';\nimport type { ModelCallMessages } from './types';\nimport type {\n  AICallResult,\n  CallAIOptions,\n  ModelCallContext,\n  ModelCallResult,\n} from './types';\nimport {\n  appendAIRequestFailureSummary,\n  buildUsageInfo,\n  nextInternalCallId,\n  normalizeRetryCount,\n  toError,\n} from './utils';\n\nconst DEFAULT_MODEL_PROTOCOL = 'openai-chat';\n\nexport async function callAI(\n  messages: ModelCallMessages,\n  modelRuntime: ModelRuntime,\n  options?: CallAIOptions,\n): Promise<AICallResult> {\n  options?.abortSignal?.throwIfAborted();\n  const isStreaming = options?.stream === true;\n  if (isStreaming) {\n    assert(\n      typeof options?.onChunk === 'function',\n      'onChunk is required when stream is true',\n    );\n  }\n\n  // Low-level callers without a TaskRunner still need a stable ID for the\n  // lifetime of this model call (including its network retries).\n  const executionId = modelRuntime.executionId ?? `unscoped-${uuid()}`;\n\n  // Stable internal ID for this call, used by the agent to deduplicate usage\n  // across the onUsage callback and the task-dump-based collectUsageMetrics()\n  // path when the provider does not return a request_id.\n  const internalCallId = nextInternalCallId();\n\n  const { config: modelConfig } = modelRuntime;\n\n  const prepare = async (): Promise<PreparedModelInput> => {\n    const input = { messages, modelRuntime, options };\n    if (isCodexAppServerProvider(modelConfig.openaiBaseURL)) {\n      return { protocol: 'codex', input: prepareCodexCall(input) };\n    }\n    const protocol = modelConfig.protocol ?? DEFAULT_MODEL_PROTOCOL;\n\n    assert(\n      modelRuntime.adapter.supportedProtocols.includes(protocol),\n      `Model adapter \"${modelConfig.modelFamily ?? 'default'}\" does not support protocol \"${protocol}\"`,\n    );\n    if (protocol === 'openai-responses') {\n      return { protocol: 'responses', input: await prepareResponses(input) };\n    }\n    return {\n      protocol: 'chat-completion',\n      input: await prepareChatCompletion(input),\n    };\n  };\n\n  const prepared = options?.abortSignal\n    ? await runWithAbortSignal(options.abortSignal, prepare)\n    : await prepare();\n\n  const result = await callModelWithRetry({\n    messages,\n    modelRuntime,\n    options,\n    executionId,\n    internalCallId,\n    prepared,\n  });\n\n  const {\n    rawUsage,\n    timeCost,\n    totalTimeCost,\n    retryCount,\n    requestId,\n    responseModelName,\n    ...response\n  } = result;\n\n  const usage = buildUsageInfo({\n    apiType: prepared.protocol,\n    usageData: rawUsage,\n    timeCost,\n    totalTimeCost,\n    retryCount,\n    requestId,\n    responseModelName,\n    modelName: modelConfig.modelName,\n    modelDescription: modelConfig.modelDescription,\n    slot: modelConfig.slot,\n    internalCallId,\n  });\n\n  const debugProfileStats = getDebug('ai:profile:stats');\n  const debugProfileDetail = getDebug('ai:profile:detail');\n  debugProfileStats(\n    `model, ${modelConfig.modelName}, mode, ${modelConfig.modelFamily || 'default'}, streaming, ${response.isStreamed}, prompt-tokens, ${usage?.prompt_tokens ?? ''}, completion-tokens, ${usage?.completion_tokens ?? ''}, total-tokens, ${usage?.total_tokens ?? ''}, cached-input, ${usage?.cached_input ?? ''}, cost-ms, ${timeCost ?? ''}, requestId, ${requestId ?? ''}, slot, ${modelConfig.slot}, configured-temperature, ${modelConfig.temperature ?? ''}`,\n  );\n  debugProfileDetail(`model usage detail: ${JSON.stringify(usage)}`);\n\n  if (usage && modelRuntime.onUsage) {\n    modelRuntime.onUsage(usage);\n  }\n  return { ...response, usage };\n}\n\ntype PreparedModelInput =\n  | { protocol: 'codex'; input: ReturnType<typeof prepareCodexCall> }\n  | PreparedOpenAIInput;\n\ntype ModelCallInput = {\n  messages: ModelCallMessages;\n  modelRuntime: ModelRuntime;\n  options?: CallAIOptions;\n  executionId: string;\n  internalCallId: string;\n  prepared: PreparedModelInput;\n};\n\n/** Carries the original failure across the retry boundary without replaying output. */\nclass NonRetryableModelCallError extends Error {\n  readonly cause: Error;\n\n  constructor(cause: Error) {\n    super(cause.message, { cause });\n    this.cause = cause;\n  }\n}\n\nasync function callModelWithRetry(\n  input: ModelCallInput,\n): Promise<ModelCallResult & { totalTimeCost: number; retryCount: number }> {\n  const startTime = Date.now();\n  const {\n    modelRuntime: { config },\n    options,\n  } = input;\n  const maxAttempts = normalizeRetryCount(config.retryCount) + 1;\n  const retryInterval = config.retryInterval ?? 2000;\n  const attemptErrors: Array<{ attempt: number; error: unknown }> = [];\n\n  for (let attempt = 1; ; attempt++) {\n    options?.abortSignal?.throwIfAborted();\n    try {\n      const result = await callModelOnce(input, attempt);\n      return {\n        ...result,\n        totalTimeCost: Date.now() - startTime,\n        retryCount: attempt - 1,\n      };\n    } catch (error) {\n      options?.abortSignal?.throwIfAborted();\n      if (error instanceof NonRetryableModelCallError) {\n        throw error.cause;\n      }\n      const lastError = toError(error);\n      attemptErrors.push({ attempt, error: lastError });\n      if (attempt === maxAttempts) {\n        throw appendAIRequestFailureSummary(\n          lastError,\n          attemptErrors,\n          maxAttempts,\n        );\n      }\n      getDebug('ai:call', { console: true })(\n        `AI call failed (attempt ${attempt}/${maxAttempts}), retrying in ${retryInterval}ms... Error: ${lastError.message}`,\n      );\n    }\n    await waitForRetry(retryInterval, options?.abortSignal);\n  }\n}\n\nasync function callModelOnce(\n  {\n    messages,\n    modelRuntime,\n    options,\n    executionId,\n    internalCallId,\n    prepared,\n  }: ModelCallInput,\n  attempt: number,\n): Promise<ModelCallResult> {\n  const { config: modelConfig } = modelRuntime;\n  const recordEvent = isModelCallRecordingEnabled()\n    ? (event: Record<string, unknown>) => {\n        void recordModelCallEvent({\n          executionId,\n          callId: internalCallId,\n          semanticRetryAttempt: options?.semanticRetryAttempt,\n          slot: modelConfig.slot,\n          intent: modelConfig.intent,\n          modelFamily: modelConfig.modelFamily,\n          ...event,\n          attempt,\n        });\n      }\n    : undefined;\n  const effectiveTimeoutMs = resolveEffectiveTimeoutMs(modelConfig.timeout);\n  const { signal: requestSignal, cleanup } = buildRequestAbortSignal(\n    effectiveTimeoutMs,\n    options?.abortSignal,\n  );\n  let deliveredChunk = false;\n  const onChunk = options?.onChunk;\n  const context: ModelCallContext = {\n    messages,\n    modelRuntime,\n    executionId,\n    recordEvent,\n    requestSignal,\n    effectiveTimeoutMs,\n    options: onChunk\n      ? {\n          ...options,\n          onChunk: (chunk) => {\n            requestSignal.throwIfAborted();\n            deliveredChunk = true;\n            onChunk(chunk);\n          },\n        }\n      : options,\n  };\n  try {\n    return await runWithAbortSignal(requestSignal, () =>\n      prepared.protocol === 'codex'\n        ? callCodex(context, prepared.input)\n        : callOpenAI(context, prepared),\n    );\n  } catch (error) {\n    options?.abortSignal?.throwIfAborted();\n    const lastError = toError(error);\n    if (deliveredChunk) {\n      throw new NonRetryableModelCallError(lastError);\n    }\n    throw lastError;\n  } finally {\n    cleanup();\n  }\n}\n"],"names":["DEFAULT_MODEL_PROTOCOL","callAI","messages","modelRuntime","options","isStreaming","assert","executionId","uuid","internalCallId","nextInternalCallId","modelConfig","prepare","input","isCodexAppServerProvider","prepareCodexCall","protocol","prepareResponses","prepareChatCompletion","prepared","runWithAbortSignal","result","callModelWithRetry","rawUsage","timeCost","totalTimeCost","retryCount","requestId","responseModelName","response","usage","buildUsageInfo","debugProfileStats","getDebug","debugProfileDetail","JSON","NonRetryableModelCallError","Error","cause","startTime","Date","config","maxAttempts","normalizeRetryCount","retryInterval","attemptErrors","attempt","callModelOnce","error","lastError","toError","appendAIRequestFailureSummary","waitForRetry","recordEvent","isModelCallRecordingEnabled","event","recordModelCallEvent","undefined","effectiveTimeoutMs","resolveEffectiveTimeoutMs","requestSignal","cleanup","buildRequestAbortSignal","deliveredChunk","onChunk","context","chunk","callCodex","callOpenAI"],"mappings":";;;;;;;;;;;;;;;;;;;;AAiCA,MAAMA,yBAAyB;AAExB,eAAeC,OACpBC,QAA2B,EAC3BC,YAA0B,EAC1BC,OAAuB;IAEvBA,SAAS,aAAa;IACtB,MAAMC,cAAcD,SAAS,WAAW;IACxC,IAAIC,aACFC,OACE,AAA4B,cAA5B,OAAOF,SAAS,SAChB;IAMJ,MAAMG,cAAcJ,aAAa,WAAW,IAAI,CAAC,SAAS,EAAEK,QAAQ;IAKpE,MAAMC,iBAAiBC;IAEvB,MAAM,EAAE,QAAQC,WAAW,EAAE,GAAGR;IAEhC,MAAMS,UAAU;QACd,MAAMC,QAAQ;YAAEX;YAAUC;YAAcC;QAAQ;QAChD,IAAIU,yBAAyBH,YAAY,aAAa,GACpD,OAAO;YAAE,UAAU;YAAS,OAAOI,iBAAiBF;QAAO;QAE7D,MAAMG,WAAWL,YAAY,QAAQ,IAAIX;QAEzCM,OACEH,aAAa,OAAO,CAAC,kBAAkB,CAAC,QAAQ,CAACa,WACjD,CAAC,eAAe,EAAEL,YAAY,WAAW,IAAI,UAAU,6BAA6B,EAAEK,SAAS,CAAC,CAAC;QAEnG,IAAIA,AAAa,uBAAbA,UACF,OAAO;YAAE,UAAU;YAAa,OAAO,MAAMC,iBAAiBJ;QAAO;QAEvE,OAAO;YACL,UAAU;YACV,OAAO,MAAMK,sBAAsBL;QACrC;IACF;IAEA,MAAMM,WAAWf,SAAS,cACtB,MAAMgB,mBAAmBhB,QAAQ,WAAW,EAAEQ,WAC9C,MAAMA;IAEV,MAAMS,SAAS,MAAMC,mBAAmB;QACtCpB;QACAC;QACAC;QACAG;QACAE;QACAU;IACF;IAEA,MAAM,EACJI,QAAQ,EACRC,QAAQ,EACRC,aAAa,EACbC,UAAU,EACVC,SAAS,EACTC,iBAAiB,EACjB,GAAGC,UACJ,GAAGR;IAEJ,MAAMS,QAAQC,eAAe;QAC3B,SAASZ,SAAS,QAAQ;QAC1B,WAAWI;QACXC;QACAC;QACAC;QACAC;QACAC;QACA,WAAWjB,YAAY,SAAS;QAChC,kBAAkBA,YAAY,gBAAgB;QAC9C,MAAMA,YAAY,IAAI;QACtBF;IACF;IAEA,MAAMuB,oBAAoBC,SAAS;IACnC,MAAMC,qBAAqBD,SAAS;IACpCD,kBACE,CAAC,OAAO,EAAErB,YAAY,SAAS,CAAC,QAAQ,EAAEA,YAAY,WAAW,IAAI,UAAU,aAAa,EAAEkB,SAAS,UAAU,CAAC,iBAAiB,EAAEC,OAAO,iBAAiB,GAAG,qBAAqB,EAAEA,OAAO,qBAAqB,GAAG,gBAAgB,EAAEA,OAAO,gBAAgB,GAAG,gBAAgB,EAAEA,OAAO,gBAAgB,GAAG,WAAW,EAAEN,YAAY,GAAG,aAAa,EAAEG,aAAa,GAAG,QAAQ,EAAEhB,YAAY,IAAI,CAAC,0BAA0B,EAAEA,YAAY,WAAW,IAAI,IAAI;IAEjcuB,mBAAmB,CAAC,oBAAoB,EAAEC,KAAK,SAAS,CAACL,QAAQ;IAEjE,IAAIA,SAAS3B,aAAa,OAAO,EAC/BA,aAAa,OAAO,CAAC2B;IAEvB,OAAO;QAAE,GAAGD,QAAQ;QAAEC;IAAM;AAC9B;AAgBA,MAAMM,mCAAmCC;IAGvC,YAAYC,KAAY,CAAE;QACxB,KAAK,CAACA,MAAM,OAAO,EAAE;YAAEA;QAAM,IAH/B,uBAAS,SAAT;QAIE,IAAI,CAAC,KAAK,GAAGA;IACf;AACF;AAEA,eAAehB,mBACbT,KAAqB;IAErB,MAAM0B,YAAYC,KAAK,GAAG;IAC1B,MAAM,EACJ,cAAc,EAAEC,MAAM,EAAE,EACxBrC,OAAO,EACR,GAAGS;IACJ,MAAM6B,cAAcC,oBAAoBF,OAAO,UAAU,IAAI;IAC7D,MAAMG,gBAAgBH,OAAO,aAAa,IAAI;IAC9C,MAAMI,gBAA4D,EAAE;IAEpE,IAAK,IAAIC,UAAU,IAAKA,UAAW;QACjC1C,SAAS,aAAa;QACtB,IAAI;YACF,MAAMiB,SAAS,MAAM0B,cAAclC,OAAOiC;YAC1C,OAAO;gBACL,GAAGzB,MAAM;gBACT,eAAemB,KAAK,GAAG,KAAKD;gBAC5B,YAAYO,UAAU;YACxB;QACF,EAAE,OAAOE,OAAO;YACd5C,SAAS,aAAa;YACtB,IAAI4C,iBAAiBZ,4BACnB,MAAMY,MAAM,KAAK;YAEnB,MAAMC,YAAYC,QAAQF;YAC1BH,cAAc,IAAI,CAAC;gBAAEC;gBAAS,OAAOG;YAAU;YAC/C,IAAIH,YAAYJ,aACd,MAAMS,8BACJF,WACAJ,eACAH;YAGJT,SAAS,WAAW;gBAAE,SAAS;YAAK,GAClC,CAAC,wBAAwB,EAAEa,QAAQ,CAAC,EAAEJ,YAAY,eAAe,EAAEE,cAAc,aAAa,EAAEK,UAAU,OAAO,EAAE;QAEvH;QACA,MAAMG,aAAaR,eAAexC,SAAS;IAC7C;AACF;AAEA,eAAe2C,cACb,EACE7C,QAAQ,EACRC,YAAY,EACZC,OAAO,EACPG,WAAW,EACXE,cAAc,EACdU,QAAQ,EACO,EACjB2B,OAAe;IAEf,MAAM,EAAE,QAAQnC,WAAW,EAAE,GAAGR;IAChC,MAAMkD,cAAcC,gCAChB,CAACC;QACMC,qBAAqB;YACxBjD;YACA,QAAQE;YACR,sBAAsBL,SAAS;YAC/B,MAAMO,YAAY,IAAI;YACtB,QAAQA,YAAY,MAAM;YAC1B,aAAaA,YAAY,WAAW;YACpC,GAAG4C,KAAK;YACRT;QACF;IACF,IACAW;IACJ,MAAMC,qBAAqBC,0BAA0BhD,YAAY,OAAO;IACxE,MAAM,EAAE,QAAQiD,aAAa,EAAEC,OAAO,EAAE,GAAGC,wBACzCJ,oBACAtD,SAAS;IAEX,IAAI2D,iBAAiB;IACrB,MAAMC,UAAU5D,SAAS;IACzB,MAAM6D,UAA4B;QAChC/D;QACAC;QACAI;QACA8C;QACAO;QACAF;QACA,SAASM,UACL;YACE,GAAG5D,OAAO;YACV,SAAS,CAAC8D;gBACRN,cAAc,cAAc;gBAC5BG,iBAAiB;gBACjBC,QAAQE;YACV;QACF,IACA9D;IACN;IACA,IAAI;QACF,OAAO,MAAMgB,mBAAmBwC,eAAe,IAC7CzC,AAAsB,YAAtBA,SAAS,QAAQ,GACbgD,UAAUF,SAAS9C,SAAS,KAAK,IACjCiD,WAAWH,SAAS9C;IAE5B,EAAE,OAAO6B,OAAO;QACd5C,SAAS,aAAa;QACtB,MAAM6C,YAAYC,QAAQF;QAC1B,IAAIe,gBACF,MAAM,IAAI3B,2BAA2Ba;QAEvC,MAAMA;IACR,SAAU;QACRY;IACF;AACF"}

package/dist/es/ai-model/service-caller/codex/call-codex.mjs

@@ -1,8 +1,7 @@
-import { applyImageDetail } from "../utils.mjs";
 import { callAIWithCodexAppServer } from "./codex-app-server.mjs";
 const prepareCodexCall = ({ messages, modelRuntime, options })=>{
     const { config: modelConfig, adapter } = modelRuntime;
-    const { config, imageDetail } = adapter.buildCodexAppServerParams({
+    const { config } = adapter.buildCodexAppServerParams({
         intent: modelConfig.intent,
         userConfig: {
             reasoningEnabled: modelConfig.reasoningEnabled,
@@ -11,16 +10,18 @@ const prepareCodexCall = ({ messages, modelRuntime, options })=>{
         },
         requiresOriginalImageDetail: options?.requiresOriginalImageDetail
     });
-    const messagesWithImageDetail = applyImageDetail({
-        imageDetail,
-        messages
-    });
+    const resolveImageDetail = ({ imageDetail })=>adapter.resolveImageDetail({
+            imageDetail,
+            intent: modelConfig.intent,
+            requiresOriginalImageDetail: options?.requiresOriginalImageDetail
+        });
     return {
-        messages: messagesWithImageDetail,
+        messages,
+        resolveImageDetail,
         requestParams: config
     };
 };
-const callCodex = async ({ modelRuntime, options, recordEvent, requestSignal }, { messages, requestParams })=>{
+const callCodex = async ({ modelRuntime, options, recordEvent, requestSignal }, { messages, resolveImageDetail, requestParams })=>{
     const { config: modelConfig } = modelRuntime;
     let protocolChunkSequence = 0;
     const codexStartTime = Date.now();
@@ -46,6 +47,7 @@ const callCodex = async ({ modelRuntime, options, recordEvent, requestSignal },
             stream: options?.stream,
             onChunk: options?.onChunk,
             params: requestParams,
+            resolveImageDetail,
             abortSignal: requestSignal,
             onRecordEvent: recordCodexEvent
         });

package/dist/es/ai-model/service-caller/codex/call-codex.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/service-caller/codex/call-codex.mjs","sources":["../../../../../src/ai-model/service-caller/codex/call-codex.ts"],"sourcesContent":["import type { ModelCallContext, ModelCallResult } from '../types';\nimport { applyImageDetail } from '../utils';\nimport {\n  type CodexAppServerRecordEvent,\n  callAIWithCodexAppServer,\n} from './codex-app-server';\n\nexport const prepareCodexCall = ({\n  messages,\n  modelRuntime,\n  options,\n}: Pick<ModelCallContext, 'messages' | 'modelRuntime' | 'options'>) => {\n  const { config: modelConfig, adapter } = modelRuntime;\n  const { config, imageDetail } = adapter.buildCodexAppServerParams({\n    intent: modelConfig.intent,\n    userConfig: {\n      reasoningEnabled: modelConfig.reasoningEnabled,\n      reasoningEffort: modelConfig.reasoningEffort,\n      reasoningBudget: modelConfig.reasoningBudget,\n    },\n    requiresOriginalImageDetail: options?.requiresOriginalImageDetail,\n  });\n  const messagesWithImageDetail = applyImageDetail({ imageDetail, messages });\n  return { messages: messagesWithImageDetail, requestParams: config };\n};\n\nexport const callCodex = async (\n  { modelRuntime, options, recordEvent, requestSignal }: ModelCallContext,\n  { messages, requestParams }: ReturnType<typeof prepareCodexCall>,\n): Promise<ModelCallResult> => {\n  const { config: modelConfig } = modelRuntime;\n  let protocolChunkSequence = 0;\n  const codexStartTime = Date.now();\n  const recordCodexEvent = recordEvent\n    ? (event: CodexAppServerRecordEvent) => {\n        if (event.type === 'chunk') {\n          protocolChunkSequence += 1;\n          recordEvent({\n            ...event,\n            attempt: 1,\n            sequence: protocolChunkSequence,\n            provider: 'codex-app-server',\n          });\n          return;\n        }\n\n        recordEvent({\n          ...event,\n          attempt: 1,\n          provider: 'codex-app-server',\n        });\n      }\n    : undefined;\n\n  try {\n    const codexResult = await callAIWithCodexAppServer(messages, modelConfig, {\n      stream: options?.stream,\n      onChunk: options?.onChunk,\n      params: requestParams,\n      abortSignal: requestSignal,\n      onRecordEvent: recordCodexEvent,\n    });\n    requestSignal.throwIfAborted();\n    const { protocolMetadata, usage: rawUsage, ...response } = codexResult;\n    const timeCost = Date.now() - codexStartTime;\n    recordEvent?.({\n      type: 'response',\n      attempt: 1,\n      provider: 'codex-app-server',\n      final: {\n        content: response.content,\n        reasoningContent: response.reasoning_content,\n        usage: rawUsage,\n        timeCost,\n        protocol: protocolMetadata,\n      },\n    });\n    return {\n      ...response,\n      rawUsage,\n      timeCost,\n      requestId: protocolMetadata.turnId,\n    };\n  } catch (error) {\n    recordEvent?.({\n      type: 'error',\n      attempt: 1,\n      provider: 'codex-app-server',\n      error:\n        error instanceof Error\n          ? {\n              name: error.name,\n              message: error.message,\n              stack: error.stack,\n            }\n          : String(error),\n    });\n    throw error;\n  }\n};\n"],"names":["prepareCodexCall","messages","modelRuntime","options","modelConfig","adapter","config","imageDetail","messagesWithImageDetail","applyImageDetail","callCodex","recordEvent","requestSignal","requestParams","protocolChunkSequence","codexStartTime","Date","recordCodexEvent","event","undefined","codexResult","callAIWithCodexAppServer","protocolMetadata","rawUsage","response","timeCost","error","Error","String"],"mappings":";;AAOO,MAAMA,mBAAmB,CAAC,EAC/BC,QAAQ,EACRC,YAAY,EACZC,OAAO,EACyD;IAChE,MAAM,EAAE,QAAQC,WAAW,EAAEC,OAAO,EAAE,GAAGH;IACzC,MAAM,EAAEI,MAAM,EAAEC,WAAW,EAAE,GAAGF,QAAQ,yBAAyB,CAAC;QAChE,QAAQD,YAAY,MAAM;QAC1B,YAAY;YACV,kBAAkBA,YAAY,gBAAgB;YAC9C,iBAAiBA,YAAY,eAAe;YAC5C,iBAAiBA,YAAY,eAAe;QAC9C;QACA,6BAA6BD,SAAS;IACxC;IACA,MAAMK,0BAA0BC,iBAAiB;QAAEF;QAAaN;IAAS;IACzE,OAAO;QAAE,UAAUO;QAAyB,eAAeF;IAAO;AACpE;AAEO,MAAMI,YAAY,OACvB,EAAER,YAAY,EAAEC,OAAO,EAAEQ,WAAW,EAAEC,aAAa,EAAoB,EACvE,EAAEX,QAAQ,EAAEY,aAAa,EAAuC;IAEhE,MAAM,EAAE,QAAQT,WAAW,EAAE,GAAGF;IAChC,IAAIY,wBAAwB;IAC5B,MAAMC,iBAAiBC,KAAK,GAAG;IAC/B,MAAMC,mBAAmBN,cACrB,CAACO;QACC,IAAIA,AAAe,YAAfA,MAAM,IAAI,EAAc;YAC1BJ,yBAAyB;YACzBH,YAAY;gBACV,GAAGO,KAAK;gBACR,SAAS;gBACT,UAAUJ;gBACV,UAAU;YACZ;YACA;QACF;QAEAH,YAAY;YACV,GAAGO,KAAK;YACR,SAAS;YACT,UAAU;QACZ;IACF,IACAC;IAEJ,IAAI;QACF,MAAMC,cAAc,MAAMC,yBAAyBpB,UAAUG,aAAa;YACxE,QAAQD,SAAS;YACjB,SAASA,SAAS;YAClB,QAAQU;YACR,aAAaD;YACb,eAAeK;QACjB;QACAL,cAAc,cAAc;QAC5B,MAAM,EAAEU,gBAAgB,EAAE,OAAOC,QAAQ,EAAE,GAAGC,UAAU,GAAGJ;QAC3D,MAAMK,WAAWT,KAAK,GAAG,KAAKD;QAC9BJ,cAAc;YACZ,MAAM;YACN,SAAS;YACT,UAAU;YACV,OAAO;gBACL,SAASa,SAAS,OAAO;gBACzB,kBAAkBA,SAAS,iBAAiB;gBAC5C,OAAOD;gBACPE;gBACA,UAAUH;YACZ;QACF;QACA,OAAO;YACL,GAAGE,QAAQ;YACXD;YACAE;YACA,WAAWH,iBAAiB,MAAM;QACpC;IACF,EAAE,OAAOI,OAAO;QACdf,cAAc;YACZ,MAAM;YACN,SAAS;YACT,UAAU;YACV,OACEe,iBAAiBC,QACb;gBACE,MAAMD,MAAM,IAAI;gBAChB,SAASA,MAAM,OAAO;gBACtB,OAAOA,MAAM,KAAK;YACpB,IACAE,OAAOF;QACf;QACA,MAAMA;IACR;AACF"}
+{"version":3,"file":"ai-model/service-caller/codex/call-codex.mjs","sources":["../../../../../src/ai-model/service-caller/codex/call-codex.ts"],"sourcesContent":["import type { ResolveImageDetail } from '../../model-adapter/types';\nimport type { ModelCallContext, ModelCallResult } from '../types';\nimport {\n  type CodexAppServerRecordEvent,\n  callAIWithCodexAppServer,\n} from './codex-app-server';\n\nexport const prepareCodexCall = ({\n  messages,\n  modelRuntime,\n  options,\n}: Pick<ModelCallContext, 'messages' | 'modelRuntime' | 'options'>) => {\n  const { config: modelConfig, adapter } = modelRuntime;\n  const { config } = adapter.buildCodexAppServerParams({\n    intent: modelConfig.intent,\n    userConfig: {\n      reasoningEnabled: modelConfig.reasoningEnabled,\n      reasoningEffort: modelConfig.reasoningEffort,\n      reasoningBudget: modelConfig.reasoningBudget,\n    },\n    requiresOriginalImageDetail: options?.requiresOriginalImageDetail,\n  });\n  const resolveImageDetail: ResolveImageDetail = ({ imageDetail }) =>\n    adapter.resolveImageDetail({\n      imageDetail,\n      intent: modelConfig.intent,\n      requiresOriginalImageDetail: options?.requiresOriginalImageDetail,\n    });\n  return { messages, resolveImageDetail, requestParams: config };\n};\n\nexport const callCodex = async (\n  { modelRuntime, options, recordEvent, requestSignal }: ModelCallContext,\n  {\n    messages,\n    resolveImageDetail,\n    requestParams,\n  }: ReturnType<typeof prepareCodexCall>,\n): Promise<ModelCallResult> => {\n  const { config: modelConfig } = modelRuntime;\n  let protocolChunkSequence = 0;\n  const codexStartTime = Date.now();\n  const recordCodexEvent = recordEvent\n    ? (event: CodexAppServerRecordEvent) => {\n        if (event.type === 'chunk') {\n          protocolChunkSequence += 1;\n          recordEvent({\n            ...event,\n            attempt: 1,\n            sequence: protocolChunkSequence,\n            provider: 'codex-app-server',\n          });\n          return;\n        }\n\n        recordEvent({\n          ...event,\n          attempt: 1,\n          provider: 'codex-app-server',\n        });\n      }\n    : undefined;\n\n  try {\n    const codexResult = await callAIWithCodexAppServer(messages, modelConfig, {\n      stream: options?.stream,\n      onChunk: options?.onChunk,\n      params: requestParams,\n      resolveImageDetail,\n      abortSignal: requestSignal,\n      onRecordEvent: recordCodexEvent,\n    });\n    requestSignal.throwIfAborted();\n    const { protocolMetadata, usage: rawUsage, ...response } = codexResult;\n    const timeCost = Date.now() - codexStartTime;\n    recordEvent?.({\n      type: 'response',\n      attempt: 1,\n      provider: 'codex-app-server',\n      final: {\n        content: response.content,\n        reasoningContent: response.reasoning_content,\n        usage: rawUsage,\n        timeCost,\n        protocol: protocolMetadata,\n      },\n    });\n    return {\n      ...response,\n      rawUsage,\n      timeCost,\n      requestId: protocolMetadata.turnId,\n    };\n  } catch (error) {\n    recordEvent?.({\n      type: 'error',\n      attempt: 1,\n      provider: 'codex-app-server',\n      error:\n        error instanceof Error\n          ? {\n              name: error.name,\n              message: error.message,\n              stack: error.stack,\n            }\n          : String(error),\n    });\n    throw error;\n  }\n};\n"],"names":["prepareCodexCall","messages","modelRuntime","options","modelConfig","adapter","config","resolveImageDetail","imageDetail","callCodex","recordEvent","requestSignal","requestParams","protocolChunkSequence","codexStartTime","Date","recordCodexEvent","event","undefined","codexResult","callAIWithCodexAppServer","protocolMetadata","rawUsage","response","timeCost","error","Error","String"],"mappings":";AAOO,MAAMA,mBAAmB,CAAC,EAC/BC,QAAQ,EACRC,YAAY,EACZC,OAAO,EACyD;IAChE,MAAM,EAAE,QAAQC,WAAW,EAAEC,OAAO,EAAE,GAAGH;IACzC,MAAM,EAAEI,MAAM,EAAE,GAAGD,QAAQ,yBAAyB,CAAC;QACnD,QAAQD,YAAY,MAAM;QAC1B,YAAY;YACV,kBAAkBA,YAAY,gBAAgB;YAC9C,iBAAiBA,YAAY,eAAe;YAC5C,iBAAiBA,YAAY,eAAe;QAC9C;QACA,6BAA6BD,SAAS;IACxC;IACA,MAAMI,qBAAyC,CAAC,EAAEC,WAAW,EAAE,GAC7DH,QAAQ,kBAAkB,CAAC;YACzBG;YACA,QAAQJ,YAAY,MAAM;YAC1B,6BAA6BD,SAAS;QACxC;IACF,OAAO;QAAEF;QAAUM;QAAoB,eAAeD;IAAO;AAC/D;AAEO,MAAMG,YAAY,OACvB,EAAEP,YAAY,EAAEC,OAAO,EAAEO,WAAW,EAAEC,aAAa,EAAoB,EACvE,EACEV,QAAQ,EACRM,kBAAkB,EAClBK,aAAa,EACuB;IAEtC,MAAM,EAAE,QAAQR,WAAW,EAAE,GAAGF;IAChC,IAAIW,wBAAwB;IAC5B,MAAMC,iBAAiBC,KAAK,GAAG;IAC/B,MAAMC,mBAAmBN,cACrB,CAACO;QACC,IAAIA,AAAe,YAAfA,MAAM,IAAI,EAAc;YAC1BJ,yBAAyB;YACzBH,YAAY;gBACV,GAAGO,KAAK;gBACR,SAAS;gBACT,UAAUJ;gBACV,UAAU;YACZ;YACA;QACF;QAEAH,YAAY;YACV,GAAGO,KAAK;YACR,SAAS;YACT,UAAU;QACZ;IACF,IACAC;IAEJ,IAAI;QACF,MAAMC,cAAc,MAAMC,yBAAyBnB,UAAUG,aAAa;YACxE,QAAQD,SAAS;YACjB,SAASA,SAAS;YAClB,QAAQS;YACRL;YACA,aAAaI;YACb,eAAeK;QACjB;QACAL,cAAc,cAAc;QAC5B,MAAM,EAAEU,gBAAgB,EAAE,OAAOC,QAAQ,EAAE,GAAGC,UAAU,GAAGJ;QAC3D,MAAMK,WAAWT,KAAK,GAAG,KAAKD;QAC9BJ,cAAc;YACZ,MAAM;YACN,SAAS;YACT,UAAU;YACV,OAAO;gBACL,SAASa,SAAS,OAAO;gBACzB,kBAAkBA,SAAS,iBAAiB;gBAC5C,OAAOD;gBACPE;gBACA,UAAUH;YACZ;QACF;QACA,OAAO;YACL,GAAGE,QAAQ;YACXD;YACAE;YACA,WAAWH,iBAAiB,MAAM;QACpC;IACF,EAAE,OAAOI,OAAO;QACdf,cAAc;YACZ,MAAM;YACN,SAAS;YACT,UAAU;YACV,OACEe,iBAAiBC,QACb;gBACE,MAAMD,MAAM,IAAI;gBAChB,SAASA,MAAM,OAAO;gBACtB,OAAOA,MAAM,KAAK;YACpB,IACAE,OAAOF;QACf;QACA,MAAMA;IACR;AACF"}

package/dist/es/ai-model/service-caller/codex/codex-app-server.mjs

@@ -52,7 +52,6 @@ const toNonEmptyString = (value)=>{
     const trimmed = value.trim();
     return trimmed || void 0;
 };
-const toCodexImageDetail = (value)=>'auto' === value || 'low' === value || 'high' === value || 'original' === value ? value : void 0;
 const normalizeCodexLocalImagePath = (imageUrl, platform = process.platform)=>{
     if (!imageUrl.startsWith('file://')) return imageUrl;
     try {
@@ -71,54 +70,54 @@ const normalizeCodexLocalImagePath = (imageUrl, platform = process.platform)=>{
     }
 };
 const extractTextFromMessage = (message)=>{
-    const content = message.content;
+    const { content } = message;
     if ('string' == typeof content) return content;
-    if (Array.isArray(content)) return content.map((part)=>{
-        if (!part || 'object' != typeof part) return '';
-        if ('text' === part.type && 'string' == typeof part.text) return part.text;
-        if ('input_text' === part.type && 'string' == typeof part.text) return part.text;
-        return '';
-    }).filter(Boolean).join('\n');
-    return '';
+    return content.map((part)=>'text' === part.type ? part.text : '').filter(Boolean).join('\n');
 };
-const extractImageInputs = (message)=>{
-    const content = message.content;
+const extractImageInputs = (message, resolveImageDetail)=>{
+    const { content } = message;
     if (!Array.isArray(content)) return [];
     const inputs = [];
     for (const part of content){
-        if (!part || 'object' != typeof part) continue;
-        const partType = String(part.type || '');
-        const imageUrl = 'image_url' === partType ? toNonEmptyString(part.image_url?.url) : 'input_image' === partType ? toNonEmptyString(part.image_url || part.url) : void 0;
-        const imageDetail = toCodexImageDetail('image_url' === partType ? part.image_url?.detail : part.detail);
+        if ('image' !== part.type) continue;
+        const imageUrl = toNonEmptyString(part.url);
+        const detail = resolveImageDetail({
+            imageDetail: part.detail
+        });
         if (imageUrl) {
             if (imageUrl.startsWith('/') || imageUrl.startsWith('./') || imageUrl.startsWith('../') || imageUrl.startsWith('file://')) {
                 const path = imageUrl.startsWith('file://') ? normalizeCodexLocalImagePath(imageUrl) : imageUrl;
                 inputs.push({
                     type: 'localImage',
                     path,
-                    ...imageDetail ? {
-                        detail: imageDetail
-                    } : {}
+                    detail
                 });
                 continue;
             }
             inputs.push({
                 type: 'image',
                 url: imageUrl,
-                ...imageDetail ? {
-                    detail: imageDetail
-                } : {}
+                detail
             });
         }
     }
     return inputs;
 };
-const buildCodexTurnPayloadFromMessages = (messages)=>{
+const buildCodexTurnPayloadFromMessages = (messages, resolveImageDetail)=>{
     const developerInstructionParts = [];
     const transcriptParts = [];
     const imageInputs = [];
-    for (const message of messages){
-        const role = String(message.role || 'user');
+    for (const entry of messages){
+        const message = (()=>{
+            if ('role' in entry) return entry;
+            if ('input-message' === entry.type) return entry.message;
+            if ('chat-completion' !== entry.output.type) throw new Error('Cannot replay Responses output in a Codex conversation');
+            return {
+                role: 'assistant',
+                content: entry.output.rawValue.content ?? ''
+            };
+        })();
+        const { role } = message;
         const text = extractTextFromMessage(message);
         if ('system' === role) {
             if (text.trim()) developerInstructionParts.push(text.trim());
@@ -127,7 +126,7 @@ const buildCodexTurnPayloadFromMessages = (messages)=>{
         const roleTag = role.toUpperCase();
         if (text.trim()) transcriptParts.push(`[${roleTag}]\n${text.trim()}`);
         else transcriptParts.push(`[${roleTag}]\n(no text content)`);
-        if ('user' === role) imageInputs.push(...extractImageInputs(message));
+        if ('user' === role) imageInputs.push(...extractImageInputs(message, resolveImageDetail));
     }
     const fullTranscript = transcriptParts.join('\n\n');
     const transcriptText = (fullTranscript.length > CODEX_TEXT_INPUT_MAX_LENGTH ? fullTranscript.slice(-CODEX_TEXT_INPUT_MAX_LENGTH) : fullTranscript) || 'Please answer the latest user request.';
@@ -175,9 +174,9 @@ class CodexAppServerConnection {
     isClosed() {
         return this.closed;
     }
-    async runTurn({ messages, modelConfig, stream, onChunk, params, abortSignal, onRecordEvent }) {
+    async runTurn({ messages, resolveImageDetail, modelConfig, stream, onChunk, params, abortSignal, onRecordEvent }) {
         const isStreaming = !!(stream && onChunk);
-        const { developerInstructions, input } = buildCodexTurnPayloadFromMessages(messages);
+        const { developerInstructions, input } = buildCodexTurnPayloadFromMessages(messages, resolveImageDetail);
         let threadId;
         let turnId;
         let latestErrorMessage;
@@ -550,7 +549,7 @@ class CodexAppServerConnection {
     }
 }
 class CodexAppServerConnectionManager {
-    async runTurn({ messages, modelConfig, stream, onChunk, params, abortSignal, onRecordEvent }) {
+    async runTurn({ messages, resolveImageDetail, modelConfig, stream, onChunk, params, abortSignal, onRecordEvent }) {
         return this.runner.run(async ()=>{
             abortSignal?.throwIfAborted();
             const connection = await this.getConnection();
@@ -558,6 +557,7 @@ class CodexAppServerConnectionManager {
             try {
                 return await connection.runTurn({
                     messages,
+                    resolveImageDetail,
                     modelConfig,
                     stream,
                     onChunk,
@@ -599,6 +599,7 @@ async function callAIWithCodexAppServer(messages, modelConfig, options) {
     return codexConnectionManager.runTurn({
         messages,
         modelConfig,
+        resolveImageDetail: options.resolveImageDetail,
         stream: options?.stream,
         onChunk: options?.onChunk,
         params: options?.params,