@midscene/core 1.13.4-beta-20260928072823.0 → 1.14.0

package/dist/es/ai-model/model-adapter/chat-completion.mjs

@@ -1,4 +1,4 @@
-const defaultImageDetail = (_input)=>void 0;
+import { getTemperatureWithSemanticRetry } from "./temperature-with-semantic-retry.mjs";
 const defaultChatCompletionParams = ({ midsceneDefaults, userConfig })=>({
         config: {
             temperature: userConfig.temperature ?? midsceneDefaults.temperature
@@ -31,7 +31,6 @@ function resolveExtractContentAndReasoning(chatCompletion) {
 }
 function resolveChatCompletion(chatCompletion) {
     const buildChatCompletionParams = chatCompletion?.buildChatCompletionParams ?? defaultChatCompletionParams;
-    const resolveImageDetail = chatCompletion?.resolveImageDetail ?? defaultImageDetail;
     const unsupportedUserConfig = chatCompletion?.unsupportedUserConfig ?? [];
     const extractContentAndReasoning = resolveExtractContentAndReasoning(chatCompletion);
     const useReasoningAsContentFallback = chatCompletion?.useReasoningAsContentFallback ?? false;
@@ -45,20 +44,18 @@ function resolveChatCompletion(chatCompletion) {
                 midsceneDefaults: midsceneChatCompletionDefaults
             };
             const params = buildChatCompletionParams(context);
-            const retryAttempt = input.semanticRetryAttempt ?? 0;
-            if (retryAttempt > 0 && input.userConfig?.temperature === void 0 && 0 === params.config.temperature) return {
+            return {
+                ...params,
                 config: {
                     ...params.config,
-                    temperature: 0.2
+                    temperature: getTemperatureWithSemanticRetry({
+                        temperature: params.config.temperature,
+                        userTemperature: input.userConfig?.temperature,
+                        semanticRetryAttempt: input.semanticRetryAttempt
+                    })
                 }
             };
-            return params;
         },
-        resolveImageDetail: (input)=>resolveImageDetail({
-                ...input,
-                userConfig: input.userConfig ?? {},
-                midsceneDefaults: midsceneChatCompletionDefaults
-            }),
         extractContentAndReasoning,
         useReasoningAsContentFallback,
         replayRawAssistantMessage

package/dist/es/ai-model/model-adapter/chat-completion.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/model-adapter/chat-completion.mjs","sources":["../../../../src/ai-model/model-adapter/chat-completion.ts"],"sourcesContent":["import type {\n  ChatCompletionAdapter,\n  ChatCompletionCallContext,\n  ExtractContentAndReasoning,\n  MidsceneChatCompletionDefaults,\n  ModelAdapterDefinition,\n} from './types';\n\nconst defaultImageDetail = (_input: unknown) => undefined;\n\nconst defaultChatCompletionParams = ({\n  midsceneDefaults,\n  userConfig,\n}: ChatCompletionCallContext) => ({\n  config: {\n    temperature: userConfig.temperature ?? midsceneDefaults.temperature,\n  },\n});\n\nconst midsceneChatCompletionDefaults: MidsceneChatCompletionDefaults = {\n  temperature: 0,\n};\n\nfunction createDefaultExtractContentAndReasoning(\n  reasoningContentKeys: string[],\n): ExtractContentAndReasoning {\n  return (message) => {\n    if (!message) {\n      return {\n        content: '',\n        reasoning_content: '',\n      };\n    }\n\n    const messageRecord = message as Record<string, unknown>;\n    const rawReasoning = reasoningContentKeys\n      .map((key) => messageRecord[key])\n      .find((value) => typeof value === 'string');\n    const messageReasoning =\n      typeof rawReasoning === 'string' ? rawReasoning : '';\n\n    return {\n      content: typeof message.content === 'string' ? message.content : '',\n      reasoning_content: messageReasoning,\n    };\n  };\n}\n\nfunction resolveExtractContentAndReasoning(\n  chatCompletion: ModelAdapterDefinition['chatCompletion'],\n): ExtractContentAndReasoning {\n  const messageExtraction = chatCompletion?.messageExtraction;\n  if (messageExtraction?.kind === 'custom') {\n    return messageExtraction.extractContentAndReasoning;\n  }\n\n  return createDefaultExtractContentAndReasoning(\n    messageExtraction?.reasoningContentKeys ?? ['reasoning_content'],\n  );\n}\n\nexport function resolveChatCompletion(\n  chatCompletion: ModelAdapterDefinition['chatCompletion'],\n): ChatCompletionAdapter {\n  const buildChatCompletionParams =\n    chatCompletion?.buildChatCompletionParams ?? defaultChatCompletionParams;\n  const resolveImageDetail =\n    chatCompletion?.resolveImageDetail ?? defaultImageDetail;\n  const unsupportedUserConfig = chatCompletion?.unsupportedUserConfig ?? [];\n  const extractContentAndReasoning =\n    resolveExtractContentAndReasoning(chatCompletion);\n  const useReasoningAsContentFallback =\n    chatCompletion?.useReasoningAsContentFallback ?? false;\n  const replayRawAssistantMessage =\n    chatCompletion?.replayRawAssistantMessage ?? false;\n\n  return {\n    unsupportedUserConfig,\n    buildChatCompletionParams: (input) => {\n      const context = {\n        ...input,\n        userConfig: input.userConfig ?? {},\n        midsceneDefaults: midsceneChatCompletionDefaults,\n      };\n      const params = buildChatCompletionParams(context);\n      const retryAttempt = input.semanticRetryAttempt ?? 0;\n      // Only perturb the default deterministic request after a semantic parse\n      // failure: preserve an explicit user temperature, and avoid adding a\n      // temperature to adapters whose normal parameters do not include one.\n      if (\n        retryAttempt > 0 &&\n        input.userConfig?.temperature === undefined &&\n        params.config.temperature === 0\n      ) {\n        return {\n          config: {\n            ...params.config,\n            temperature: 0.2,\n          },\n        };\n      }\n      return params;\n    },\n    resolveImageDetail: (input) =>\n      resolveImageDetail({\n        ...input,\n        userConfig: input.userConfig ?? {},\n        midsceneDefaults: midsceneChatCompletionDefaults,\n      }),\n    extractContentAndReasoning,\n    useReasoningAsContentFallback,\n    replayRawAssistantMessage,\n  };\n}\n"],"names":["defaultImageDetail","_input","undefined","defaultChatCompletionParams","midsceneDefaults","userConfig","midsceneChatCompletionDefaults","createDefaultExtractContentAndReasoning","reasoningContentKeys","message","messageRecord","rawReasoning","key","value","messageReasoning","resolveExtractContentAndReasoning","chatCompletion","messageExtraction","resolveChatCompletion","buildChatCompletionParams","resolveImageDetail","unsupportedUserConfig","extractContentAndReasoning","useReasoningAsContentFallback","replayRawAssistantMessage","input","context","params","retryAttempt"],"mappings":"AAQA,MAAMA,qBAAqB,CAACC,SAAoBC;AAEhD,MAAMC,8BAA8B,CAAC,EACnCC,gBAAgB,EAChBC,UAAU,EACgB,GAAM;QAChC,QAAQ;YACN,aAAaA,WAAW,WAAW,IAAID,iBAAiB,WAAW;QACrE;IACF;AAEA,MAAME,iCAAiE;IACrE,aAAa;AACf;AAEA,SAASC,wCACPC,oBAA8B;IAE9B,OAAO,CAACC;QACN,IAAI,CAACA,SACH,OAAO;YACL,SAAS;YACT,mBAAmB;QACrB;QAGF,MAAMC,gBAAgBD;QACtB,MAAME,eAAeH,qBAClB,GAAG,CAAC,CAACI,MAAQF,aAAa,CAACE,IAAI,EAC/B,IAAI,CAAC,CAACC,QAAU,AAAiB,YAAjB,OAAOA;QAC1B,MAAMC,mBACJ,AAAwB,YAAxB,OAAOH,eAA4BA,eAAe;QAEpD,OAAO;YACL,SAAS,AAA2B,YAA3B,OAAOF,QAAQ,OAAO,GAAgBA,QAAQ,OAAO,GAAG;YACjE,mBAAmBK;QACrB;IACF;AACF;AAEA,SAASC,kCACPC,cAAwD;IAExD,MAAMC,oBAAoBD,gBAAgB;IAC1C,IAAIC,mBAAmB,SAAS,UAC9B,OAAOA,kBAAkB,0BAA0B;IAGrD,OAAOV,wCACLU,mBAAmB,wBAAwB;QAAC;KAAoB;AAEpE;AAEO,SAASC,sBACdF,cAAwD;IAExD,MAAMG,4BACJH,gBAAgB,6BAA6Bb;IAC/C,MAAMiB,qBACJJ,gBAAgB,sBAAsBhB;IACxC,MAAMqB,wBAAwBL,gBAAgB,yBAAyB,EAAE;IACzE,MAAMM,6BACJP,kCAAkCC;IACpC,MAAMO,gCACJP,gBAAgB,iCAAiC;IACnD,MAAMQ,4BACJR,gBAAgB,6BAA6B;IAE/C,OAAO;QACLK;QACA,2BAA2B,CAACI;YAC1B,MAAMC,UAAU;gBACd,GAAGD,KAAK;gBACR,YAAYA,MAAM,UAAU,IAAI,CAAC;gBACjC,kBAAkBnB;YACpB;YACA,MAAMqB,SAASR,0BAA0BO;YACzC,MAAME,eAAeH,MAAM,oBAAoB,IAAI;YAInD,IACEG,eAAe,KACfH,MAAM,UAAU,EAAE,gBAAgBvB,UAClCyB,AAA8B,MAA9BA,OAAO,MAAM,CAAC,WAAW,EAEzB,OAAO;gBACL,QAAQ;oBACN,GAAGA,OAAO,MAAM;oBAChB,aAAa;gBACf;YACF;YAEF,OAAOA;QACT;QACA,oBAAoB,CAACF,QACnBL,mBAAmB;gBACjB,GAAGK,KAAK;gBACR,YAAYA,MAAM,UAAU,IAAI,CAAC;gBACjC,kBAAkBnB;YACpB;QACFgB;QACAC;QACAC;IACF;AACF"}
+{"version":3,"file":"ai-model/model-adapter/chat-completion.mjs","sources":["../../../../src/ai-model/model-adapter/chat-completion.ts"],"sourcesContent":["import { getTemperatureWithSemanticRetry } from './temperature-with-semantic-retry';\nimport type {\n  ChatCompletionAdapter,\n  ChatCompletionCallContext,\n  ExtractContentAndReasoning,\n  MidsceneChatCompletionDefaults,\n  ModelAdapterDefinition,\n} from './types';\n\nconst defaultChatCompletionParams = ({\n  midsceneDefaults,\n  userConfig,\n}: ChatCompletionCallContext) => ({\n  config: {\n    temperature: userConfig.temperature ?? midsceneDefaults.temperature,\n  },\n});\n\nconst midsceneChatCompletionDefaults: MidsceneChatCompletionDefaults = {\n  temperature: 0,\n};\n\nfunction createDefaultExtractContentAndReasoning(\n  reasoningContentKeys: string[],\n): ExtractContentAndReasoning {\n  return (message) => {\n    if (!message) {\n      return {\n        content: '',\n        reasoning_content: '',\n      };\n    }\n\n    const messageRecord = message as Record<string, unknown>;\n    const rawReasoning = reasoningContentKeys\n      .map((key) => messageRecord[key])\n      .find((value) => typeof value === 'string');\n    const messageReasoning =\n      typeof rawReasoning === 'string' ? rawReasoning : '';\n\n    return {\n      content: typeof message.content === 'string' ? message.content : '',\n      reasoning_content: messageReasoning,\n    };\n  };\n}\n\nfunction resolveExtractContentAndReasoning(\n  chatCompletion: ModelAdapterDefinition['chatCompletion'],\n): ExtractContentAndReasoning {\n  const messageExtraction = chatCompletion?.messageExtraction;\n  if (messageExtraction?.kind === 'custom') {\n    return messageExtraction.extractContentAndReasoning;\n  }\n\n  return createDefaultExtractContentAndReasoning(\n    messageExtraction?.reasoningContentKeys ?? ['reasoning_content'],\n  );\n}\n\nexport function resolveChatCompletion(\n  chatCompletion: ModelAdapterDefinition['chatCompletion'],\n): ChatCompletionAdapter {\n  const buildChatCompletionParams =\n    chatCompletion?.buildChatCompletionParams ?? defaultChatCompletionParams;\n  const unsupportedUserConfig = chatCompletion?.unsupportedUserConfig ?? [];\n  const extractContentAndReasoning =\n    resolveExtractContentAndReasoning(chatCompletion);\n  const useReasoningAsContentFallback =\n    chatCompletion?.useReasoningAsContentFallback ?? false;\n  const replayRawAssistantMessage =\n    chatCompletion?.replayRawAssistantMessage ?? false;\n\n  return {\n    unsupportedUserConfig,\n    buildChatCompletionParams: (input) => {\n      const context = {\n        ...input,\n        userConfig: input.userConfig ?? {},\n        midsceneDefaults: midsceneChatCompletionDefaults,\n      };\n      const params = buildChatCompletionParams(context);\n      return {\n        ...params,\n        config: {\n          ...params.config,\n          temperature: getTemperatureWithSemanticRetry({\n            temperature: params.config.temperature,\n            userTemperature: input.userConfig?.temperature,\n            semanticRetryAttempt: input.semanticRetryAttempt,\n          }),\n        },\n      };\n    },\n    extractContentAndReasoning,\n    useReasoningAsContentFallback,\n    replayRawAssistantMessage,\n  };\n}\n"],"names":["defaultChatCompletionParams","midsceneDefaults","userConfig","midsceneChatCompletionDefaults","createDefaultExtractContentAndReasoning","reasoningContentKeys","message","messageRecord","rawReasoning","key","value","messageReasoning","resolveExtractContentAndReasoning","chatCompletion","messageExtraction","resolveChatCompletion","buildChatCompletionParams","unsupportedUserConfig","extractContentAndReasoning","useReasoningAsContentFallback","replayRawAssistantMessage","input","context","params","getTemperatureWithSemanticRetry"],"mappings":";AASA,MAAMA,8BAA8B,CAAC,EACnCC,gBAAgB,EAChBC,UAAU,EACgB,GAAM;QAChC,QAAQ;YACN,aAAaA,WAAW,WAAW,IAAID,iBAAiB,WAAW;QACrE;IACF;AAEA,MAAME,iCAAiE;IACrE,aAAa;AACf;AAEA,SAASC,wCACPC,oBAA8B;IAE9B,OAAO,CAACC;QACN,IAAI,CAACA,SACH,OAAO;YACL,SAAS;YACT,mBAAmB;QACrB;QAGF,MAAMC,gBAAgBD;QACtB,MAAME,eAAeH,qBAClB,GAAG,CAAC,CAACI,MAAQF,aAAa,CAACE,IAAI,EAC/B,IAAI,CAAC,CAACC,QAAU,AAAiB,YAAjB,OAAOA;QAC1B,MAAMC,mBACJ,AAAwB,YAAxB,OAAOH,eAA4BA,eAAe;QAEpD,OAAO;YACL,SAAS,AAA2B,YAA3B,OAAOF,QAAQ,OAAO,GAAgBA,QAAQ,OAAO,GAAG;YACjE,mBAAmBK;QACrB;IACF;AACF;AAEA,SAASC,kCACPC,cAAwD;IAExD,MAAMC,oBAAoBD,gBAAgB;IAC1C,IAAIC,mBAAmB,SAAS,UAC9B,OAAOA,kBAAkB,0BAA0B;IAGrD,OAAOV,wCACLU,mBAAmB,wBAAwB;QAAC;KAAoB;AAEpE;AAEO,SAASC,sBACdF,cAAwD;IAExD,MAAMG,4BACJH,gBAAgB,6BAA6Bb;IAC/C,MAAMiB,wBAAwBJ,gBAAgB,yBAAyB,EAAE;IACzE,MAAMK,6BACJN,kCAAkCC;IACpC,MAAMM,gCACJN,gBAAgB,iCAAiC;IACnD,MAAMO,4BACJP,gBAAgB,6BAA6B;IAE/C,OAAO;QACLI;QACA,2BAA2B,CAACI;YAC1B,MAAMC,UAAU;gBACd,GAAGD,KAAK;gBACR,YAAYA,MAAM,UAAU,IAAI,CAAC;gBACjC,kBAAkBlB;YACpB;YACA,MAAMoB,SAASP,0BAA0BM;YACzC,OAAO;gBACL,GAAGC,MAAM;gBACT,QAAQ;oBACN,GAAGA,OAAO,MAAM;oBAChB,aAAaC,gCAAgC;wBAC3C,aAAaD,OAAO,MAAM,CAAC,WAAW;wBACtC,iBAAiBF,MAAM,UAAU,EAAE;wBACnC,sBAAsBA,MAAM,oBAAoB;oBAClD;gBACF;YACF;QACF;QACAH;QACAC;QACAC;IACF;AACF"}

package/dist/es/ai-model/model-adapter/default-locate-protocol.mjs

@@ -1,6 +1,7 @@
 import { getPreferredLanguage } from "@midscene/shared/env";
 import { assertJsonObject } from "../shared/json.mjs";
 import { formatLocateExampleValue } from "../shared/model-locate-result/index.mjs";
+import { readLocateResultField } from "../shared/model-locate-result/result-field.mjs";
 const markdownCodeFence = '```';
 const defaultLocateSystemPromptIntroduction = `## Role:
 You are an AI assistant that helps identify UI elements.
@@ -89,7 +90,7 @@ const createParseRawResponse = (jsonParser, source, options)=>(content, promptSp
         });
         assertJsonObject(parsedResponse);
         const record = parsedResponse;
-        const target = void 0 !== record[promptSpec.resultKey] ? record[promptSpec.resultKey] : options.acceptBbox2dAlias && 'bbox' === promptSpec.resultKey ? record.bbox_2d : void 0;
+        const target = readLocateResultField(record, promptSpec);
         const error = 'string' == typeof record.error ? record.error : void 0;
         if (void 0 === target) throw new Error(`Missing required coordinate field "${promptSpec.resultKey}". Expected "${promptSpec.resultKey}": ${promptSpec.resultValueSchema}; use an empty array when no element is found.${error ? ` Model error: ${error}` : ''}`);
         if (Array.isArray(target) && 0 === target.length) return {
@@ -105,7 +106,7 @@ const createParseRawResponse = (jsonParser, source, options)=>(content, promptSp
                 error
             } : {}
         };
-        const rawReferences = record[`references_${promptSpec.resultKey}`];
+        const rawReferences = readLocateResultField(record, promptSpec, 'references_');
         const references = null == rawReferences ? void 0 : Array.isArray(rawReferences) ? rawReferences : [
             rawReferences
         ];
@@ -120,24 +121,22 @@ const createParseRawResponse = (jsonParser, source, options)=>(content, promptSp
             } : {}
         };
     };
-const createDefaultElementProtocol = ({ jsonParser }, options = {})=>({
+const createDefaultElementProtocol = ({ jsonParser })=>({
         systemPromptIntroduction: defaultLocateSystemPromptIntroduction,
         buildResponseInstructions,
         buildUserPrompt,
         expectedJsonObjectResponse: true,
         parseRawResponse: createParseRawResponse(jsonParser, 'locate', {
-            includeReferences: false,
-            acceptBbox2dAlias: options.acceptBbox2dAlias
+            includeReferences: false
         })
     });
-const createDefaultSearchAreaProtocol = ({ jsonParser }, options = {})=>({
+const createDefaultSearchAreaProtocol = ({ jsonParser })=>({
         systemPromptIntroduction: defaultSearchAreaSystemPromptIntroduction,
         buildResponseInstructions: buildSearchAreaResponseInstructions,
         buildUserPrompt: buildSearchAreaUserPrompt,
         expectedJsonObjectResponse: true,
         parseRawResponse: createParseRawResponse(jsonParser, 'section-locator', {
-            includeReferences: true,
-            acceptBbox2dAlias: options.acceptBbox2dAlias
+            includeReferences: true
         })
     });
 export { createDefaultElementProtocol, createDefaultSearchAreaProtocol };

package/dist/es/ai-model/model-adapter/default-locate-protocol.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/model-adapter/default-locate-protocol.mjs","sources":["../../../../src/ai-model/model-adapter/default-locate-protocol.ts"],"sourcesContent":["import { getPreferredLanguage } from '@midscene/shared/env';\nimport {\n  type JsonParser,\n  type JsonParserSource,\n  assertJsonObject,\n} from '../shared/json';\nimport {\n  type LocateResultPromptSpec,\n  formatLocateExampleValue,\n} from '../shared/model-locate-result';\nimport type {\n  ParsedLocateResponse,\n  StandardLocateProtocol,\n  StandardLocateProtocolContext,\n} from './locate-protocol';\n\nconst markdownCodeFence = '```';\n\nconst defaultLocateSystemPromptIntroduction = `## Role:\nYou are an AI assistant that helps identify UI elements.\n\n## Objective:\n- Identify elements in screenshots that match the user's description.\n- Provide the coordinates of the element that matches the user's description.`;\n\nconst defaultSearchAreaSystemPromptIntroduction = `## Role:\nYou are an AI assistant that helps identify UI elements.\n\n## Objective:\n- Find a section containing the target element\n- If the description mentions reference elements, also locate sections containing those references`;\n\nconst buildResponseInstructions = (promptSpec: LocateResultPromptSpec) => {\n  const preferredLanguage = getPreferredLanguage();\n  const resultKey = promptSpec.resultKey;\n  const exampleValueText = formatLocateExampleValue(\n    promptSpec.exampleValues[0],\n  );\n  const resultFieldDescription = `the ${promptSpec.resultNoun} of the element that matches the user's description`;\n  return `## Output Format:\n${markdownCodeFence}json\n{\n  \"${resultKey}\": ${promptSpec.resultValueSchema},  // ${promptSpec.resultValueDescription}\n  \"error\": string // optional\n}\n${markdownCodeFence}\n\nFields:\n* \\`${resultKey}\\` is ${resultFieldDescription}\n* \\`error\\` is an optional error message (if any)\n\nFor example, when an element is found:\n${markdownCodeFence}json\n{\n  \"${resultKey}\": ${exampleValueText}\n}\n${markdownCodeFence}\n\nWhen no element is found:\n${markdownCodeFence}json\n{\n  \"${resultKey}\": [],\n  \"error\": \"I can see ..., but {some element} is not found. Use ${preferredLanguage}.\"\n}\n${markdownCodeFence}\n`;\n};\n\nconst buildUserPrompt = (targetElementDescription: string) =>\n  `Find: ${targetElementDescription}`;\n\nconst buildSearchAreaResponseInstructions = (\n  promptSpec: LocateResultPromptSpec,\n) => {\n  const preferredLanguage = getPreferredLanguage();\n  const resultKey = promptSpec.resultKey;\n  const exampleValueText = formatLocateExampleValue(\n    promptSpec.exampleValues[0],\n  );\n  const resultJsonProperty = `\"${resultKey}\": ${promptSpec.resultValueSchema},  // ${promptSpec.resultValueDescription}`;\n  const resultFieldDescription = `${promptSpec.resultNoun} of the section containing the target element`;\n  const referenceFieldDescription = `Optional array of ${promptSpec.resultNounPlural} of reference elements`;\n  return `## Output Format:\n${markdownCodeFence}json\n{\n  ${resultJsonProperty}\n  \"references_${resultKey}\"?: [\n    ${promptSpec.resultValueSchema},\n    ...\n  ],\n  \"error\"?: string\n}\n${markdownCodeFence}\n\nFields:\n* \\`${resultKey}\\` - ${resultFieldDescription}\n* \\`references_${resultKey}\\` - ${referenceFieldDescription}\n* \\`error\\` - Optional error message if the section cannot be found. Use ${preferredLanguage}.\n\nExample:\nIf the description is \"delete button on the second row with title 'Peter'\", return:\n${markdownCodeFence}json\n{\n  \"${resultKey}\": ${exampleValueText},\n  \"references_${resultKey}\": [${exampleValueText}]\n}\n${markdownCodeFence}\n`;\n};\n\nconst buildSearchAreaUserPrompt = (sectionDescription: string) =>\n  `Find section containing: ${sectionDescription}`;\n\ntype ParseRawResponseOptions = {\n  includeReferences: boolean;\n  acceptBbox2dAlias?: boolean;\n};\n\nconst createParseRawResponse =\n  (\n    jsonParser: JsonParser,\n    source: JsonParserSource,\n    options: ParseRawResponseOptions,\n  ) =>\n  (\n    content: string,\n    promptSpec: LocateResultPromptSpec,\n  ): ParsedLocateResponse => {\n    const parsedResponse = jsonParser(content, {\n      source,\n    });\n    assertJsonObject(parsedResponse);\n    const record = parsedResponse;\n    const target =\n      record[promptSpec.resultKey] !== undefined\n        ? record[promptSpec.resultKey]\n        : options.acceptBbox2dAlias && promptSpec.resultKey === 'bbox'\n          ? record.bbox_2d\n          : undefined;\n    const error = typeof record.error === 'string' ? record.error : undefined;\n    if (target === undefined) {\n      throw new Error(\n        `Missing required coordinate field \"${promptSpec.resultKey}\". Expected \"${promptSpec.resultKey}\": ${promptSpec.resultValueSchema}; use an empty array when no element is found.${error ? ` Model error: ${error}` : ''}`,\n      );\n    }\n    if (Array.isArray(target) && target.length === 0) {\n      return { kind: 'not-found', ...(error ? { error } : {}) };\n    }\n\n    if (!options.includeReferences) {\n      return { kind: 'located', target, ...(error ? { error } : {}) };\n    }\n\n    const rawReferences = record[`references_${promptSpec.resultKey}`];\n    const references =\n      rawReferences === undefined || rawReferences === null\n        ? undefined\n        : Array.isArray(rawReferences)\n          ? rawReferences\n          : [rawReferences];\n    return {\n      kind: 'located',\n      target,\n      ...(references?.length ? { references } : {}),\n      ...(error ? { error } : {}),\n    };\n  };\n\ntype DefaultLocateProtocolOptions = {\n  acceptBbox2dAlias?: boolean;\n};\n\nexport const createDefaultElementProtocol = (\n  { jsonParser }: StandardLocateProtocolContext,\n  options: DefaultLocateProtocolOptions = {},\n): StandardLocateProtocol => ({\n  systemPromptIntroduction: defaultLocateSystemPromptIntroduction,\n  buildResponseInstructions,\n  buildUserPrompt,\n  expectedJsonObjectResponse: true,\n  parseRawResponse: createParseRawResponse(jsonParser, 'locate', {\n    includeReferences: false,\n    acceptBbox2dAlias: options.acceptBbox2dAlias,\n  }),\n});\n\nexport const createDefaultSearchAreaProtocol = (\n  { jsonParser }: StandardLocateProtocolContext,\n  options: DefaultLocateProtocolOptions = {},\n): StandardLocateProtocol => ({\n  systemPromptIntroduction: defaultSearchAreaSystemPromptIntroduction,\n  buildResponseInstructions: buildSearchAreaResponseInstructions,\n  buildUserPrompt: buildSearchAreaUserPrompt,\n  expectedJsonObjectResponse: true,\n  parseRawResponse: createParseRawResponse(jsonParser, 'section-locator', {\n    includeReferences: true,\n    acceptBbox2dAlias: options.acceptBbox2dAlias,\n  }),\n});\n"],"names":["markdownCodeFence","defaultLocateSystemPromptIntroduction","defaultSearchAreaSystemPromptIntroduction","buildResponseInstructions","promptSpec","preferredLanguage","getPreferredLanguage","resultKey","exampleValueText","formatLocateExampleValue","resultFieldDescription","buildUserPrompt","targetElementDescription","buildSearchAreaResponseInstructions","resultJsonProperty","referenceFieldDescription","buildSearchAreaUserPrompt","sectionDescription","createParseRawResponse","jsonParser","source","options","content","parsedResponse","assertJsonObject","record","target","undefined","error","Error","Array","rawReferences","references","createDefaultElementProtocol","createDefaultSearchAreaProtocol"],"mappings":";;;AAgBA,MAAMA,oBAAoB;AAE1B,MAAMC,wCAAwC,CAAC;;;;;6EAK8B,CAAC;AAE9E,MAAMC,4CAA4C,CAAC;;;;;kGAK+C,CAAC;AAEnG,MAAMC,4BAA4B,CAACC;IACjC,MAAMC,oBAAoBC;IAC1B,MAAMC,YAAYH,WAAW,SAAS;IACtC,MAAMI,mBAAmBC,yBACvBL,WAAW,aAAa,CAAC,EAAE;IAE7B,MAAMM,yBAAyB,CAAC,IAAI,EAAEN,WAAW,UAAU,CAAC,mDAAmD,CAAC;IAChH,OAAO,CAAC;AACV,EAAEJ,kBAAkB;;GAEjB,EAAEO,UAAU,GAAG,EAAEH,WAAW,iBAAiB,CAAC,MAAM,EAAEA,WAAW,sBAAsB,CAAC;;;AAG3F,EAAEJ,kBAAkB;;;IAGhB,EAAEO,UAAU,MAAM,EAAEG,uBAAuB;;;;AAI/C,EAAEV,kBAAkB;;GAEjB,EAAEO,UAAU,GAAG,EAAEC,iBAAiB;;AAErC,EAAER,kBAAkB;;;AAGpB,EAAEA,kBAAkB;;GAEjB,EAAEO,UAAU;gEACiD,EAAEF,kBAAkB;;AAEpF,EAAEL,kBAAkB;AACpB,CAAC;AACD;AAEA,MAAMW,kBAAkB,CAACC,2BACvB,CAAC,MAAM,EAAEA,0BAA0B;AAErC,MAAMC,sCAAsC,CAC1CT;IAEA,MAAMC,oBAAoBC;IAC1B,MAAMC,YAAYH,WAAW,SAAS;IACtC,MAAMI,mBAAmBC,yBACvBL,WAAW,aAAa,CAAC,EAAE;IAE7B,MAAMU,qBAAqB,CAAC,CAAC,EAAEP,UAAU,GAAG,EAAEH,WAAW,iBAAiB,CAAC,MAAM,EAAEA,WAAW,sBAAsB,EAAE;IACtH,MAAMM,yBAAyB,GAAGN,WAAW,UAAU,CAAC,6CAA6C,CAAC;IACtG,MAAMW,4BAA4B,CAAC,kBAAkB,EAAEX,WAAW,gBAAgB,CAAC,sBAAsB,CAAC;IAC1G,OAAO,CAAC;AACV,EAAEJ,kBAAkB;;EAElB,EAAEc,mBAAmB;cACT,EAAEP,UAAU;IACtB,EAAEH,WAAW,iBAAiB,CAAC;;;;;AAKnC,EAAEJ,kBAAkB;;;IAGhB,EAAEO,UAAU,KAAK,EAAEG,uBAAuB;eAC/B,EAAEH,UAAU,KAAK,EAAEQ,0BAA0B;yEACa,EAAEV,kBAAkB;;;;AAI7F,EAAEL,kBAAkB;;GAEjB,EAAEO,UAAU,GAAG,EAAEC,iBAAiB;cACvB,EAAED,UAAU,IAAI,EAAEC,iBAAiB;;AAEjD,EAAER,kBAAkB;AACpB,CAAC;AACD;AAEA,MAAMgB,4BAA4B,CAACC,qBACjC,CAAC,yBAAyB,EAAEA,oBAAoB;AAOlD,MAAMC,yBACJ,CACEC,YACAC,QACAC,UAEF,CACEC,SACAlB;QAEA,MAAMmB,iBAAiBJ,WAAWG,SAAS;YACzCF;QACF;QACAI,iBAAiBD;QACjB,MAAME,SAASF;QACf,MAAMG,SACJD,AAAiCE,WAAjCF,MAAM,CAACrB,WAAW,SAAS,CAAC,GACxBqB,MAAM,CAACrB,WAAW,SAAS,CAAC,GAC5BiB,QAAQ,iBAAiB,IAAIjB,AAAyB,WAAzBA,WAAW,SAAS,GAC/CqB,OAAO,OAAO,GACdE;QACR,MAAMC,QAAQ,AAAwB,YAAxB,OAAOH,OAAO,KAAK,GAAgBA,OAAO,KAAK,GAAGE;QAChE,IAAID,AAAWC,WAAXD,QACF,MAAM,IAAIG,MACR,CAAC,mCAAmC,EAAEzB,WAAW,SAAS,CAAC,aAAa,EAAEA,WAAW,SAAS,CAAC,GAAG,EAAEA,WAAW,iBAAiB,CAAC,8CAA8C,EAAEwB,QAAQ,CAAC,cAAc,EAAEA,OAAO,GAAG,IAAI;QAG5N,IAAIE,MAAM,OAAO,CAACJ,WAAWA,AAAkB,MAAlBA,OAAO,MAAM,EACxC,OAAO;YAAE,MAAM;YAAa,GAAIE,QAAQ;gBAAEA;YAAM,IAAI,CAAC,CAAC;QAAE;QAG1D,IAAI,CAACP,QAAQ,iBAAiB,EAC5B,OAAO;YAAE,MAAM;YAAWK;YAAQ,GAAIE,QAAQ;gBAAEA;YAAM,IAAI,CAAC,CAAC;QAAE;QAGhE,MAAMG,gBAAgBN,MAAM,CAAC,CAAC,WAAW,EAAErB,WAAW,SAAS,EAAE,CAAC;QAClE,MAAM4B,aACJD,QAAAA,gBACIJ,SACAG,MAAM,OAAO,CAACC,iBACZA,gBACA;YAACA;SAAc;QACvB,OAAO;YACL,MAAM;YACNL;YACA,GAAIM,YAAY,SAAS;gBAAEA;YAAW,IAAI,CAAC,CAAC;YAC5C,GAAIJ,QAAQ;gBAAEA;YAAM,IAAI,CAAC,CAAC;QAC5B;IACF;AAMK,MAAMK,+BAA+B,CAC1C,EAAEd,UAAU,EAAiC,EAC7CE,UAAwC,CAAC,CAAC,GACd;QAC5B,0BAA0BpB;QAC1BE;QACAQ;QACA,4BAA4B;QAC5B,kBAAkBO,uBAAuBC,YAAY,UAAU;YAC7D,mBAAmB;YACnB,mBAAmBE,QAAQ,iBAAiB;QAC9C;IACF;AAEO,MAAMa,kCAAkC,CAC7C,EAAEf,UAAU,EAAiC,EAC7CE,UAAwC,CAAC,CAAC,GACd;QAC5B,0BAA0BnB;QAC1B,2BAA2BW;QAC3B,iBAAiBG;QACjB,4BAA4B;QAC5B,kBAAkBE,uBAAuBC,YAAY,mBAAmB;YACtE,mBAAmB;YACnB,mBAAmBE,QAAQ,iBAAiB;QAC9C;IACF"}
+{"version":3,"file":"ai-model/model-adapter/default-locate-protocol.mjs","sources":["../../../../src/ai-model/model-adapter/default-locate-protocol.ts"],"sourcesContent":["import { getPreferredLanguage } from '@midscene/shared/env';\nimport {\n  type JsonParser,\n  type JsonParserSource,\n  assertJsonObject,\n} from '../shared/json';\nimport {\n  type LocateResultPromptSpec,\n  formatLocateExampleValue,\n} from '../shared/model-locate-result';\nimport { readLocateResultField } from '../shared/model-locate-result/result-field';\nimport type {\n  ParsedLocateResponse,\n  StandardLocateProtocol,\n  StandardLocateProtocolContext,\n} from './locate-protocol';\n\nconst markdownCodeFence = '```';\n\nconst defaultLocateSystemPromptIntroduction = `## Role:\nYou are an AI assistant that helps identify UI elements.\n\n## Objective:\n- Identify elements in screenshots that match the user's description.\n- Provide the coordinates of the element that matches the user's description.`;\n\nconst defaultSearchAreaSystemPromptIntroduction = `## Role:\nYou are an AI assistant that helps identify UI elements.\n\n## Objective:\n- Find a section containing the target element\n- If the description mentions reference elements, also locate sections containing those references`;\n\nconst buildResponseInstructions = (promptSpec: LocateResultPromptSpec) => {\n  const preferredLanguage = getPreferredLanguage();\n  const resultKey = promptSpec.resultKey;\n  const exampleValueText = formatLocateExampleValue(\n    promptSpec.exampleValues[0],\n  );\n  const resultFieldDescription = `the ${promptSpec.resultNoun} of the element that matches the user's description`;\n  return `## Output Format:\n${markdownCodeFence}json\n{\n  \"${resultKey}\": ${promptSpec.resultValueSchema},  // ${promptSpec.resultValueDescription}\n  \"error\": string // optional\n}\n${markdownCodeFence}\n\nFields:\n* \\`${resultKey}\\` is ${resultFieldDescription}\n* \\`error\\` is an optional error message (if any)\n\nFor example, when an element is found:\n${markdownCodeFence}json\n{\n  \"${resultKey}\": ${exampleValueText}\n}\n${markdownCodeFence}\n\nWhen no element is found:\n${markdownCodeFence}json\n{\n  \"${resultKey}\": [],\n  \"error\": \"I can see ..., but {some element} is not found. Use ${preferredLanguage}.\"\n}\n${markdownCodeFence}\n`;\n};\n\nconst buildUserPrompt = (targetElementDescription: string) =>\n  `Find: ${targetElementDescription}`;\n\nconst buildSearchAreaResponseInstructions = (\n  promptSpec: LocateResultPromptSpec,\n) => {\n  const preferredLanguage = getPreferredLanguage();\n  const resultKey = promptSpec.resultKey;\n  const exampleValueText = formatLocateExampleValue(\n    promptSpec.exampleValues[0],\n  );\n  const resultJsonProperty = `\"${resultKey}\": ${promptSpec.resultValueSchema},  // ${promptSpec.resultValueDescription}`;\n  const resultFieldDescription = `${promptSpec.resultNoun} of the section containing the target element`;\n  const referenceFieldDescription = `Optional array of ${promptSpec.resultNounPlural} of reference elements`;\n  return `## Output Format:\n${markdownCodeFence}json\n{\n  ${resultJsonProperty}\n  \"references_${resultKey}\"?: [\n    ${promptSpec.resultValueSchema},\n    ...\n  ],\n  \"error\"?: string\n}\n${markdownCodeFence}\n\nFields:\n* \\`${resultKey}\\` - ${resultFieldDescription}\n* \\`references_${resultKey}\\` - ${referenceFieldDescription}\n* \\`error\\` - Optional error message if the section cannot be found. Use ${preferredLanguage}.\n\nExample:\nIf the description is \"delete button on the second row with title 'Peter'\", return:\n${markdownCodeFence}json\n{\n  \"${resultKey}\": ${exampleValueText},\n  \"references_${resultKey}\": [${exampleValueText}]\n}\n${markdownCodeFence}\n`;\n};\n\nconst buildSearchAreaUserPrompt = (sectionDescription: string) =>\n  `Find section containing: ${sectionDescription}`;\n\ntype ParseRawResponseOptions = {\n  includeReferences: boolean;\n};\n\nconst createParseRawResponse =\n  (\n    jsonParser: JsonParser,\n    source: JsonParserSource,\n    options: ParseRawResponseOptions,\n  ) =>\n  (\n    content: string,\n    promptSpec: LocateResultPromptSpec,\n  ): ParsedLocateResponse => {\n    const parsedResponse = jsonParser(content, {\n      source,\n    });\n    assertJsonObject(parsedResponse);\n    const record = parsedResponse;\n    const target = readLocateResultField(record, promptSpec);\n    const error = typeof record.error === 'string' ? record.error : undefined;\n    if (target === undefined) {\n      throw new Error(\n        `Missing required coordinate field \"${promptSpec.resultKey}\". Expected \"${promptSpec.resultKey}\": ${promptSpec.resultValueSchema}; use an empty array when no element is found.${error ? ` Model error: ${error}` : ''}`,\n      );\n    }\n    if (Array.isArray(target) && target.length === 0) {\n      return { kind: 'not-found', ...(error ? { error } : {}) };\n    }\n\n    if (!options.includeReferences) {\n      return { kind: 'located', target, ...(error ? { error } : {}) };\n    }\n\n    const rawReferences = readLocateResultField(\n      record,\n      promptSpec,\n      'references_',\n    );\n    const references =\n      rawReferences === undefined || rawReferences === null\n        ? undefined\n        : Array.isArray(rawReferences)\n          ? rawReferences\n          : [rawReferences];\n    return {\n      kind: 'located',\n      target,\n      ...(references?.length ? { references } : {}),\n      ...(error ? { error } : {}),\n    };\n  };\n\nexport const createDefaultElementProtocol = ({\n  jsonParser,\n}: StandardLocateProtocolContext): StandardLocateProtocol => ({\n  systemPromptIntroduction: defaultLocateSystemPromptIntroduction,\n  buildResponseInstructions,\n  buildUserPrompt,\n  expectedJsonObjectResponse: true,\n  parseRawResponse: createParseRawResponse(jsonParser, 'locate', {\n    includeReferences: false,\n  }),\n});\n\nexport const createDefaultSearchAreaProtocol = ({\n  jsonParser,\n}: StandardLocateProtocolContext): StandardLocateProtocol => ({\n  systemPromptIntroduction: defaultSearchAreaSystemPromptIntroduction,\n  buildResponseInstructions: buildSearchAreaResponseInstructions,\n  buildUserPrompt: buildSearchAreaUserPrompt,\n  expectedJsonObjectResponse: true,\n  parseRawResponse: createParseRawResponse(jsonParser, 'section-locator', {\n    includeReferences: true,\n  }),\n});\n"],"names":["markdownCodeFence","defaultLocateSystemPromptIntroduction","defaultSearchAreaSystemPromptIntroduction","buildResponseInstructions","promptSpec","preferredLanguage","getPreferredLanguage","resultKey","exampleValueText","formatLocateExampleValue","resultFieldDescription","buildUserPrompt","targetElementDescription","buildSearchAreaResponseInstructions","resultJsonProperty","referenceFieldDescription","buildSearchAreaUserPrompt","sectionDescription","createParseRawResponse","jsonParser","source","options","content","parsedResponse","assertJsonObject","record","target","readLocateResultField","error","undefined","Error","Array","rawReferences","references","createDefaultElementProtocol","createDefaultSearchAreaProtocol"],"mappings":";;;;AAiBA,MAAMA,oBAAoB;AAE1B,MAAMC,wCAAwC,CAAC;;;;;6EAK8B,CAAC;AAE9E,MAAMC,4CAA4C,CAAC;;;;;kGAK+C,CAAC;AAEnG,MAAMC,4BAA4B,CAACC;IACjC,MAAMC,oBAAoBC;IAC1B,MAAMC,YAAYH,WAAW,SAAS;IACtC,MAAMI,mBAAmBC,yBACvBL,WAAW,aAAa,CAAC,EAAE;IAE7B,MAAMM,yBAAyB,CAAC,IAAI,EAAEN,WAAW,UAAU,CAAC,mDAAmD,CAAC;IAChH,OAAO,CAAC;AACV,EAAEJ,kBAAkB;;GAEjB,EAAEO,UAAU,GAAG,EAAEH,WAAW,iBAAiB,CAAC,MAAM,EAAEA,WAAW,sBAAsB,CAAC;;;AAG3F,EAAEJ,kBAAkB;;;IAGhB,EAAEO,UAAU,MAAM,EAAEG,uBAAuB;;;;AAI/C,EAAEV,kBAAkB;;GAEjB,EAAEO,UAAU,GAAG,EAAEC,iBAAiB;;AAErC,EAAER,kBAAkB;;;AAGpB,EAAEA,kBAAkB;;GAEjB,EAAEO,UAAU;gEACiD,EAAEF,kBAAkB;;AAEpF,EAAEL,kBAAkB;AACpB,CAAC;AACD;AAEA,MAAMW,kBAAkB,CAACC,2BACvB,CAAC,MAAM,EAAEA,0BAA0B;AAErC,MAAMC,sCAAsC,CAC1CT;IAEA,MAAMC,oBAAoBC;IAC1B,MAAMC,YAAYH,WAAW,SAAS;IACtC,MAAMI,mBAAmBC,yBACvBL,WAAW,aAAa,CAAC,EAAE;IAE7B,MAAMU,qBAAqB,CAAC,CAAC,EAAEP,UAAU,GAAG,EAAEH,WAAW,iBAAiB,CAAC,MAAM,EAAEA,WAAW,sBAAsB,EAAE;IACtH,MAAMM,yBAAyB,GAAGN,WAAW,UAAU,CAAC,6CAA6C,CAAC;IACtG,MAAMW,4BAA4B,CAAC,kBAAkB,EAAEX,WAAW,gBAAgB,CAAC,sBAAsB,CAAC;IAC1G,OAAO,CAAC;AACV,EAAEJ,kBAAkB;;EAElB,EAAEc,mBAAmB;cACT,EAAEP,UAAU;IACtB,EAAEH,WAAW,iBAAiB,CAAC;;;;;AAKnC,EAAEJ,kBAAkB;;;IAGhB,EAAEO,UAAU,KAAK,EAAEG,uBAAuB;eAC/B,EAAEH,UAAU,KAAK,EAAEQ,0BAA0B;yEACa,EAAEV,kBAAkB;;;;AAI7F,EAAEL,kBAAkB;;GAEjB,EAAEO,UAAU,GAAG,EAAEC,iBAAiB;cACvB,EAAED,UAAU,IAAI,EAAEC,iBAAiB;;AAEjD,EAAER,kBAAkB;AACpB,CAAC;AACD;AAEA,MAAMgB,4BAA4B,CAACC,qBACjC,CAAC,yBAAyB,EAAEA,oBAAoB;AAMlD,MAAMC,yBACJ,CACEC,YACAC,QACAC,UAEF,CACEC,SACAlB;QAEA,MAAMmB,iBAAiBJ,WAAWG,SAAS;YACzCF;QACF;QACAI,iBAAiBD;QACjB,MAAME,SAASF;QACf,MAAMG,SAASC,sBAAsBF,QAAQrB;QAC7C,MAAMwB,QAAQ,AAAwB,YAAxB,OAAOH,OAAO,KAAK,GAAgBA,OAAO,KAAK,GAAGI;QAChE,IAAIH,AAAWG,WAAXH,QACF,MAAM,IAAII,MACR,CAAC,mCAAmC,EAAE1B,WAAW,SAAS,CAAC,aAAa,EAAEA,WAAW,SAAS,CAAC,GAAG,EAAEA,WAAW,iBAAiB,CAAC,8CAA8C,EAAEwB,QAAQ,CAAC,cAAc,EAAEA,OAAO,GAAG,IAAI;QAG5N,IAAIG,MAAM,OAAO,CAACL,WAAWA,AAAkB,MAAlBA,OAAO,MAAM,EACxC,OAAO;YAAE,MAAM;YAAa,GAAIE,QAAQ;gBAAEA;YAAM,IAAI,CAAC,CAAC;QAAE;QAG1D,IAAI,CAACP,QAAQ,iBAAiB,EAC5B,OAAO;YAAE,MAAM;YAAWK;YAAQ,GAAIE,QAAQ;gBAAEA;YAAM,IAAI,CAAC,CAAC;QAAE;QAGhE,MAAMI,gBAAgBL,sBACpBF,QACArB,YACA;QAEF,MAAM6B,aACJD,QAAAA,gBACIH,SACAE,MAAM,OAAO,CAACC,iBACZA,gBACA;YAACA;SAAc;QACvB,OAAO;YACL,MAAM;YACNN;YACA,GAAIO,YAAY,SAAS;gBAAEA;YAAW,IAAI,CAAC,CAAC;YAC5C,GAAIL,QAAQ;gBAAEA;YAAM,IAAI,CAAC,CAAC;QAC5B;IACF;AAEK,MAAMM,+BAA+B,CAAC,EAC3Cf,UAAU,EACoB,GAA8B;QAC5D,0BAA0BlB;QAC1BE;QACAQ;QACA,4BAA4B;QAC5B,kBAAkBO,uBAAuBC,YAAY,UAAU;YAC7D,mBAAmB;QACrB;IACF;AAEO,MAAMgB,kCAAkC,CAAC,EAC9ChB,UAAU,EACoB,GAA8B;QAC5D,0BAA0BjB;QAC1B,2BAA2BW;QAC3B,iBAAiBG;QACjB,4BAA4B;QAC5B,kBAAkBE,uBAAuBC,YAAY,mBAAmB;YACtE,mBAAmB;QACrB;IACF"}

package/dist/es/ai-model/model-adapter/locate.mjs

@@ -9,7 +9,7 @@ const defaultLocateResultFormatDefinition = {
     }
 };
 const resolveLocateProtocolDefinition = (protocolDefinition, protocolContext)=>'function' == typeof protocolDefinition ? protocolDefinition(protocolContext) : protocolDefinition;
-function resolveLocate(locate, resolvedCustomPlanner, protocolContext, acceptBbox2dAlias = false) {
+function resolveLocate(locate, resolvedCustomPlanner, protocolContext) {
     if (locate?.kind === 'custom') {
         let locateFn = locate.locateFn;
         if (!locateFn) {
@@ -23,14 +23,10 @@ function resolveLocate(locate, resolvedCustomPlanner, protocolContext, acceptBbo
             locateFn
         };
     }
-    const elementProtocol = resolveLocateProtocolDefinition(locate?.element?.protocol ?? ((context)=>createDefaultElementProtocol(context, {
-            acceptBbox2dAlias
-        })), protocolContext);
+    const elementProtocol = resolveLocateProtocolDefinition(locate?.element?.protocol ?? createDefaultElementProtocol, protocolContext);
     const elementResultFormat = locate?.element?.resultFormat ?? defaultLocateResultFormatDefinition;
     const elementResultCodec = createLocateResultCodec(elementResultFormat);
-    const searchAreaProtocol = locate?.searchArea === false ? void 0 : resolveLocateProtocolDefinition(locate?.searchArea?.protocol ?? ((context)=>createDefaultSearchAreaProtocol(context, {
-            acceptBbox2dAlias
-        })), protocolContext);
+    const searchAreaProtocol = locate?.searchArea === false ? void 0 : resolveLocateProtocolDefinition(locate?.searchArea?.protocol ?? createDefaultSearchAreaProtocol, protocolContext);
     const searchAreaResultFormat = locate?.searchArea === false ? void 0 : locate?.searchArea?.resultFormat ?? elementResultFormat;
     const searchAreaResultCodec = searchAreaResultFormat ? createLocateResultCodec(searchAreaResultFormat) : void 0;
     return {

package/dist/es/ai-model/model-adapter/locate.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/model-adapter/locate.mjs","sources":["../../../../src/ai-model/model-adapter/locate.ts"],"sourcesContent":["import { createLocateResultCodec } from '../shared/model-locate-result/factory';\nimport type { LocateResultFormatDefinition } from '../shared/model-locate-result/types';\nimport { resolvePlanningTapLocator } from '../workflows/grounding/planning-action-locate';\nimport type { ResolvedCustomPlanningDefinition } from './custom-planning-types';\nimport {\n  createDefaultElementProtocol,\n  createDefaultSearchAreaProtocol,\n} from './default-locate-protocol';\nimport type {\n  StandardLocateProtocolContext,\n  StandardLocateProtocolDefinition,\n} from './locate-protocol';\nimport type { LocateAdapter, ModelAdapterDefinition } from './types';\n\nconst defaultLocateResultFormatDefinition: LocateResultFormatDefinition = {\n  coordinates: { shape: 'bbox', order: 'xy', normalizedBy: 1000 },\n};\n\nconst resolveLocateProtocolDefinition = (\n  protocolDefinition: StandardLocateProtocolDefinition,\n  protocolContext: StandardLocateProtocolContext,\n) =>\n  typeof protocolDefinition === 'function'\n    ? protocolDefinition(protocolContext)\n    : protocolDefinition;\n\nexport function resolveLocate(\n  locate: ModelAdapterDefinition['locate'],\n  resolvedCustomPlanner: ResolvedCustomPlanningDefinition | undefined,\n  protocolContext: StandardLocateProtocolContext,\n  acceptBbox2dAlias = false,\n): LocateAdapter {\n  if (locate?.kind === 'custom') {\n    let locateFn = locate.locateFn;\n\n    if (!locateFn) {\n      const planningTapLocator = locate.planningTapLocator;\n\n      if (!planningTapLocator) {\n        throw new Error(\n          'Custom locate definition requires either locateFn or planningTapLocator',\n        );\n      }\n\n      if (!resolvedCustomPlanner) {\n        throw new Error(\n          'Custom planning tap locator requires a custom planning planner definition',\n        );\n      }\n      locateFn = resolvePlanningTapLocator(\n        planningTapLocator,\n        resolvedCustomPlanner,\n      );\n    }\n\n    return {\n      kind: 'custom',\n      locateFn,\n    };\n  }\n\n  const elementProtocol = resolveLocateProtocolDefinition(\n    locate?.element?.protocol ??\n      ((context) =>\n        createDefaultElementProtocol(context, { acceptBbox2dAlias })),\n    protocolContext,\n  );\n  const elementResultFormat =\n    locate?.element?.resultFormat ?? defaultLocateResultFormatDefinition;\n  const elementResultCodec = createLocateResultCodec(elementResultFormat);\n  const searchAreaProtocol =\n    locate?.searchArea === false\n      ? undefined\n      : resolveLocateProtocolDefinition(\n          locate?.searchArea?.protocol ??\n            ((context) =>\n              createDefaultSearchAreaProtocol(context, {\n                acceptBbox2dAlias,\n              })),\n          protocolContext,\n        );\n  const searchAreaResultFormat =\n    locate?.searchArea === false\n      ? undefined\n      : (locate?.searchArea?.resultFormat ?? elementResultFormat);\n  const searchAreaResultCodec = searchAreaResultFormat\n    ? createLocateResultCodec(searchAreaResultFormat)\n    : undefined;\n\n  return {\n    kind: 'standard',\n    userMessageContentOrder: locate?.userMessageContentOrder ?? 'image-first',\n    element: {\n      protocol: elementProtocol,\n      resultCodec: elementResultCodec,\n    },\n    ...(searchAreaProtocol && searchAreaResultCodec\n      ? {\n          searchArea: {\n            protocol: searchAreaProtocol,\n            resultCodec: searchAreaResultCodec,\n          },\n        }\n      : {}),\n  };\n}\n"],"names":["defaultLocateResultFormatDefinition","resolveLocateProtocolDefinition","protocolDefinition","protocolContext","resolveLocate","locate","resolvedCustomPlanner","acceptBbox2dAlias","locateFn","planningTapLocator","Error","resolvePlanningTapLocator","elementProtocol","context","createDefaultElementProtocol","elementResultFormat","elementResultCodec","createLocateResultCodec","searchAreaProtocol","undefined","createDefaultSearchAreaProtocol","searchAreaResultFormat","searchAreaResultCodec"],"mappings":";;;AAcA,MAAMA,sCAAoE;IACxE,aAAa;QAAE,OAAO;QAAQ,OAAO;QAAM,cAAc;IAAK;AAChE;AAEA,MAAMC,kCAAkC,CACtCC,oBACAC,kBAEA,AAA8B,cAA9B,OAAOD,qBACHA,mBAAmBC,mBACnBD;AAEC,SAASE,cACdC,MAAwC,EACxCC,qBAAmE,EACnEH,eAA8C,EAC9CI,oBAAoB,KAAK;IAEzB,IAAIF,QAAQ,SAAS,UAAU;QAC7B,IAAIG,WAAWH,OAAO,QAAQ;QAE9B,IAAI,CAACG,UAAU;YACb,MAAMC,qBAAqBJ,OAAO,kBAAkB;YAEpD,IAAI,CAACI,oBACH,MAAM,IAAIC,MACR;YAIJ,IAAI,CAACJ,uBACH,MAAM,IAAII,MACR;YAGJF,WAAWG,0BACTF,oBACAH;QAEJ;QAEA,OAAO;YACL,MAAM;YACNE;QACF;IACF;IAEA,MAAMI,kBAAkBX,gCACtBI,QAAQ,SAAS,YACb,EAAAQ,UACAC,6BAA6BD,SAAS;YAAEN;QAAkB,EAAC,GAC/DJ;IAEF,MAAMY,sBACJV,QAAQ,SAAS,gBAAgBL;IACnC,MAAMgB,qBAAqBC,wBAAwBF;IACnD,MAAMG,qBACJb,QAAQ,eAAe,QACnBc,SACAlB,gCACEI,QAAQ,YAAY,YAChB,EAAAQ,UACAO,gCAAgCP,SAAS;YACvCN;QACF,EAAC,GACLJ;IAER,MAAMkB,yBACJhB,QAAQ,eAAe,QACnBc,SACCd,QAAQ,YAAY,gBAAgBU;IAC3C,MAAMO,wBAAwBD,yBAC1BJ,wBAAwBI,0BACxBF;IAEJ,OAAO;QACL,MAAM;QACN,yBAAyBd,QAAQ,2BAA2B;QAC5D,SAAS;YACP,UAAUO;YACV,aAAaI;QACf;QACA,GAAIE,sBAAsBI,wBACtB;YACE,YAAY;gBACV,UAAUJ;gBACV,aAAaI;YACf;QACF,IACA,CAAC,CAAC;IACR;AACF"}
+{"version":3,"file":"ai-model/model-adapter/locate.mjs","sources":["../../../../src/ai-model/model-adapter/locate.ts"],"sourcesContent":["import { createLocateResultCodec } from '../shared/model-locate-result/factory';\nimport type { LocateResultFormatDefinition } from '../shared/model-locate-result/types';\nimport { resolvePlanningTapLocator } from '../workflows/grounding/planning-action-locate';\nimport type { ResolvedCustomPlanningDefinition } from './custom-planning-types';\nimport {\n  createDefaultElementProtocol,\n  createDefaultSearchAreaProtocol,\n} from './default-locate-protocol';\nimport type {\n  StandardLocateProtocolContext,\n  StandardLocateProtocolDefinition,\n} from './locate-protocol';\nimport type { LocateAdapter, ModelAdapterDefinition } from './types';\n\nconst defaultLocateResultFormatDefinition: LocateResultFormatDefinition = {\n  coordinates: { shape: 'bbox', order: 'xy', normalizedBy: 1000 },\n};\n\nconst resolveLocateProtocolDefinition = (\n  protocolDefinition: StandardLocateProtocolDefinition,\n  protocolContext: StandardLocateProtocolContext,\n) =>\n  typeof protocolDefinition === 'function'\n    ? protocolDefinition(protocolContext)\n    : protocolDefinition;\n\nexport function resolveLocate(\n  locate: ModelAdapterDefinition['locate'],\n  resolvedCustomPlanner: ResolvedCustomPlanningDefinition | undefined,\n  protocolContext: StandardLocateProtocolContext,\n): LocateAdapter {\n  if (locate?.kind === 'custom') {\n    let locateFn = locate.locateFn;\n\n    if (!locateFn) {\n      const planningTapLocator = locate.planningTapLocator;\n\n      if (!planningTapLocator) {\n        throw new Error(\n          'Custom locate definition requires either locateFn or planningTapLocator',\n        );\n      }\n\n      if (!resolvedCustomPlanner) {\n        throw new Error(\n          'Custom planning tap locator requires a custom planning planner definition',\n        );\n      }\n      locateFn = resolvePlanningTapLocator(\n        planningTapLocator,\n        resolvedCustomPlanner,\n      );\n    }\n\n    return {\n      kind: 'custom',\n      locateFn,\n    };\n  }\n\n  const elementProtocol = resolveLocateProtocolDefinition(\n    locate?.element?.protocol ?? createDefaultElementProtocol,\n    protocolContext,\n  );\n  const elementResultFormat =\n    locate?.element?.resultFormat ?? defaultLocateResultFormatDefinition;\n  const elementResultCodec = createLocateResultCodec(elementResultFormat);\n  const searchAreaProtocol =\n    locate?.searchArea === false\n      ? undefined\n      : resolveLocateProtocolDefinition(\n          locate?.searchArea?.protocol ?? createDefaultSearchAreaProtocol,\n          protocolContext,\n        );\n  const searchAreaResultFormat =\n    locate?.searchArea === false\n      ? undefined\n      : (locate?.searchArea?.resultFormat ?? elementResultFormat);\n  const searchAreaResultCodec = searchAreaResultFormat\n    ? createLocateResultCodec(searchAreaResultFormat)\n    : undefined;\n\n  return {\n    kind: 'standard',\n    userMessageContentOrder: locate?.userMessageContentOrder ?? 'image-first',\n    element: {\n      protocol: elementProtocol,\n      resultCodec: elementResultCodec,\n    },\n    ...(searchAreaProtocol && searchAreaResultCodec\n      ? {\n          searchArea: {\n            protocol: searchAreaProtocol,\n            resultCodec: searchAreaResultCodec,\n          },\n        }\n      : {}),\n  };\n}\n"],"names":["defaultLocateResultFormatDefinition","resolveLocateProtocolDefinition","protocolDefinition","protocolContext","resolveLocate","locate","resolvedCustomPlanner","locateFn","planningTapLocator","Error","resolvePlanningTapLocator","elementProtocol","createDefaultElementProtocol","elementResultFormat","elementResultCodec","createLocateResultCodec","searchAreaProtocol","undefined","createDefaultSearchAreaProtocol","searchAreaResultFormat","searchAreaResultCodec"],"mappings":";;;AAcA,MAAMA,sCAAoE;IACxE,aAAa;QAAE,OAAO;QAAQ,OAAO;QAAM,cAAc;IAAK;AAChE;AAEA,MAAMC,kCAAkC,CACtCC,oBACAC,kBAEA,AAA8B,cAA9B,OAAOD,qBACHA,mBAAmBC,mBACnBD;AAEC,SAASE,cACdC,MAAwC,EACxCC,qBAAmE,EACnEH,eAA8C;IAE9C,IAAIE,QAAQ,SAAS,UAAU;QAC7B,IAAIE,WAAWF,OAAO,QAAQ;QAE9B,IAAI,CAACE,UAAU;YACb,MAAMC,qBAAqBH,OAAO,kBAAkB;YAEpD,IAAI,CAACG,oBACH,MAAM,IAAIC,MACR;YAIJ,IAAI,CAACH,uBACH,MAAM,IAAIG,MACR;YAGJF,WAAWG,0BACTF,oBACAF;QAEJ;QAEA,OAAO;YACL,MAAM;YACNC;QACF;IACF;IAEA,MAAMI,kBAAkBV,gCACtBI,QAAQ,SAAS,YAAYO,8BAC7BT;IAEF,MAAMU,sBACJR,QAAQ,SAAS,gBAAgBL;IACnC,MAAMc,qBAAqBC,wBAAwBF;IACnD,MAAMG,qBACJX,QAAQ,eAAe,QACnBY,SACAhB,gCACEI,QAAQ,YAAY,YAAYa,iCAChCf;IAER,MAAMgB,yBACJd,QAAQ,eAAe,QACnBY,SACCZ,QAAQ,YAAY,gBAAgBQ;IAC3C,MAAMO,wBAAwBD,yBAC1BJ,wBAAwBI,0BACxBF;IAEJ,OAAO;QACL,MAAM;QACN,yBAAyBZ,QAAQ,2BAA2B;QAC5D,SAAS;YACP,UAAUM;YACV,aAAaG;QACf;QACA,GAAIE,sBAAsBI,wBACtB;YACE,YAAY;gBACV,UAAUJ;gBACV,aAAaI;YACf;QACF,IACA,CAAC,CAAC;IACR;AACF"}

package/dist/es/ai-model/model-adapter/resolve.mjs

@@ -4,6 +4,7 @@ import { buildDefaultCodexAppServerParams } from "./codex-app-server.mjs";
 import { resolveInsight } from "./insight.mjs";
 import { resolveLocate } from "./locate.mjs";
 import { resolveCustomPlanningDefinition, resolvePlanning } from "./planning.mjs";
+import { resolveResponses } from "./responses.mjs";
 function _define_property(obj, key, value) {
     if (key in obj) Object.defineProperty(obj, key, {
         value: value,
@@ -14,6 +15,7 @@ function _define_property(obj, key, value) {
     else obj[key] = value;
     return obj;
 }
+const defaultImageDetail = ({ imageDetail })=>imageDetail ?? 'high';
 function resolveJsonParser(jsonParser) {
     if (!jsonParser || 'lenient-json' === jsonParser) return parseModelResponseJson;
     if ('function' == typeof jsonParser) return jsonParser;
@@ -26,18 +28,24 @@ function resolveImagePreprocess(imagePreprocess) {
 }
 class ResolvedModelAdapter {
     constructor(config, modelFamily){
+        _define_property(this, "supportedProtocols", void 0);
         _define_property(this, "jsonParser", void 0);
         _define_property(this, "chatCompletion", void 0);
+        _define_property(this, "resolveImageDetail", void 0);
+        _define_property(this, "responses", void 0);
         _define_property(this, "buildCodexAppServerParams", void 0);
-        _define_property(this, "acceptBbox2dAlias", void 0);
         _define_property(this, "imagePreprocess", void 0);
         _define_property(this, "insight", void 0);
         _define_property(this, "planning", void 0);
         _define_property(this, "locate", void 0);
+        this.supportedProtocols = config.supportedProtocols ?? [
+            'openai-chat'
+        ];
         this.jsonParser = resolveJsonParser(config.jsonParser);
         this.chatCompletion = resolveChatCompletion(config.chatCompletion);
+        this.resolveImageDetail = config.resolveImageDetail ?? defaultImageDetail;
+        this.responses = resolveResponses(config.responses);
         this.buildCodexAppServerParams = config.buildCodexAppServerParams ?? buildDefaultCodexAppServerParams;
-        this.acceptBbox2dAlias = config.acceptBbox2dAlias ?? false;
         this.imagePreprocess = resolveImagePreprocess(config.imagePreprocess);
         this.insight = resolveInsight(config.insight, {
             jsonParser: this.jsonParser
@@ -46,7 +54,7 @@ class ResolvedModelAdapter {
         const resolvedCustomPlanner = customPlanner ? resolveCustomPlanningDefinition(customPlanner) : void 0;
         this.locate = resolveLocate(config.locate, resolvedCustomPlanner, {
             jsonParser: this.jsonParser
-        }, this.acceptBbox2dAlias);
+        });
         this.planning = resolvePlanning(config.planning, resolvedCustomPlanner, {
             jsonParser: this.jsonParser
         }, 'standard' === this.locate.kind ? this.locate.element.resultCodec : void 0);

package/dist/es/ai-model/model-adapter/resolve.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/model-adapter/resolve.mjs","sources":["../../../../src/ai-model/model-adapter/resolve.ts"],"sourcesContent":["import { parseModelResponseJson } from '../shared/json';\nimport { resolveChatCompletion } from './chat-completion';\nimport { buildDefaultCodexAppServerParams } from './codex-app-server';\nimport { resolveInsight } from './insight';\nimport type { InsightAdapter } from './insight-protocol';\nimport { resolveLocate } from './locate';\nimport { resolveCustomPlanningDefinition, resolvePlanning } from './planning';\nimport type {\n  BuildCodexAppServerParams,\n  ChatCompletionAdapter,\n  ImagePreprocessPolicy,\n  JsonParser,\n  LocateAdapter,\n  ModelAdapter,\n  ModelAdapterDefinition,\n  PlanningAdapter,\n} from './types';\n\nfunction resolveJsonParser(\n  jsonParser: ModelAdapterDefinition['jsonParser'],\n): JsonParser {\n  if (!jsonParser || jsonParser === 'lenient-json') {\n    return parseModelResponseJson;\n  }\n\n  if (typeof jsonParser === 'function') {\n    return jsonParser;\n  }\n\n  throw new Error(`Unknown json parser preset: ${jsonParser}`);\n}\n\nfunction resolveImagePreprocess(\n  imagePreprocess: ModelAdapterDefinition['imagePreprocess'],\n): ImagePreprocessPolicy {\n  return {\n    padBlockSize: imagePreprocess?.padBlockSize,\n  };\n}\n\nexport class ResolvedModelAdapter implements ModelAdapter {\n  readonly jsonParser: JsonParser;\n  readonly chatCompletion: ChatCompletionAdapter;\n  readonly buildCodexAppServerParams: BuildCodexAppServerParams;\n  readonly acceptBbox2dAlias: boolean;\n  readonly imagePreprocess: ImagePreprocessPolicy;\n  readonly insight: InsightAdapter;\n  readonly planning: PlanningAdapter;\n  readonly locate: LocateAdapter;\n\n  constructor(config: ModelAdapterDefinition, modelFamily: string) {\n    this.jsonParser = resolveJsonParser(config.jsonParser);\n    this.chatCompletion = resolveChatCompletion(config.chatCompletion);\n    this.buildCodexAppServerParams =\n      config.buildCodexAppServerParams ?? buildDefaultCodexAppServerParams;\n    this.acceptBbox2dAlias = config.acceptBbox2dAlias ?? false;\n    this.imagePreprocess = resolveImagePreprocess(config.imagePreprocess);\n    this.insight = resolveInsight(config.insight, {\n      jsonParser: this.jsonParser,\n    });\n    const customPlanner =\n      config.planning?.kind === 'custom' ? config.planning.planner : undefined;\n    const resolvedCustomPlanner = customPlanner\n      ? resolveCustomPlanningDefinition(customPlanner)\n      : undefined;\n    this.locate = resolveLocate(\n      config.locate,\n      resolvedCustomPlanner,\n      {\n        jsonParser: this.jsonParser,\n      },\n      this.acceptBbox2dAlias,\n    );\n    this.planning = resolvePlanning(\n      config.planning,\n      resolvedCustomPlanner,\n      { jsonParser: this.jsonParser },\n      this.locate.kind === 'standard'\n        ? this.locate.element.resultCodec\n        : undefined,\n    );\n  }\n}\n"],"names":["resolveJsonParser","jsonParser","parseModelResponseJson","Error","resolveImagePreprocess","imagePreprocess","ResolvedModelAdapter","config","modelFamily","resolveChatCompletion","buildDefaultCodexAppServerParams","resolveInsight","customPlanner","undefined","resolvedCustomPlanner","resolveCustomPlanningDefinition","resolveLocate","resolvePlanning"],"mappings":";;;;;;;;;;;;;;;;AAkBA,SAASA,kBACPC,UAAgD;IAEhD,IAAI,CAACA,cAAcA,AAAe,mBAAfA,YACjB,OAAOC;IAGT,IAAI,AAAsB,cAAtB,OAAOD,YACT,OAAOA;IAGT,MAAM,IAAIE,MAAM,CAAC,4BAA4B,EAAEF,YAAY;AAC7D;AAEA,SAASG,uBACPC,eAA0D;IAE1D,OAAO;QACL,cAAcA,iBAAiB;IACjC;AACF;AAEO,MAAMC;IAUX,YAAYC,MAA8B,EAAEC,WAAmB,CAAE;QATjE,uBAAS,cAAT;QACA,uBAAS,kBAAT;QACA,uBAAS,6BAAT;QACA,uBAAS,qBAAT;QACA,uBAAS,mBAAT;QACA,uBAAS,WAAT;QACA,uBAAS,YAAT;QACA,uBAAS,UAAT;QAGE,IAAI,CAAC,UAAU,GAAGR,kBAAkBO,OAAO,UAAU;QACrD,IAAI,CAAC,cAAc,GAAGE,sBAAsBF,OAAO,cAAc;QACjE,IAAI,CAAC,yBAAyB,GAC5BA,OAAO,yBAAyB,IAAIG;QACtC,IAAI,CAAC,iBAAiB,GAAGH,OAAO,iBAAiB,IAAI;QACrD,IAAI,CAAC,eAAe,GAAGH,uBAAuBG,OAAO,eAAe;QACpE,IAAI,CAAC,OAAO,GAAGI,eAAeJ,OAAO,OAAO,EAAE;YAC5C,YAAY,IAAI,CAAC,UAAU;QAC7B;QACA,MAAMK,gBACJL,OAAO,QAAQ,EAAE,SAAS,WAAWA,OAAO,QAAQ,CAAC,OAAO,GAAGM;QACjE,MAAMC,wBAAwBF,gBAC1BG,gCAAgCH,iBAChCC;QACJ,IAAI,CAAC,MAAM,GAAGG,cACZT,OAAO,MAAM,EACbO,uBACA;YACE,YAAY,IAAI,CAAC,UAAU;QAC7B,GACA,IAAI,CAAC,iBAAiB;QAExB,IAAI,CAAC,QAAQ,GAAGG,gBACdV,OAAO,QAAQ,EACfO,uBACA;YAAE,YAAY,IAAI,CAAC,UAAU;QAAC,GAC9B,AAAqB,eAArB,IAAI,CAAC,MAAM,CAAC,IAAI,GACZ,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,WAAW,GAC/BD;IAER;AACF"}
+{"version":3,"file":"ai-model/model-adapter/resolve.mjs","sources":["../../../../src/ai-model/model-adapter/resolve.ts"],"sourcesContent":["import { parseModelResponseJson } from '../shared/json';\nimport { resolveChatCompletion } from './chat-completion';\nimport { buildDefaultCodexAppServerParams } from './codex-app-server';\nimport { resolveInsight } from './insight';\nimport type { InsightAdapter } from './insight-protocol';\nimport { resolveLocate } from './locate';\nimport { resolveCustomPlanningDefinition, resolvePlanning } from './planning';\nimport { resolveResponses } from './responses';\nimport type {\n  BuildCodexAppServerParams,\n  ChatCompletionAdapter,\n  ImagePreprocessPolicy,\n  JsonParser,\n  LocateAdapter,\n  ModelAdapter,\n  ModelAdapterDefinition,\n  PlanningAdapter,\n  ResolveImageDetail,\n  ResponsesAdapter,\n} from './types';\n\nconst defaultImageDetail: ResolveImageDetail = ({ imageDetail }) =>\n  imageDetail ?? 'high';\n\nfunction resolveJsonParser(\n  jsonParser: ModelAdapterDefinition['jsonParser'],\n): JsonParser {\n  if (!jsonParser || jsonParser === 'lenient-json') {\n    return parseModelResponseJson;\n  }\n\n  if (typeof jsonParser === 'function') {\n    return jsonParser;\n  }\n\n  throw new Error(`Unknown json parser preset: ${jsonParser}`);\n}\n\nfunction resolveImagePreprocess(\n  imagePreprocess: ModelAdapterDefinition['imagePreprocess'],\n): ImagePreprocessPolicy {\n  return {\n    padBlockSize: imagePreprocess?.padBlockSize,\n  };\n}\n\nexport class ResolvedModelAdapter implements ModelAdapter {\n  readonly supportedProtocols: ModelAdapter['supportedProtocols'];\n  readonly jsonParser: JsonParser;\n  readonly chatCompletion: ChatCompletionAdapter;\n  readonly resolveImageDetail: ResolveImageDetail;\n  readonly responses: ResponsesAdapter;\n  readonly buildCodexAppServerParams: BuildCodexAppServerParams;\n  readonly imagePreprocess: ImagePreprocessPolicy;\n  readonly insight: InsightAdapter;\n  readonly planning: PlanningAdapter;\n  readonly locate: LocateAdapter;\n\n  constructor(config: ModelAdapterDefinition, modelFamily: string) {\n    this.supportedProtocols = config.supportedProtocols ?? ['openai-chat'];\n    this.jsonParser = resolveJsonParser(config.jsonParser);\n    this.chatCompletion = resolveChatCompletion(config.chatCompletion);\n    this.resolveImageDetail = config.resolveImageDetail ?? defaultImageDetail;\n    this.responses = resolveResponses(config.responses);\n    this.buildCodexAppServerParams =\n      config.buildCodexAppServerParams ?? buildDefaultCodexAppServerParams;\n    this.imagePreprocess = resolveImagePreprocess(config.imagePreprocess);\n    this.insight = resolveInsight(config.insight, {\n      jsonParser: this.jsonParser,\n    });\n    const customPlanner =\n      config.planning?.kind === 'custom' ? config.planning.planner : undefined;\n    const resolvedCustomPlanner = customPlanner\n      ? resolveCustomPlanningDefinition(customPlanner)\n      : undefined;\n    this.locate = resolveLocate(config.locate, resolvedCustomPlanner, {\n      jsonParser: this.jsonParser,\n    });\n    this.planning = resolvePlanning(\n      config.planning,\n      resolvedCustomPlanner,\n      { jsonParser: this.jsonParser },\n      this.locate.kind === 'standard'\n        ? this.locate.element.resultCodec\n        : undefined,\n    );\n  }\n}\n"],"names":["defaultImageDetail","imageDetail","resolveJsonParser","jsonParser","parseModelResponseJson","Error","resolveImagePreprocess","imagePreprocess","ResolvedModelAdapter","config","modelFamily","resolveChatCompletion","resolveResponses","buildDefaultCodexAppServerParams","resolveInsight","customPlanner","undefined","resolvedCustomPlanner","resolveCustomPlanningDefinition","resolveLocate","resolvePlanning"],"mappings":";;;;;;;;;;;;;;;;;AAqBA,MAAMA,qBAAyC,CAAC,EAAEC,WAAW,EAAE,GAC7DA,eAAe;AAEjB,SAASC,kBACPC,UAAgD;IAEhD,IAAI,CAACA,cAAcA,AAAe,mBAAfA,YACjB,OAAOC;IAGT,IAAI,AAAsB,cAAtB,OAAOD,YACT,OAAOA;IAGT,MAAM,IAAIE,MAAM,CAAC,4BAA4B,EAAEF,YAAY;AAC7D;AAEA,SAASG,uBACPC,eAA0D;IAE1D,OAAO;QACL,cAAcA,iBAAiB;IACjC;AACF;AAEO,MAAMC;IAYX,YAAYC,MAA8B,EAAEC,WAAmB,CAAE;QAXjE,uBAAS,sBAAT;QACA,uBAAS,cAAT;QACA,uBAAS,kBAAT;QACA,uBAAS,sBAAT;QACA,uBAAS,aAAT;QACA,uBAAS,6BAAT;QACA,uBAAS,mBAAT;QACA,uBAAS,WAAT;QACA,uBAAS,YAAT;QACA,uBAAS,UAAT;QAGE,IAAI,CAAC,kBAAkB,GAAGD,OAAO,kBAAkB,IAAI;YAAC;SAAc;QACtE,IAAI,CAAC,UAAU,GAAGP,kBAAkBO,OAAO,UAAU;QACrD,IAAI,CAAC,cAAc,GAAGE,sBAAsBF,OAAO,cAAc;QACjE,IAAI,CAAC,kBAAkB,GAAGA,OAAO,kBAAkB,IAAIT;QACvD,IAAI,CAAC,SAAS,GAAGY,iBAAiBH,OAAO,SAAS;QAClD,IAAI,CAAC,yBAAyB,GAC5BA,OAAO,yBAAyB,IAAII;QACtC,IAAI,CAAC,eAAe,GAAGP,uBAAuBG,OAAO,eAAe;QACpE,IAAI,CAAC,OAAO,GAAGK,eAAeL,OAAO,OAAO,EAAE;YAC5C,YAAY,IAAI,CAAC,UAAU;QAC7B;QACA,MAAMM,gBACJN,OAAO,QAAQ,EAAE,SAAS,WAAWA,OAAO,QAAQ,CAAC,OAAO,GAAGO;QACjE,MAAMC,wBAAwBF,gBAC1BG,gCAAgCH,iBAChCC;QACJ,IAAI,CAAC,MAAM,GAAGG,cAAcV,OAAO,MAAM,EAAEQ,uBAAuB;YAChE,YAAY,IAAI,CAAC,UAAU;QAC7B;QACA,IAAI,CAAC,QAAQ,GAAGG,gBACdX,OAAO,QAAQ,EACfQ,uBACA;YAAE,YAAY,IAAI,CAAC,UAAU;QAAC,GAC9B,AAAqB,eAArB,IAAI,CAAC,MAAM,CAAC,IAAI,GACZ,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,WAAW,GAC/BD;IAER;AACF"}

package/dist/es/ai-model/model-adapter/responses.mjs

@@ -0,0 +1,38 @@
+import { getTemperatureWithSemanticRetry } from "./temperature-with-semantic-retry.mjs";
+const midsceneResponsesDefaults = {
+    temperature: 0
+};
+const buildDefaultResponsesParams = ({ midsceneDefaults, userConfig })=>({
+        config: {
+            temperature: userConfig.temperature ?? midsceneDefaults.temperature
+        }
+    });
+function resolveResponses(responses) {
+    const buildResponsesParams = responses?.buildResponsesParams ?? buildDefaultResponsesParams;
+    return {
+        replayRawAssistantOutput: responses?.replayRawAssistantOutput ?? false,
+        unsupportedUserConfig: responses?.unsupportedUserConfig ?? [],
+        buildResponsesParams: (input)=>{
+            const context = {
+                ...input,
+                userConfig: input.userConfig ?? {},
+                midsceneDefaults: midsceneResponsesDefaults
+            };
+            const params = buildResponsesParams(context);
+            return {
+                ...params,
+                config: {
+                    ...params.config,
+                    temperature: getTemperatureWithSemanticRetry({
+                        temperature: params.config.temperature,
+                        userTemperature: input.userConfig?.temperature,
+                        semanticRetryAttempt: input.semanticRetryAttempt
+                    })
+                }
+            };
+        }
+    };
+}
+export { buildDefaultResponsesParams, resolveResponses };
+
+//# sourceMappingURL=responses.mjs.map

package/dist/es/ai-model/model-adapter/responses.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/model-adapter/responses.mjs","sources":["../../../../src/ai-model/model-adapter/responses.ts"],"sourcesContent":["import { getTemperatureWithSemanticRetry } from './temperature-with-semantic-retry';\nimport type {\n  BuildResponsesParams,\n  MidsceneResponsesDefaults,\n  ModelAdapterDefinition,\n  ResponsesAdapter,\n} from './types';\n\nconst midsceneResponsesDefaults: MidsceneResponsesDefaults = {\n  temperature: 0,\n};\n\nexport const buildDefaultResponsesParams: BuildResponsesParams = ({\n  midsceneDefaults,\n  userConfig,\n}) => ({\n  config: {\n    temperature: userConfig.temperature ?? midsceneDefaults.temperature,\n  },\n});\n\nexport function resolveResponses(\n  responses: ModelAdapterDefinition['responses'],\n): ResponsesAdapter {\n  const buildResponsesParams =\n    responses?.buildResponsesParams ?? buildDefaultResponsesParams;\n  return {\n    replayRawAssistantOutput: responses?.replayRawAssistantOutput ?? false,\n    unsupportedUserConfig: responses?.unsupportedUserConfig ?? [],\n    buildResponsesParams: (input) => {\n      const context = {\n        ...input,\n        userConfig: input.userConfig ?? {},\n        midsceneDefaults: midsceneResponsesDefaults,\n      };\n      const params = buildResponsesParams(context);\n      return {\n        ...params,\n        config: {\n          ...params.config,\n          temperature: getTemperatureWithSemanticRetry({\n            temperature: params.config.temperature,\n            userTemperature: input.userConfig?.temperature,\n            semanticRetryAttempt: input.semanticRetryAttempt,\n          }),\n        },\n      };\n    },\n  };\n}\n"],"names":["midsceneResponsesDefaults","buildDefaultResponsesParams","midsceneDefaults","userConfig","resolveResponses","responses","buildResponsesParams","input","context","params","getTemperatureWithSemanticRetry"],"mappings":";AAQA,MAAMA,4BAAuD;IAC3D,aAAa;AACf;AAEO,MAAMC,8BAAoD,CAAC,EAChEC,gBAAgB,EAChBC,UAAU,EACX,GAAM;QACL,QAAQ;YACN,aAAaA,WAAW,WAAW,IAAID,iBAAiB,WAAW;QACrE;IACF;AAEO,SAASE,iBACdC,SAA8C;IAE9C,MAAMC,uBACJD,WAAW,wBAAwBJ;IACrC,OAAO;QACL,0BAA0BI,WAAW,4BAA4B;QACjE,uBAAuBA,WAAW,yBAAyB,EAAE;QAC7D,sBAAsB,CAACE;YACrB,MAAMC,UAAU;gBACd,GAAGD,KAAK;gBACR,YAAYA,MAAM,UAAU,IAAI,CAAC;gBACjC,kBAAkBP;YACpB;YACA,MAAMS,SAASH,qBAAqBE;YACpC,OAAO;gBACL,GAAGC,MAAM;gBACT,QAAQ;oBACN,GAAGA,OAAO,MAAM;oBAChB,aAAaC,gCAAgC;wBAC3C,aAAaD,OAAO,MAAM,CAAC,WAAW;wBACtC,iBAAiBF,MAAM,UAAU,EAAE;wBACnC,sBAAsBA,MAAM,oBAAoB;oBAClD;gBACF;YACF;QACF;IACF;AACF"}

package/dist/es/ai-model/model-adapter/temperature-with-semantic-retry.mjs

@@ -0,0 +1,7 @@
+function getTemperatureWithSemanticRetry({ temperature, userTemperature, semanticRetryAttempt = 0 }) {
+    if (semanticRetryAttempt > 0 && void 0 === userTemperature && 0 === temperature) return 0.2;
+    return temperature;
+}
+export { getTemperatureWithSemanticRetry };
+
+//# sourceMappingURL=temperature-with-semantic-retry.mjs.map

package/dist/es/ai-model/model-adapter/temperature-with-semantic-retry.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/model-adapter/temperature-with-semantic-retry.mjs","sources":["../../../../src/ai-model/model-adapter/temperature-with-semantic-retry.ts"],"sourcesContent":["export function getTemperatureWithSemanticRetry<T>({\n  temperature,\n  userTemperature,\n  semanticRetryAttempt = 0,\n}: {\n  temperature: T;\n  userTemperature?: number;\n  semanticRetryAttempt?: number;\n}): T | number {\n  // Only perturb the default deterministic request after a semantic parse\n  // failure: preserve an explicit user temperature, and avoid adding a\n  // temperature to adapters whose normal parameters do not include one.\n  if (\n    semanticRetryAttempt > 0 &&\n    userTemperature === undefined &&\n    temperature === 0\n  ) {\n    return 0.2;\n  }\n  return temperature;\n}\n"],"names":["getTemperatureWithSemanticRetry","temperature","userTemperature","semanticRetryAttempt","undefined"],"mappings":"AAAO,SAASA,gCAAmC,EACjDC,WAAW,EACXC,eAAe,EACfC,uBAAuB,CAAC,EAKzB;IAIC,IACEA,uBAAuB,KACvBD,AAAoBE,WAApBF,mBACAD,AAAgB,MAAhBA,aAEA,OAAO;IAET,OAAOA;AACT"}

package/dist/es/ai-model/models/gpt.mjs

@@ -1,5 +1,5 @@
 import { isLocateIntent } from "./utils/intent.mjs";
-const originalImageDetailForDefaultIntent = (input)=>isLocateIntent(input.intent) || input.requiresOriginalImageDetail ? 'original' : void 0;
+const originalImageDetailForDefaultIntent = (input)=>isLocateIntent(input.intent) || input.requiresOriginalImageDetail ? 'original' : input.imageDetail ?? 'high';
 const resolveGpt5ReasoningEffort = ({ reasoningEnabled, reasoningEffort })=>{
     if ('default' === reasoningEnabled) return;
     return true === reasoningEnabled ? reasoningEffort ?? 'medium' : 'none';
@@ -11,15 +11,56 @@ const resolveGpt6ReasoningEffort = ({ reasoningEnabled, reasoningEffort })=>{
 const buildGpt5CodexAppServerParams = (input)=>({
         config: {
             effort: resolveGpt5ReasoningEffort(input.userConfig ?? {})
-        },
-        imageDetail: originalImageDetailForDefaultIntent(input)
+        }
     });
 const buildGpt6CodexAppServerParams = (input)=>({
         config: {
             effort: resolveGpt6ReasoningEffort(input.userConfig ?? {})
-        },
-        imageDetail: originalImageDetailForDefaultIntent(input)
+        }
     });
+const buildGpt5ResponsesParams = (input)=>{
+    const { midsceneDefaults, userConfig, expectedJsonObjectResponse } = input;
+    const commonOverrideConfig = {};
+    if (void 0 !== userConfig.temperature) commonOverrideConfig.temperature = userConfig.temperature;
+    if ('none' !== userConfig.responseFormat && expectedJsonObjectResponse) commonOverrideConfig.text = {
+        format: {
+            type: 'json_object'
+        }
+    };
+    const effectiveReasoningEffort = resolveGpt5ReasoningEffort(userConfig);
+    return {
+        config: {
+            ...midsceneDefaults,
+            ...commonOverrideConfig,
+            reasoning: {
+                effort: effectiveReasoningEffort,
+                context: 'current_turn'
+            }
+        }
+    };
+};
+const buildGpt6ResponsesParams = (input)=>{
+    const { midsceneDefaults, userConfig, expectedJsonObjectResponse } = input;
+    const { responseFormat } = userConfig;
+    const commonOverrideConfig = {};
+    commonOverrideConfig.temperature = void 0;
+    if ('none' !== responseFormat && expectedJsonObjectResponse) commonOverrideConfig.text = {
+        format: {
+            type: 'json_object'
+        }
+    };
+    const effectiveReasoningEffort = resolveGpt6ReasoningEffort(userConfig);
+    return {
+        config: {
+            ...midsceneDefaults,
+            ...commonOverrideConfig,
+            reasoning: {
+                effort: effectiveReasoningEffort,
+                context: 'current_turn'
+            }
+        }
+    };
+};
 const buildGpt5ChatCompletionParams = (input)=>{
     const { midsceneDefaults, userConfig } = input;
     const commonOverrideConfig = {};
@@ -55,13 +96,24 @@ const buildGpt6ChatCompletionParams = (input)=>{
 };
 const gptAdapters = {
     'gpt-5': {
+        supportedProtocols: [
+            'openai-chat',
+            'openai-responses'
+        ],
+        resolveImageDetail: originalImageDetailForDefaultIntent,
+        responses: {
+            replayRawAssistantOutput: false,
+            unsupportedUserConfig: [
+                'reasoningBudget'
+            ],
+            buildResponsesParams: buildGpt5ResponsesParams
+        },
         buildCodexAppServerParams: buildGpt5CodexAppServerParams,
         chatCompletion: {
             unsupportedUserConfig: [
                 'reasoningBudget'
             ],
-            buildChatCompletionParams: buildGpt5ChatCompletionParams,
-            resolveImageDetail: originalImageDetailForDefaultIntent
+            buildChatCompletionParams: buildGpt5ChatCompletionParams
         },
         locate: {
             element: {
@@ -75,14 +127,26 @@ const gptAdapters = {
         }
     },
     'gpt-6': {
+        supportedProtocols: [
+            'openai-chat',
+            'openai-responses'
+        ],
+        resolveImageDetail: originalImageDetailForDefaultIntent,
+        responses: {
+            replayRawAssistantOutput: false,
+            unsupportedUserConfig: [
+                'temperature',
+                'reasoningBudget'
+            ],
+            buildResponsesParams: buildGpt6ResponsesParams
+        },
         buildCodexAppServerParams: buildGpt6CodexAppServerParams,
         chatCompletion: {
             unsupportedUserConfig: [
                 'temperature',
                 'reasoningBudget'
             ],
-            buildChatCompletionParams: buildGpt6ChatCompletionParams,
-            resolveImageDetail: originalImageDetailForDefaultIntent
+            buildChatCompletionParams: buildGpt6ChatCompletionParams
         },
         locate: {
             element: {

package/dist/es/ai-model/models/gpt.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/models/gpt.mjs","sources":["../../../../src/ai-model/models/gpt.ts"],"sourcesContent":["import type { TModelFamily } from '@midscene/shared/env';\nimport type {\n  ChatCompletionCallContext,\n  ChatCompletionParamsResult,\n  CodexAppServerCallInput,\n  CodexAppServerParamsResult,\n  ImageDetail,\n  ModelAdapterDefinition,\n  ReasoningInput,\n} from '../model-adapter/types';\nimport { isLocateIntent } from './utils/intent';\n\nconst originalImageDetailForDefaultIntent = (\n  input: Pick<\n    CodexAppServerCallInput,\n    'intent' | 'requiresOriginalImageDetail'\n  >,\n): ImageDetail | undefined =>\n  isLocateIntent(input.intent) || input.requiresOriginalImageDetail\n    ? 'original'\n    : undefined;\n\nconst resolveGpt5ReasoningEffort = ({\n  reasoningEnabled,\n  reasoningEffort,\n}: ReasoningInput): string | undefined => {\n  if (reasoningEnabled === 'default') return undefined;\n  return reasoningEnabled === true ? (reasoningEffort ?? 'medium') : 'none';\n};\n\n// Astra cannot disable reasoning; use its lowest effort when disabled.\nconst resolveGpt6ReasoningEffort = ({\n  reasoningEnabled,\n  reasoningEffort,\n}: ReasoningInput): string | undefined => {\n  if (reasoningEnabled === 'default') return undefined;\n  return reasoningEnabled === true ? (reasoningEffort ?? 'medium') : 'low';\n};\n\nconst buildGpt5CodexAppServerParams = (\n  input: CodexAppServerCallInput,\n): CodexAppServerParamsResult => ({\n  config: { effort: resolveGpt5ReasoningEffort(input.userConfig ?? {}) },\n  imageDetail: originalImageDetailForDefaultIntent(input),\n});\n\nconst buildGpt6CodexAppServerParams = (\n  input: CodexAppServerCallInput,\n): CodexAppServerParamsResult => ({\n  config: { effort: resolveGpt6ReasoningEffort(input.userConfig ?? {}) },\n  imageDetail: originalImageDetailForDefaultIntent(input),\n});\n\nconst buildGpt5ChatCompletionParams = (\n  input: ChatCompletionCallContext,\n): ChatCompletionParamsResult => {\n  const { midsceneDefaults, userConfig } = input;\n  const commonOverrideConfig: Record<string, unknown> = {};\n\n  if (userConfig.temperature !== undefined) {\n    commonOverrideConfig.temperature = userConfig.temperature;\n  }\n\n  // OpenAI Chat Completions JSON mode:\n  // https://platform.openai.com/docs/guides/structured-outputs?api-mode=chat#json-mode\n  if (\n    input.userConfig.responseFormat !== 'none' &&\n    input.expectedJsonObjectResponse\n  ) {\n    commonOverrideConfig.response_format = { type: 'json_object' };\n  }\n\n  const effectiveReasoningEffort = resolveGpt5ReasoningEffort(userConfig);\n\n  return {\n    config: {\n      ...midsceneDefaults,\n      ...commonOverrideConfig,\n      reasoning_effort: effectiveReasoningEffort,\n    },\n  };\n};\n\nconst buildGpt6ChatCompletionParams = (\n  input: ChatCompletionCallContext,\n): ChatCompletionParamsResult => {\n  const { midsceneDefaults, userConfig, expectedJsonObjectResponse } = input;\n  const { responseFormat } = userConfig;\n  const commonOverrideConfig: Record<string, unknown> = {};\n  // GPT-6 does not support temperature; omit it from the serialized request.\n  commonOverrideConfig.temperature = undefined;\n  if (responseFormat !== 'none' && expectedJsonObjectResponse) {\n    commonOverrideConfig.response_format = { type: 'json_object' };\n  }\n\n  const effectiveReasoningEffort = resolveGpt6ReasoningEffort(userConfig);\n\n  return {\n    config: {\n      ...midsceneDefaults,\n      ...commonOverrideConfig,\n      reasoning_effort: effectiveReasoningEffort,\n    },\n  };\n};\n\nexport const gptAdapters = {\n  'gpt-5': {\n    buildCodexAppServerParams: buildGpt5CodexAppServerParams,\n    chatCompletion: {\n      unsupportedUserConfig: ['reasoningBudget'],\n      buildChatCompletionParams: buildGpt5ChatCompletionParams,\n      resolveImageDetail: originalImageDetailForDefaultIntent,\n    },\n    locate: {\n      element: {\n        resultFormat: {\n          coordinates: { shape: 'bbox', order: 'xy' },\n        },\n      },\n    },\n  },\n  'gpt-6': {\n    buildCodexAppServerParams: buildGpt6CodexAppServerParams,\n    chatCompletion: {\n      unsupportedUserConfig: ['temperature', 'reasoningBudget'],\n      buildChatCompletionParams: buildGpt6ChatCompletionParams,\n      resolveImageDetail: originalImageDetailForDefaultIntent,\n    },\n    locate: {\n      element: {\n        resultFormat: {\n          coordinates: { shape: 'bbox', order: 'xy' },\n        },\n      },\n    },\n  },\n} satisfies Pick<\n  Record<TModelFamily, ModelAdapterDefinition>,\n  'gpt-5' | 'gpt-6'\n>;\n"],"names":["originalImageDetailForDefaultIntent","input","isLocateIntent","undefined","resolveGpt5ReasoningEffort","reasoningEnabled","reasoningEffort","resolveGpt6ReasoningEffort","buildGpt5CodexAppServerParams","buildGpt6CodexAppServerParams","buildGpt5ChatCompletionParams","midsceneDefaults","userConfig","commonOverrideConfig","effectiveReasoningEffort","buildGpt6ChatCompletionParams","expectedJsonObjectResponse","responseFormat","gptAdapters"],"mappings":";AAYA,MAAMA,sCAAsC,CAC1CC,QAKAC,eAAeD,MAAM,MAAM,KAAKA,MAAM,2BAA2B,GAC7D,aACAE;AAEN,MAAMC,6BAA6B,CAAC,EAClCC,gBAAgB,EAChBC,eAAe,EACA;IACf,IAAID,AAAqB,cAArBA,kBAAgC;IACpC,OAAOA,AAAqB,SAArBA,mBAA6BC,mBAAmB,WAAY;AACrE;AAGA,MAAMC,6BAA6B,CAAC,EAClCF,gBAAgB,EAChBC,eAAe,EACA;IACf,IAAID,AAAqB,cAArBA,kBAAgC;IACpC,OAAOA,AAAqB,SAArBA,mBAA6BC,mBAAmB,WAAY;AACrE;AAEA,MAAME,gCAAgC,CACpCP,QACgC;QAChC,QAAQ;YAAE,QAAQG,2BAA2BH,MAAM,UAAU,IAAI,CAAC;QAAG;QACrE,aAAaD,oCAAoCC;IACnD;AAEA,MAAMQ,gCAAgC,CACpCR,QACgC;QAChC,QAAQ;YAAE,QAAQM,2BAA2BN,MAAM,UAAU,IAAI,CAAC;QAAG;QACrE,aAAaD,oCAAoCC;IACnD;AAEA,MAAMS,gCAAgC,CACpCT;IAEA,MAAM,EAAEU,gBAAgB,EAAEC,UAAU,EAAE,GAAGX;IACzC,MAAMY,uBAAgD,CAAC;IAEvD,IAAID,AAA2BT,WAA3BS,WAAW,WAAW,EACxBC,qBAAqB,WAAW,GAAGD,WAAW,WAAW;IAK3D,IACEX,AAAoC,WAApCA,MAAM,UAAU,CAAC,cAAc,IAC/BA,MAAM,0BAA0B,EAEhCY,qBAAqB,eAAe,GAAG;QAAE,MAAM;IAAc;IAG/D,MAAMC,2BAA2BV,2BAA2BQ;IAE5D,OAAO;QACL,QAAQ;YACN,GAAGD,gBAAgB;YACnB,GAAGE,oBAAoB;YACvB,kBAAkBC;QACpB;IACF;AACF;AAEA,MAAMC,gCAAgC,CACpCd;IAEA,MAAM,EAAEU,gBAAgB,EAAEC,UAAU,EAAEI,0BAA0B,EAAE,GAAGf;IACrE,MAAM,EAAEgB,cAAc,EAAE,GAAGL;IAC3B,MAAMC,uBAAgD,CAAC;IAEvDA,qBAAqB,WAAW,GAAGV;IACnC,IAAIc,AAAmB,WAAnBA,kBAA6BD,4BAC/BH,qBAAqB,eAAe,GAAG;QAAE,MAAM;IAAc;IAG/D,MAAMC,2BAA2BP,2BAA2BK;IAE5D,OAAO;QACL,QAAQ;YACN,GAAGD,gBAAgB;YACnB,GAAGE,oBAAoB;YACvB,kBAAkBC;QACpB;IACF;AACF;AAEO,MAAMI,cAAc;IACzB,SAAS;QACP,2BAA2BV;QAC3B,gBAAgB;YACd,uBAAuB;gBAAC;aAAkB;YAC1C,2BAA2BE;YAC3B,oBAAoBV;QACtB;QACA,QAAQ;YACN,SAAS;gBACP,cAAc;oBACZ,aAAa;wBAAE,OAAO;wBAAQ,OAAO;oBAAK;gBAC5C;YACF;QACF;IACF;IACA,SAAS;QACP,2BAA2BS;QAC3B,gBAAgB;YACd,uBAAuB;gBAAC;gBAAe;aAAkB;YACzD,2BAA2BM;YAC3B,oBAAoBf;QACtB;QACA,QAAQ;YACN,SAAS;gBACP,cAAc;oBACZ,aAAa;wBAAE,OAAO;wBAAQ,OAAO;oBAAK;gBAC5C;YACF;QACF;IACF;AACF"}
+{"version":3,"file":"ai-model/models/gpt.mjs","sources":["../../../../src/ai-model/models/gpt.ts"],"sourcesContent":["import type { TModelFamily } from '@midscene/shared/env';\nimport type {\n  BuildResponsesParams,\n  ChatCompletionCallContext,\n  ChatCompletionParamsResult,\n  CodexAppServerCallInput,\n  CodexAppServerParamsResult,\n  ModelAdapterDefinition,\n  ReasoningInput,\n  ResolveImageDetail,\n} from '../model-adapter/types';\nimport { isLocateIntent } from './utils/intent';\n\nconst originalImageDetailForDefaultIntent: ResolveImageDetail = (input) =>\n  isLocateIntent(input.intent) || input.requiresOriginalImageDetail\n    ? 'original'\n    : (input.imageDetail ?? 'high');\n\nconst resolveGpt5ReasoningEffort = ({\n  reasoningEnabled,\n  reasoningEffort,\n}: ReasoningInput): string | undefined => {\n  if (reasoningEnabled === 'default') return undefined;\n  return reasoningEnabled === true ? (reasoningEffort ?? 'medium') : 'none';\n};\n\n// Astra cannot disable reasoning; use its lowest effort when disabled.\nconst resolveGpt6ReasoningEffort = ({\n  reasoningEnabled,\n  reasoningEffort,\n}: ReasoningInput): string | undefined => {\n  if (reasoningEnabled === 'default') return undefined;\n  return reasoningEnabled === true ? (reasoningEffort ?? 'medium') : 'low';\n};\n\nconst buildGpt5CodexAppServerParams = (\n  input: CodexAppServerCallInput,\n): CodexAppServerParamsResult => ({\n  config: { effort: resolveGpt5ReasoningEffort(input.userConfig ?? {}) },\n});\n\nconst buildGpt6CodexAppServerParams = (\n  input: CodexAppServerCallInput,\n): CodexAppServerParamsResult => ({\n  config: { effort: resolveGpt6ReasoningEffort(input.userConfig ?? {}) },\n});\n\nconst buildGpt5ResponsesParams: BuildResponsesParams = (input) => {\n  const { midsceneDefaults, userConfig, expectedJsonObjectResponse } = input;\n  const commonOverrideConfig: Record<string, unknown> = {};\n\n  if (userConfig.temperature !== undefined) {\n    commonOverrideConfig.temperature = userConfig.temperature;\n  }\n\n  if (userConfig.responseFormat !== 'none' && expectedJsonObjectResponse) {\n    commonOverrideConfig.text = { format: { type: 'json_object' } };\n  }\n\n  const effectiveReasoningEffort = resolveGpt5ReasoningEffort(userConfig);\n\n  return {\n    config: {\n      ...midsceneDefaults,\n      ...commonOverrideConfig,\n      // Use current-turn reasoning only. Switching to all_turns requires\n      // responses.replayRawAssistantOutput to preserve reasoning history.\n      reasoning: { effort: effectiveReasoningEffort, context: 'current_turn' },\n    },\n  };\n};\n\nconst buildGpt6ResponsesParams: BuildResponsesParams = (input) => {\n  const { midsceneDefaults, userConfig, expectedJsonObjectResponse } = input;\n  const { responseFormat } = userConfig;\n  const commonOverrideConfig: Record<string, unknown> = {};\n  // GPT-6 does not support temperature; omit it from the serialized request.\n  commonOverrideConfig.temperature = undefined;\n  if (responseFormat !== 'none' && expectedJsonObjectResponse) {\n    commonOverrideConfig.text = { format: { type: 'json_object' } };\n  }\n\n  const effectiveReasoningEffort = resolveGpt6ReasoningEffort(userConfig);\n\n  return {\n    config: {\n      ...midsceneDefaults,\n      ...commonOverrideConfig,\n      // Use current-turn reasoning only. Switching to all_turns requires\n      // responses.replayRawAssistantOutput to preserve reasoning history.\n      reasoning: { effort: effectiveReasoningEffort, context: 'current_turn' },\n    },\n  };\n};\n\nconst buildGpt5ChatCompletionParams = (\n  input: ChatCompletionCallContext,\n): ChatCompletionParamsResult => {\n  const { midsceneDefaults, userConfig } = input;\n  const commonOverrideConfig: Record<string, unknown> = {};\n\n  if (userConfig.temperature !== undefined) {\n    commonOverrideConfig.temperature = userConfig.temperature;\n  }\n\n  // OpenAI Chat Completions JSON mode:\n  // https://platform.openai.com/docs/guides/structured-outputs?api-mode=chat#json-mode\n  if (\n    input.userConfig.responseFormat !== 'none' &&\n    input.expectedJsonObjectResponse\n  ) {\n    commonOverrideConfig.response_format = { type: 'json_object' };\n  }\n\n  const effectiveReasoningEffort = resolveGpt5ReasoningEffort(userConfig);\n\n  return {\n    config: {\n      ...midsceneDefaults,\n      ...commonOverrideConfig,\n      reasoning_effort: effectiveReasoningEffort,\n    },\n  };\n};\n\nconst buildGpt6ChatCompletionParams = (\n  input: ChatCompletionCallContext,\n): ChatCompletionParamsResult => {\n  const { midsceneDefaults, userConfig, expectedJsonObjectResponse } = input;\n  const { responseFormat } = userConfig;\n  const commonOverrideConfig: Record<string, unknown> = {};\n  // GPT-6 does not support temperature; omit it from the serialized request.\n  commonOverrideConfig.temperature = undefined;\n  if (responseFormat !== 'none' && expectedJsonObjectResponse) {\n    commonOverrideConfig.response_format = { type: 'json_object' };\n  }\n\n  const effectiveReasoningEffort = resolveGpt6ReasoningEffort(userConfig);\n\n  return {\n    config: {\n      ...midsceneDefaults,\n      ...commonOverrideConfig,\n      reasoning_effort: effectiveReasoningEffort,\n    },\n  };\n};\n\nexport const gptAdapters = {\n  'gpt-5': {\n    supportedProtocols: ['openai-chat', 'openai-responses'],\n    resolveImageDetail: originalImageDetailForDefaultIntent,\n    responses: {\n      replayRawAssistantOutput: false,\n      unsupportedUserConfig: ['reasoningBudget'],\n      buildResponsesParams: buildGpt5ResponsesParams,\n    },\n    buildCodexAppServerParams: buildGpt5CodexAppServerParams,\n    chatCompletion: {\n      unsupportedUserConfig: ['reasoningBudget'],\n      buildChatCompletionParams: buildGpt5ChatCompletionParams,\n    },\n    locate: {\n      element: {\n        resultFormat: {\n          coordinates: { shape: 'bbox', order: 'xy' },\n        },\n      },\n    },\n  },\n  'gpt-6': {\n    supportedProtocols: ['openai-chat', 'openai-responses'],\n    resolveImageDetail: originalImageDetailForDefaultIntent,\n    responses: {\n      replayRawAssistantOutput: false,\n      unsupportedUserConfig: ['temperature', 'reasoningBudget'],\n      buildResponsesParams: buildGpt6ResponsesParams,\n    },\n    buildCodexAppServerParams: buildGpt6CodexAppServerParams,\n    chatCompletion: {\n      unsupportedUserConfig: ['temperature', 'reasoningBudget'],\n      buildChatCompletionParams: buildGpt6ChatCompletionParams,\n    },\n    locate: {\n      element: {\n        resultFormat: {\n          coordinates: { shape: 'bbox', order: 'xy' },\n        },\n      },\n    },\n  },\n} satisfies Pick<\n  Record<TModelFamily, ModelAdapterDefinition>,\n  'gpt-5' | 'gpt-6'\n>;\n"],"names":["originalImageDetailForDefaultIntent","input","isLocateIntent","resolveGpt5ReasoningEffort","reasoningEnabled","reasoningEffort","resolveGpt6ReasoningEffort","buildGpt5CodexAppServerParams","buildGpt6CodexAppServerParams","buildGpt5ResponsesParams","midsceneDefaults","userConfig","expectedJsonObjectResponse","commonOverrideConfig","undefined","effectiveReasoningEffort","buildGpt6ResponsesParams","responseFormat","buildGpt5ChatCompletionParams","buildGpt6ChatCompletionParams","gptAdapters"],"mappings":";AAaA,MAAMA,sCAA0D,CAACC,QAC/DC,eAAeD,MAAM,MAAM,KAAKA,MAAM,2BAA2B,GAC7D,aACCA,MAAM,WAAW,IAAI;AAE5B,MAAME,6BAA6B,CAAC,EAClCC,gBAAgB,EAChBC,eAAe,EACA;IACf,IAAID,AAAqB,cAArBA,kBAAgC;IACpC,OAAOA,AAAqB,SAArBA,mBAA6BC,mBAAmB,WAAY;AACrE;AAGA,MAAMC,6BAA6B,CAAC,EAClCF,gBAAgB,EAChBC,eAAe,EACA;IACf,IAAID,AAAqB,cAArBA,kBAAgC;IACpC,OAAOA,AAAqB,SAArBA,mBAA6BC,mBAAmB,WAAY;AACrE;AAEA,MAAME,gCAAgC,CACpCN,QACgC;QAChC,QAAQ;YAAE,QAAQE,2BAA2BF,MAAM,UAAU,IAAI,CAAC;QAAG;IACvE;AAEA,MAAMO,gCAAgC,CACpCP,QACgC;QAChC,QAAQ;YAAE,QAAQK,2BAA2BL,MAAM,UAAU,IAAI,CAAC;QAAG;IACvE;AAEA,MAAMQ,2BAAiD,CAACR;IACtD,MAAM,EAAES,gBAAgB,EAAEC,UAAU,EAAEC,0BAA0B,EAAE,GAAGX;IACrE,MAAMY,uBAAgD,CAAC;IAEvD,IAAIF,AAA2BG,WAA3BH,WAAW,WAAW,EACxBE,qBAAqB,WAAW,GAAGF,WAAW,WAAW;IAG3D,IAAIA,AAA8B,WAA9BA,WAAW,cAAc,IAAeC,4BAC1CC,qBAAqB,IAAI,GAAG;QAAE,QAAQ;YAAE,MAAM;QAAc;IAAE;IAGhE,MAAME,2BAA2BZ,2BAA2BQ;IAE5D,OAAO;QACL,QAAQ;YACN,GAAGD,gBAAgB;YACnB,GAAGG,oBAAoB;YAGvB,WAAW;gBAAE,QAAQE;gBAA0B,SAAS;YAAe;QACzE;IACF;AACF;AAEA,MAAMC,2BAAiD,CAACf;IACtD,MAAM,EAAES,gBAAgB,EAAEC,UAAU,EAAEC,0BAA0B,EAAE,GAAGX;IACrE,MAAM,EAAEgB,cAAc,EAAE,GAAGN;IAC3B,MAAME,uBAAgD,CAAC;IAEvDA,qBAAqB,WAAW,GAAGC;IACnC,IAAIG,AAAmB,WAAnBA,kBAA6BL,4BAC/BC,qBAAqB,IAAI,GAAG;QAAE,QAAQ;YAAE,MAAM;QAAc;IAAE;IAGhE,MAAME,2BAA2BT,2BAA2BK;IAE5D,OAAO;QACL,QAAQ;YACN,GAAGD,gBAAgB;YACnB,GAAGG,oBAAoB;YAGvB,WAAW;gBAAE,QAAQE;gBAA0B,SAAS;YAAe;QACzE;IACF;AACF;AAEA,MAAMG,gCAAgC,CACpCjB;IAEA,MAAM,EAAES,gBAAgB,EAAEC,UAAU,EAAE,GAAGV;IACzC,MAAMY,uBAAgD,CAAC;IAEvD,IAAIF,AAA2BG,WAA3BH,WAAW,WAAW,EACxBE,qBAAqB,WAAW,GAAGF,WAAW,WAAW;IAK3D,IACEV,AAAoC,WAApCA,MAAM,UAAU,CAAC,cAAc,IAC/BA,MAAM,0BAA0B,EAEhCY,qBAAqB,eAAe,GAAG;QAAE,MAAM;IAAc;IAG/D,MAAME,2BAA2BZ,2BAA2BQ;IAE5D,OAAO;QACL,QAAQ;YACN,GAAGD,gBAAgB;YACnB,GAAGG,oBAAoB;YACvB,kBAAkBE;QACpB;IACF;AACF;AAEA,MAAMI,gCAAgC,CACpClB;IAEA,MAAM,EAAES,gBAAgB,EAAEC,UAAU,EAAEC,0BAA0B,EAAE,GAAGX;IACrE,MAAM,EAAEgB,cAAc,EAAE,GAAGN;IAC3B,MAAME,uBAAgD,CAAC;IAEvDA,qBAAqB,WAAW,GAAGC;IACnC,IAAIG,AAAmB,WAAnBA,kBAA6BL,4BAC/BC,qBAAqB,eAAe,GAAG;QAAE,MAAM;IAAc;IAG/D,MAAME,2BAA2BT,2BAA2BK;IAE5D,OAAO;QACL,QAAQ;YACN,GAAGD,gBAAgB;YACnB,GAAGG,oBAAoB;YACvB,kBAAkBE;QACpB;IACF;AACF;AAEO,MAAMK,cAAc;IACzB,SAAS;QACP,oBAAoB;YAAC;YAAe;SAAmB;QACvD,oBAAoBpB;QACpB,WAAW;YACT,0BAA0B;YAC1B,uBAAuB;gBAAC;aAAkB;YAC1C,sBAAsBS;QACxB;QACA,2BAA2BF;QAC3B,gBAAgB;YACd,uBAAuB;gBAAC;aAAkB;YAC1C,2BAA2BW;QAC7B;QACA,QAAQ;YACN,SAAS;gBACP,cAAc;oBACZ,aAAa;wBAAE,OAAO;wBAAQ,OAAO;oBAAK;gBAC5C;YACF;QACF;IACF;IACA,SAAS;QACP,oBAAoB;YAAC;YAAe;SAAmB;QACvD,oBAAoBlB;QACpB,WAAW;YACT,0BAA0B;YAC1B,uBAAuB;gBAAC;gBAAe;aAAkB;YACzD,sBAAsBgB;QACxB;QACA,2BAA2BR;QAC3B,gBAAgB;YACd,uBAAuB;gBAAC;gBAAe;aAAkB;YACzD,2BAA2BW;QAC7B;QACA,QAAQ;YACN,SAAS;gBACP,cAAc;oBACZ,aAAa;wBAAE,OAAO;wBAAQ,OAAO;oBAAK;gBAC5C;YACF;QACF;IACF;AACF"}

package/dist/es/ai-model/models/qwen.mjs

@@ -63,7 +63,6 @@ const buildQwen25ChatCompletionParams = (input)=>{
     };
 };
 const qwen3Adapter = {
-    acceptBbox2dAlias: true,
     chatCompletion: {
         unsupportedUserConfig: [
             'reasoningEffort'
@@ -81,6 +80,10 @@ const qwen3Adapter = {
     locate: {
         element: {
             resultFormat: {
+                resultKey: 'bbox_2d',
+                resultKeyAliases: [
+                    'bbox'
+                ],
                 coordinates: qwen3BboxCoordinatesMeta
             }
         }
@@ -88,7 +91,6 @@ const qwen3Adapter = {
 };
 const qwenAdapters = {
     'qwen2.5-vl': {
-        acceptBbox2dAlias: true,
         chatCompletion: {
             unsupportedUserConfig: [
                 'reasoningEnabled',
@@ -103,6 +105,10 @@ const qwenAdapters = {
         locate: {
             element: {
                 resultFormat: {
+                    resultKey: 'bbox_2d',
+                    resultKeyAliases: [
+                        'bbox'
+                    ],
                     coordinates: qwen25BboxCoordinatesMeta,
                     parseRawLocateValue: parseQwen25RawLocateValue
                 }