@midscene/core 1.11.1-beta-20260821125003.0 → 1.12.0

package/dist/es/agent/utils.mjs

@@ -154,7 +154,7 @@ async function matchElementFromCache(context, cacheEntry, cachePrompt, cacheable
         return;
     }
 }
-const getMidsceneVersion = ()=>"1.11.1-beta-20260821125003.0";
+const getMidsceneVersion = ()=>"1.12.0";
 const parsePrompt = (prompt)=>{
     if ('string' == typeof prompt) return {
         textPrompt: prompt,

package/dist/es/ai-model/index.mjs

@@ -1,10 +1,9 @@
 import { AIResponseParseError, callAI, callAIWithObjectResponse, callAIWithStringResponse } from "./service-caller/index.mjs";
 import { getModelRuntime } from "./models/index.mjs";
 import { runConnectivityTest } from "./connectivity/index.mjs";
-import { systemPromptToLocateElement } from "./prompt/llm-locator.mjs";
 import { convertRecordLogIntoMarkdown, createRecorderMarkdownReplayPrompt, generatePlaywrightTest, generatePlaywrightTestStream, generateRecorderMarkdownReplay, generateRecorderSessionMetadata, generateRecorderYamlTest, generateRecorderYamlTestStream, generateYamlTest, generateYamlTestStream } from "./workflows/recorder-generation/index.mjs";
 import { AiLocateElement, AiLocateSection } from "./workflows/grounding/index.mjs";
 import { AiExtractElementInfo, AiJudgeOrderSensitive } from "./workflows/insight/index.mjs";
 import { ConversationHistory, standardPlan } from "./workflows/planning/index.mjs";
 import { PointSchema, RectSchema, SizeSchema, TMultimodalPromptSchema, TUserPromptSchema, dumpActionParam, findAllMidsceneLocatorField, getMidsceneLocationSchema, parseActionParam } from "../common.mjs";
-export { AIResponseParseError, AiExtractElementInfo, AiJudgeOrderSensitive, AiLocateElement, AiLocateSection, ConversationHistory, PointSchema, RectSchema, SizeSchema, TMultimodalPromptSchema, TUserPromptSchema, callAI, callAIWithObjectResponse, callAIWithStringResponse, convertRecordLogIntoMarkdown, createRecorderMarkdownReplayPrompt, dumpActionParam, findAllMidsceneLocatorField, generatePlaywrightTest, generatePlaywrightTestStream, generateRecorderMarkdownReplay, generateRecorderSessionMetadata, generateRecorderYamlTest, generateRecorderYamlTestStream, generateYamlTest, generateYamlTestStream, getMidsceneLocationSchema, getModelRuntime, parseActionParam, runConnectivityTest, standardPlan, systemPromptToLocateElement };
+export { AIResponseParseError, AiExtractElementInfo, AiJudgeOrderSensitive, AiLocateElement, AiLocateSection, ConversationHistory, PointSchema, RectSchema, SizeSchema, TMultimodalPromptSchema, TUserPromptSchema, callAI, callAIWithObjectResponse, callAIWithStringResponse, convertRecordLogIntoMarkdown, createRecorderMarkdownReplayPrompt, dumpActionParam, findAllMidsceneLocatorField, generatePlaywrightTest, generatePlaywrightTestStream, generateRecorderMarkdownReplay, generateRecorderSessionMetadata, generateRecorderYamlTest, generateRecorderYamlTestStream, generateYamlTest, generateYamlTestStream, getMidsceneLocationSchema, getModelRuntime, parseActionParam, runConnectivityTest, standardPlan };

package/dist/es/ai-model/model-adapter/default-locate-protocol.mjs

@@ -0,0 +1,144 @@
+import { getPreferredLanguage } from "@midscene/shared/env";
+import { formatLocateExampleValue } from "../shared/model-locate-result/index.mjs";
+const markdownCodeFence = '```';
+const defaultLocateSystemPromptIntroduction = `## Role:
+You are an AI assistant that helps identify UI elements.
+
+## Objective:
+- Identify elements in screenshots that match the user's description.
+- Provide the coordinates of the element that matches the user's description.`;
+const defaultSearchAreaSystemPromptIntroduction = `## Role:
+You are an AI assistant that helps identify UI elements.
+
+## Objective:
+- Find a section containing the target element
+- If the description mentions reference elements, also locate sections containing those references`;
+const buildResponseInstructions = (promptSpec)=>{
+    const preferredLanguage = getPreferredLanguage();
+    const resultKey = promptSpec.resultKey;
+    const exampleValueText = formatLocateExampleValue(promptSpec.exampleValues[0]);
+    const resultFieldDescription = `the ${promptSpec.resultNoun} of the element that matches the user's description`;
+    return `## Output Format:
+${markdownCodeFence}json
+{
+  "${resultKey}": ${promptSpec.resultValueSchema},  // ${promptSpec.resultValueDescription}
+  "error": string // optional
+}
+${markdownCodeFence}
+
+Fields:
+* \`${resultKey}\` is ${resultFieldDescription}
+* \`error\` is an optional error message (if any)
+
+For example, when an element is found:
+${markdownCodeFence}json
+{
+  "${resultKey}": ${exampleValueText}
+}
+${markdownCodeFence}
+
+When no element is found:
+${markdownCodeFence}json
+{
+  "${resultKey}": [],
+  "error": "I can see ..., but {some element} is not found. Use ${preferredLanguage}."
+}
+${markdownCodeFence}
+`;
+};
+const buildUserPrompt = (targetElementDescription)=>`Find: ${targetElementDescription}`;
+const buildSearchAreaResponseInstructions = (promptSpec)=>{
+    const preferredLanguage = getPreferredLanguage();
+    const resultKey = promptSpec.resultKey;
+    const exampleValueText = formatLocateExampleValue(promptSpec.exampleValues[0]);
+    const resultJsonProperty = `"${resultKey}": ${promptSpec.resultValueSchema},  // ${promptSpec.resultValueDescription}`;
+    const resultFieldDescription = `${promptSpec.resultNoun} of the section containing the target element`;
+    const referenceFieldDescription = `Optional array of ${promptSpec.resultNounPlural} of reference elements`;
+    return `## Output Format:
+${markdownCodeFence}json
+{
+  ${resultJsonProperty}
+  "references_${resultKey}"?: [
+    ${promptSpec.resultValueSchema},
+    ...
+  ],
+  "error"?: string
+}
+${markdownCodeFence}
+
+Fields:
+* \`${resultKey}\` - ${resultFieldDescription}
+* \`references_${resultKey}\` - ${referenceFieldDescription}
+* \`error\` - Optional error message if the section cannot be found. Use ${preferredLanguage}.
+
+Example:
+If the description is "delete button on the second row with title 'Peter'", return:
+${markdownCodeFence}json
+{
+  "${resultKey}": ${exampleValueText},
+  "references_${resultKey}": [${exampleValueText}]
+}
+${markdownCodeFence}
+`;
+};
+const buildSearchAreaUserPrompt = (sectionDescription)=>`Find section containing: ${sectionDescription}`;
+const createParseRawResponse = (jsonParser, source, options)=>(content, promptSpec)=>{
+        const parsedResponse = jsonParser(content, {
+            source
+        });
+        if (!parsedResponse || 'object' != typeof parsedResponse) throw new Error(`Failed to parse JSON locate response: ${content}`);
+        const record = parsedResponse;
+        const target = void 0 !== record[promptSpec.resultKey] ? record[promptSpec.resultKey] : options.acceptBbox2dAlias && 'bbox' === promptSpec.resultKey ? record.bbox_2d : void 0;
+        const error = 'string' == typeof record.error ? record.error : void 0;
+        const hasTarget = Array.isArray(target) ? target.length > 0 : void 0 !== target;
+        if (!hasTarget) return {
+            kind: 'not-found',
+            ...error ? {
+                error
+            } : {}
+        };
+        if (!options.includeReferences) return {
+            kind: 'located',
+            target,
+            ...error ? {
+                error
+            } : {}
+        };
+        const rawReferences = record[`references_${promptSpec.resultKey}`];
+        const references = null == rawReferences ? void 0 : Array.isArray(rawReferences) ? rawReferences : [
+            rawReferences
+        ];
+        return {
+            kind: 'located',
+            target,
+            ...references?.length ? {
+                references
+            } : {},
+            ...error ? {
+                error
+            } : {}
+        };
+    };
+const createDefaultElementProtocol = ({ jsonParser }, options = {})=>({
+        systemPromptIntroduction: defaultLocateSystemPromptIntroduction,
+        buildResponseInstructions,
+        buildUserPrompt,
+        expectedJsonObjectResponse: true,
+        parseRawResponse: createParseRawResponse(jsonParser, 'locate', {
+            includeReferences: false,
+            acceptBbox2dAlias: options.acceptBbox2dAlias
+        })
+    });
+const createDefaultSearchAreaProtocol = ({ jsonParser }, options = {})=>({
+        systemPromptIntroduction: defaultSearchAreaSystemPromptIntroduction,
+        buildResponseInstructions: buildSearchAreaResponseInstructions,
+        buildUserPrompt: buildSearchAreaUserPrompt,
+        expectedJsonObjectResponse: true,
+        parseRawResponse: createParseRawResponse(jsonParser, 'section-locator', {
+            includeReferences: true,
+            acceptBbox2dAlias: options.acceptBbox2dAlias
+        })
+    });
+export { createDefaultElementProtocol, createDefaultSearchAreaProtocol };
+
+//# sourceMappingURL=default-locate-protocol.mjs.map

package/dist/es/ai-model/model-adapter/default-locate-protocol.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/model-adapter/default-locate-protocol.mjs","sources":["../../../../src/ai-model/model-adapter/default-locate-protocol.ts"],"sourcesContent":["import { getPreferredLanguage } from '@midscene/shared/env';\nimport type { JsonParser, JsonParserSource } from '../shared/json';\nimport {\n  type LocateResultPromptSpec,\n  formatLocateExampleValue,\n} from '../shared/model-locate-result';\nimport type {\n  ParsedLocateResponse,\n  StandardLocateProtocol,\n  StandardLocateProtocolContext,\n} from './locate-protocol';\n\nconst markdownCodeFence = '```';\n\nconst defaultLocateSystemPromptIntroduction = `## Role:\nYou are an AI assistant that helps identify UI elements.\n\n## Objective:\n- Identify elements in screenshots that match the user's description.\n- Provide the coordinates of the element that matches the user's description.`;\n\nconst defaultSearchAreaSystemPromptIntroduction = `## Role:\nYou are an AI assistant that helps identify UI elements.\n\n## Objective:\n- Find a section containing the target element\n- If the description mentions reference elements, also locate sections containing those references`;\n\nconst buildResponseInstructions = (promptSpec: LocateResultPromptSpec) => {\n  const preferredLanguage = getPreferredLanguage();\n  const resultKey = promptSpec.resultKey;\n  const exampleValueText = formatLocateExampleValue(\n    promptSpec.exampleValues[0],\n  );\n  const resultFieldDescription = `the ${promptSpec.resultNoun} of the element that matches the user's description`;\n  return `## Output Format:\n${markdownCodeFence}json\n{\n  \"${resultKey}\": ${promptSpec.resultValueSchema},  // ${promptSpec.resultValueDescription}\n  \"error\": string // optional\n}\n${markdownCodeFence}\n\nFields:\n* \\`${resultKey}\\` is ${resultFieldDescription}\n* \\`error\\` is an optional error message (if any)\n\nFor example, when an element is found:\n${markdownCodeFence}json\n{\n  \"${resultKey}\": ${exampleValueText}\n}\n${markdownCodeFence}\n\nWhen no element is found:\n${markdownCodeFence}json\n{\n  \"${resultKey}\": [],\n  \"error\": \"I can see ..., but {some element} is not found. Use ${preferredLanguage}.\"\n}\n${markdownCodeFence}\n`;\n};\n\nconst buildUserPrompt = (targetElementDescription: string) =>\n  `Find: ${targetElementDescription}`;\n\nconst buildSearchAreaResponseInstructions = (\n  promptSpec: LocateResultPromptSpec,\n) => {\n  const preferredLanguage = getPreferredLanguage();\n  const resultKey = promptSpec.resultKey;\n  const exampleValueText = formatLocateExampleValue(\n    promptSpec.exampleValues[0],\n  );\n  const resultJsonProperty = `\"${resultKey}\": ${promptSpec.resultValueSchema},  // ${promptSpec.resultValueDescription}`;\n  const resultFieldDescription = `${promptSpec.resultNoun} of the section containing the target element`;\n  const referenceFieldDescription = `Optional array of ${promptSpec.resultNounPlural} of reference elements`;\n  return `## Output Format:\n${markdownCodeFence}json\n{\n  ${resultJsonProperty}\n  \"references_${resultKey}\"?: [\n    ${promptSpec.resultValueSchema},\n    ...\n  ],\n  \"error\"?: string\n}\n${markdownCodeFence}\n\nFields:\n* \\`${resultKey}\\` - ${resultFieldDescription}\n* \\`references_${resultKey}\\` - ${referenceFieldDescription}\n* \\`error\\` - Optional error message if the section cannot be found. Use ${preferredLanguage}.\n\nExample:\nIf the description is \"delete button on the second row with title 'Peter'\", return:\n${markdownCodeFence}json\n{\n  \"${resultKey}\": ${exampleValueText},\n  \"references_${resultKey}\": [${exampleValueText}]\n}\n${markdownCodeFence}\n`;\n};\n\nconst buildSearchAreaUserPrompt = (sectionDescription: string) =>\n  `Find section containing: ${sectionDescription}`;\n\ntype ParseRawResponseOptions = {\n  includeReferences: boolean;\n  acceptBbox2dAlias?: boolean;\n};\n\nconst createParseRawResponse =\n  (\n    jsonParser: JsonParser,\n    source: JsonParserSource,\n    options: ParseRawResponseOptions,\n  ) =>\n  (\n    content: string,\n    promptSpec: LocateResultPromptSpec,\n  ): ParsedLocateResponse => {\n    const parsedResponse = jsonParser(content, {\n      source,\n    });\n    if (!parsedResponse || typeof parsedResponse !== 'object') {\n      throw new Error(`Failed to parse JSON locate response: ${content}`);\n    }\n    const record = parsedResponse as Record<string, unknown>;\n    const target =\n      record[promptSpec.resultKey] !== undefined\n        ? record[promptSpec.resultKey]\n        : options.acceptBbox2dAlias && promptSpec.resultKey === 'bbox'\n          ? record.bbox_2d\n          : undefined;\n    const error = typeof record.error === 'string' ? record.error : undefined;\n    const hasTarget = Array.isArray(target)\n      ? target.length > 0\n      : target !== undefined;\n    if (!hasTarget) {\n      return { kind: 'not-found', ...(error ? { error } : {}) };\n    }\n\n    if (!options.includeReferences) {\n      return { kind: 'located', target, ...(error ? { error } : {}) };\n    }\n\n    const rawReferences = record[`references_${promptSpec.resultKey}`];\n    const references =\n      rawReferences === undefined || rawReferences === null\n        ? undefined\n        : Array.isArray(rawReferences)\n          ? rawReferences\n          : [rawReferences];\n    return {\n      kind: 'located',\n      target,\n      ...(references?.length ? { references } : {}),\n      ...(error ? { error } : {}),\n    };\n  };\n\ntype DefaultLocateProtocolOptions = {\n  acceptBbox2dAlias?: boolean;\n};\n\nexport const createDefaultElementProtocol = (\n  { jsonParser }: StandardLocateProtocolContext,\n  options: DefaultLocateProtocolOptions = {},\n): StandardLocateProtocol => ({\n  systemPromptIntroduction: defaultLocateSystemPromptIntroduction,\n  buildResponseInstructions,\n  buildUserPrompt,\n  expectedJsonObjectResponse: true,\n  parseRawResponse: createParseRawResponse(jsonParser, 'locate', {\n    includeReferences: false,\n    acceptBbox2dAlias: options.acceptBbox2dAlias,\n  }),\n});\n\nexport const createDefaultSearchAreaProtocol = (\n  { jsonParser }: StandardLocateProtocolContext,\n  options: DefaultLocateProtocolOptions = {},\n): StandardLocateProtocol => ({\n  systemPromptIntroduction: defaultSearchAreaSystemPromptIntroduction,\n  buildResponseInstructions: buildSearchAreaResponseInstructions,\n  buildUserPrompt: buildSearchAreaUserPrompt,\n  expectedJsonObjectResponse: true,\n  parseRawResponse: createParseRawResponse(jsonParser, 'section-locator', {\n    includeReferences: true,\n    acceptBbox2dAlias: options.acceptBbox2dAlias,\n  }),\n});\n"],"names":["markdownCodeFence","defaultLocateSystemPromptIntroduction","defaultSearchAreaSystemPromptIntroduction","buildResponseInstructions","promptSpec","preferredLanguage","getPreferredLanguage","resultKey","exampleValueText","formatLocateExampleValue","resultFieldDescription","buildUserPrompt","targetElementDescription","buildSearchAreaResponseInstructions","resultJsonProperty","referenceFieldDescription","buildSearchAreaUserPrompt","sectionDescription","createParseRawResponse","jsonParser","source","options","content","parsedResponse","Error","record","target","undefined","error","hasTarget","Array","rawReferences","references","createDefaultElementProtocol","createDefaultSearchAreaProtocol"],"mappings":";;AAYA,MAAMA,oBAAoB;AAE1B,MAAMC,wCAAwC,CAAC;;;;;6EAK8B,CAAC;AAE9E,MAAMC,4CAA4C,CAAC;;;;;kGAK+C,CAAC;AAEnG,MAAMC,4BAA4B,CAACC;IACjC,MAAMC,oBAAoBC;IAC1B,MAAMC,YAAYH,WAAW,SAAS;IACtC,MAAMI,mBAAmBC,yBACvBL,WAAW,aAAa,CAAC,EAAE;IAE7B,MAAMM,yBAAyB,CAAC,IAAI,EAAEN,WAAW,UAAU,CAAC,mDAAmD,CAAC;IAChH,OAAO,CAAC;AACV,EAAEJ,kBAAkB;;GAEjB,EAAEO,UAAU,GAAG,EAAEH,WAAW,iBAAiB,CAAC,MAAM,EAAEA,WAAW,sBAAsB,CAAC;;;AAG3F,EAAEJ,kBAAkB;;;IAGhB,EAAEO,UAAU,MAAM,EAAEG,uBAAuB;;;;AAI/C,EAAEV,kBAAkB;;GAEjB,EAAEO,UAAU,GAAG,EAAEC,iBAAiB;;AAErC,EAAER,kBAAkB;;;AAGpB,EAAEA,kBAAkB;;GAEjB,EAAEO,UAAU;gEACiD,EAAEF,kBAAkB;;AAEpF,EAAEL,kBAAkB;AACpB,CAAC;AACD;AAEA,MAAMW,kBAAkB,CAACC,2BACvB,CAAC,MAAM,EAAEA,0BAA0B;AAErC,MAAMC,sCAAsC,CAC1CT;IAEA,MAAMC,oBAAoBC;IAC1B,MAAMC,YAAYH,WAAW,SAAS;IACtC,MAAMI,mBAAmBC,yBACvBL,WAAW,aAAa,CAAC,EAAE;IAE7B,MAAMU,qBAAqB,CAAC,CAAC,EAAEP,UAAU,GAAG,EAAEH,WAAW,iBAAiB,CAAC,MAAM,EAAEA,WAAW,sBAAsB,EAAE;IACtH,MAAMM,yBAAyB,GAAGN,WAAW,UAAU,CAAC,6CAA6C,CAAC;IACtG,MAAMW,4BAA4B,CAAC,kBAAkB,EAAEX,WAAW,gBAAgB,CAAC,sBAAsB,CAAC;IAC1G,OAAO,CAAC;AACV,EAAEJ,kBAAkB;;EAElB,EAAEc,mBAAmB;cACT,EAAEP,UAAU;IACtB,EAAEH,WAAW,iBAAiB,CAAC;;;;;AAKnC,EAAEJ,kBAAkB;;;IAGhB,EAAEO,UAAU,KAAK,EAAEG,uBAAuB;eAC/B,EAAEH,UAAU,KAAK,EAAEQ,0BAA0B;yEACa,EAAEV,kBAAkB;;;;AAI7F,EAAEL,kBAAkB;;GAEjB,EAAEO,UAAU,GAAG,EAAEC,iBAAiB;cACvB,EAAED,UAAU,IAAI,EAAEC,iBAAiB;;AAEjD,EAAER,kBAAkB;AACpB,CAAC;AACD;AAEA,MAAMgB,4BAA4B,CAACC,qBACjC,CAAC,yBAAyB,EAAEA,oBAAoB;AAOlD,MAAMC,yBACJ,CACEC,YACAC,QACAC,UAEF,CACEC,SACAlB;QAEA,MAAMmB,iBAAiBJ,WAAWG,SAAS;YACzCF;QACF;QACA,IAAI,CAACG,kBAAkB,AAA0B,YAA1B,OAAOA,gBAC5B,MAAM,IAAIC,MAAM,CAAC,sCAAsC,EAAEF,SAAS;QAEpE,MAAMG,SAASF;QACf,MAAMG,SACJD,AAAiCE,WAAjCF,MAAM,CAACrB,WAAW,SAAS,CAAC,GACxBqB,MAAM,CAACrB,WAAW,SAAS,CAAC,GAC5BiB,QAAQ,iBAAiB,IAAIjB,AAAyB,WAAzBA,WAAW,SAAS,GAC/CqB,OAAO,OAAO,GACdE;QACR,MAAMC,QAAQ,AAAwB,YAAxB,OAAOH,OAAO,KAAK,GAAgBA,OAAO,KAAK,GAAGE;QAChE,MAAME,YAAYC,MAAM,OAAO,CAACJ,UAC5BA,OAAO,MAAM,GAAG,IAChBA,AAAWC,WAAXD;QACJ,IAAI,CAACG,WACH,OAAO;YAAE,MAAM;YAAa,GAAID,QAAQ;gBAAEA;YAAM,IAAI,CAAC,CAAC;QAAE;QAG1D,IAAI,CAACP,QAAQ,iBAAiB,EAC5B,OAAO;YAAE,MAAM;YAAWK;YAAQ,GAAIE,QAAQ;gBAAEA;YAAM,IAAI,CAAC,CAAC;QAAE;QAGhE,MAAMG,gBAAgBN,MAAM,CAAC,CAAC,WAAW,EAAErB,WAAW,SAAS,EAAE,CAAC;QAClE,MAAM4B,aACJD,QAAAA,gBACIJ,SACAG,MAAM,OAAO,CAACC,iBACZA,gBACA;YAACA;SAAc;QACvB,OAAO;YACL,MAAM;YACNL;YACA,GAAIM,YAAY,SAAS;gBAAEA;YAAW,IAAI,CAAC,CAAC;YAC5C,GAAIJ,QAAQ;gBAAEA;YAAM,IAAI,CAAC,CAAC;QAC5B;IACF;AAMK,MAAMK,+BAA+B,CAC1C,EAAEd,UAAU,EAAiC,EAC7CE,UAAwC,CAAC,CAAC,GACd;QAC5B,0BAA0BpB;QAC1BE;QACAQ;QACA,4BAA4B;QAC5B,kBAAkBO,uBAAuBC,YAAY,UAAU;YAC7D,mBAAmB;YACnB,mBAAmBE,QAAQ,iBAAiB;QAC9C;IACF;AAEO,MAAMa,kCAAkC,CAC7C,EAAEf,UAAU,EAAiC,EAC7CE,UAAwC,CAAC,CAAC,GACd;QAC5B,0BAA0BnB;QAC1B,2BAA2BW;QAC3B,iBAAiBG;QACjB,4BAA4B;QAC5B,kBAAkBE,uBAAuBC,YAAY,mBAAmB;YACtE,mBAAmB;YACnB,mBAAmBE,QAAQ,iBAAiB;QAC9C;IACF"}

package/dist/es/ai-model/model-adapter/locate.mjs

@@ -1,13 +1,15 @@
-import { createLocateResultAdapter } from "../shared/model-locate-result/factory.mjs";
+import { createLocateResultCodec } from "../shared/model-locate-result/factory.mjs";
 import { resolvePlanningTapLocator } from "../workflows/grounding/planning-action-locate.mjs";
-const defaultLocateResultAdapterDefinition = {
+import { createDefaultElementProtocol, createDefaultSearchAreaProtocol } from "./default-locate-protocol.mjs";
+const defaultLocateResultFormatDefinition = {
     coordinates: {
         shape: 'bbox',
         order: 'xy',
         normalizedBy: 1000
     }
 };
-function resolveLocate(locate, resolvedCustomPlanner) {
+const resolveLocateProtocolDefinition = (protocolDefinition, protocolContext)=>'function' == typeof protocolDefinition ? protocolDefinition(protocolContext) : protocolDefinition;
+function resolveLocate(locate, resolvedCustomPlanner, protocolContext, acceptBbox2dAlias = false) {
     if (locate?.kind === 'custom') {
         let locateFn = locate.locateFn;
         if (!locateFn) {
@@ -18,14 +20,31 @@ function resolveLocate(locate, resolvedCustomPlanner) {
         }
         return {
             kind: 'custom',
-            supportsSearchArea: locate.supportsSearchArea ?? false,
             locateFn
         };
     }
+    const elementProtocol = resolveLocateProtocolDefinition(locate?.element?.protocol ?? ((context)=>createDefaultElementProtocol(context, {
+            acceptBbox2dAlias
+        })), protocolContext);
+    const elementResultFormat = locate?.element?.resultFormat ?? defaultLocateResultFormatDefinition;
+    const elementResultCodec = createLocateResultCodec(elementResultFormat);
+    const searchAreaProtocol = locate?.searchArea === false ? void 0 : resolveLocateProtocolDefinition(locate?.searchArea?.protocol ?? ((context)=>createDefaultSearchAreaProtocol(context, {
+            acceptBbox2dAlias
+        })), protocolContext);
+    const searchAreaResultFormat = locate?.searchArea === false ? void 0 : locate?.searchArea?.resultFormat ?? elementResultFormat;
+    const searchAreaResultCodec = searchAreaResultFormat ? createLocateResultCodec(searchAreaResultFormat) : void 0;
     return {
         kind: 'standard',
-        supportsSearchArea: locate?.supportsSearchArea ?? true,
-        resultAdapter: createLocateResultAdapter(locate?.resultAdapter ?? defaultLocateResultAdapterDefinition)
+        element: {
+            protocol: elementProtocol,
+            resultCodec: elementResultCodec
+        },
+        ...searchAreaProtocol && searchAreaResultCodec ? {
+            searchArea: {
+                protocol: searchAreaProtocol,
+                resultCodec: searchAreaResultCodec
+            }
+        } : {}
     };
 }
 export { resolveLocate };

package/dist/es/ai-model/model-adapter/locate.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/model-adapter/locate.mjs","sources":["../../../../src/ai-model/model-adapter/locate.ts"],"sourcesContent":["import { createLocateResultAdapter } from '../shared/model-locate-result/factory';\nimport type { LocateResultAdapterDefinition } from '../shared/model-locate-result/types';\nimport { resolvePlanningTapLocator } from '../workflows/grounding/planning-action-locate';\nimport type { ResolvedCustomPlanningDefinition } from './custom-planning-types';\nimport type { LocateAdapter, ModelAdapterDefinition } from './types';\n\nconst defaultLocateResultAdapterDefinition: LocateResultAdapterDefinition = {\n  coordinates: { shape: 'bbox', order: 'xy', normalizedBy: 1000 },\n};\n\nexport function resolveLocate(\n  locate: ModelAdapterDefinition['locate'],\n  resolvedCustomPlanner: ResolvedCustomPlanningDefinition | undefined,\n): LocateAdapter {\n  if (locate?.kind === 'custom') {\n    let locateFn = locate.locateFn;\n\n    if (!locateFn) {\n      const planningTapLocator = locate.planningTapLocator;\n\n      if (!planningTapLocator) {\n        throw new Error(\n          'Custom locate definition requires either locateFn or planningTapLocator',\n        );\n      }\n\n      if (!resolvedCustomPlanner) {\n        throw new Error(\n          'Custom planning tap locator requires a custom planning planner definition',\n        );\n      }\n      locateFn = resolvePlanningTapLocator(\n        planningTapLocator,\n        resolvedCustomPlanner,\n      );\n    }\n\n    return {\n      kind: 'custom',\n      supportsSearchArea: locate.supportsSearchArea ?? false,\n      locateFn,\n    };\n  }\n\n  return {\n    kind: 'standard',\n    supportsSearchArea: locate?.supportsSearchArea ?? true,\n    resultAdapter: createLocateResultAdapter(\n      locate?.resultAdapter ?? defaultLocateResultAdapterDefinition,\n    ),\n  };\n}\n"],"names":["defaultLocateResultAdapterDefinition","resolveLocate","locate","resolvedCustomPlanner","locateFn","planningTapLocator","Error","resolvePlanningTapLocator","createLocateResultAdapter"],"mappings":";;AAMA,MAAMA,uCAAsE;IAC1E,aAAa;QAAE,OAAO;QAAQ,OAAO;QAAM,cAAc;IAAK;AAChE;AAEO,SAASC,cACdC,MAAwC,EACxCC,qBAAmE;IAEnE,IAAID,QAAQ,SAAS,UAAU;QAC7B,IAAIE,WAAWF,OAAO,QAAQ;QAE9B,IAAI,CAACE,UAAU;YACb,MAAMC,qBAAqBH,OAAO,kBAAkB;YAEpD,IAAI,CAACG,oBACH,MAAM,IAAIC,MACR;YAIJ,IAAI,CAACH,uBACH,MAAM,IAAIG,MACR;YAGJF,WAAWG,0BACTF,oBACAF;QAEJ;QAEA,OAAO;YACL,MAAM;YACN,oBAAoBD,OAAO,kBAAkB,IAAI;YACjDE;QACF;IACF;IAEA,OAAO;QACL,MAAM;QACN,oBAAoBF,QAAQ,sBAAsB;QAClD,eAAeM,0BACbN,QAAQ,iBAAiBF;IAE7B;AACF"}
+{"version":3,"file":"ai-model/model-adapter/locate.mjs","sources":["../../../../src/ai-model/model-adapter/locate.ts"],"sourcesContent":["import { createLocateResultCodec } from '../shared/model-locate-result/factory';\nimport type { LocateResultFormatDefinition } from '../shared/model-locate-result/types';\nimport { resolvePlanningTapLocator } from '../workflows/grounding/planning-action-locate';\nimport type { ResolvedCustomPlanningDefinition } from './custom-planning-types';\nimport {\n  createDefaultElementProtocol,\n  createDefaultSearchAreaProtocol,\n} from './default-locate-protocol';\nimport type {\n  StandardLocateProtocolContext,\n  StandardLocateProtocolDefinition,\n} from './locate-protocol';\nimport type { LocateAdapter, ModelAdapterDefinition } from './types';\n\nconst defaultLocateResultFormatDefinition: LocateResultFormatDefinition = {\n  coordinates: { shape: 'bbox', order: 'xy', normalizedBy: 1000 },\n};\n\nconst resolveLocateProtocolDefinition = (\n  protocolDefinition: StandardLocateProtocolDefinition,\n  protocolContext: StandardLocateProtocolContext,\n) =>\n  typeof protocolDefinition === 'function'\n    ? protocolDefinition(protocolContext)\n    : protocolDefinition;\n\nexport function resolveLocate(\n  locate: ModelAdapterDefinition['locate'],\n  resolvedCustomPlanner: ResolvedCustomPlanningDefinition | undefined,\n  protocolContext: StandardLocateProtocolContext,\n  acceptBbox2dAlias = false,\n): LocateAdapter {\n  if (locate?.kind === 'custom') {\n    let locateFn = locate.locateFn;\n\n    if (!locateFn) {\n      const planningTapLocator = locate.planningTapLocator;\n\n      if (!planningTapLocator) {\n        throw new Error(\n          'Custom locate definition requires either locateFn or planningTapLocator',\n        );\n      }\n\n      if (!resolvedCustomPlanner) {\n        throw new Error(\n          'Custom planning tap locator requires a custom planning planner definition',\n        );\n      }\n      locateFn = resolvePlanningTapLocator(\n        planningTapLocator,\n        resolvedCustomPlanner,\n      );\n    }\n\n    return {\n      kind: 'custom',\n      locateFn,\n    };\n  }\n\n  const elementProtocol = resolveLocateProtocolDefinition(\n    locate?.element?.protocol ??\n      ((context) =>\n        createDefaultElementProtocol(context, { acceptBbox2dAlias })),\n    protocolContext,\n  );\n  const elementResultFormat =\n    locate?.element?.resultFormat ?? defaultLocateResultFormatDefinition;\n  const elementResultCodec = createLocateResultCodec(elementResultFormat);\n  const searchAreaProtocol =\n    locate?.searchArea === false\n      ? undefined\n      : resolveLocateProtocolDefinition(\n          locate?.searchArea?.protocol ??\n            ((context) =>\n              createDefaultSearchAreaProtocol(context, {\n                acceptBbox2dAlias,\n              })),\n          protocolContext,\n        );\n  const searchAreaResultFormat =\n    locate?.searchArea === false\n      ? undefined\n      : (locate?.searchArea?.resultFormat ?? elementResultFormat);\n  const searchAreaResultCodec = searchAreaResultFormat\n    ? createLocateResultCodec(searchAreaResultFormat)\n    : undefined;\n\n  return {\n    kind: 'standard',\n    element: {\n      protocol: elementProtocol,\n      resultCodec: elementResultCodec,\n    },\n    ...(searchAreaProtocol && searchAreaResultCodec\n      ? {\n          searchArea: {\n            protocol: searchAreaProtocol,\n            resultCodec: searchAreaResultCodec,\n          },\n        }\n      : {}),\n  };\n}\n"],"names":["defaultLocateResultFormatDefinition","resolveLocateProtocolDefinition","protocolDefinition","protocolContext","resolveLocate","locate","resolvedCustomPlanner","acceptBbox2dAlias","locateFn","planningTapLocator","Error","resolvePlanningTapLocator","elementProtocol","context","createDefaultElementProtocol","elementResultFormat","elementResultCodec","createLocateResultCodec","searchAreaProtocol","undefined","createDefaultSearchAreaProtocol","searchAreaResultFormat","searchAreaResultCodec"],"mappings":";;;AAcA,MAAMA,sCAAoE;IACxE,aAAa;QAAE,OAAO;QAAQ,OAAO;QAAM,cAAc;IAAK;AAChE;AAEA,MAAMC,kCAAkC,CACtCC,oBACAC,kBAEA,AAA8B,cAA9B,OAAOD,qBACHA,mBAAmBC,mBACnBD;AAEC,SAASE,cACdC,MAAwC,EACxCC,qBAAmE,EACnEH,eAA8C,EAC9CI,oBAAoB,KAAK;IAEzB,IAAIF,QAAQ,SAAS,UAAU;QAC7B,IAAIG,WAAWH,OAAO,QAAQ;QAE9B,IAAI,CAACG,UAAU;YACb,MAAMC,qBAAqBJ,OAAO,kBAAkB;YAEpD,IAAI,CAACI,oBACH,MAAM,IAAIC,MACR;YAIJ,IAAI,CAACJ,uBACH,MAAM,IAAII,MACR;YAGJF,WAAWG,0BACTF,oBACAH;QAEJ;QAEA,OAAO;YACL,MAAM;YACNE;QACF;IACF;IAEA,MAAMI,kBAAkBX,gCACtBI,QAAQ,SAAS,YACb,EAAAQ,UACAC,6BAA6BD,SAAS;YAAEN;QAAkB,EAAC,GAC/DJ;IAEF,MAAMY,sBACJV,QAAQ,SAAS,gBAAgBL;IACnC,MAAMgB,qBAAqBC,wBAAwBF;IACnD,MAAMG,qBACJb,QAAQ,eAAe,QACnBc,SACAlB,gCACEI,QAAQ,YAAY,YAChB,EAAAQ,UACAO,gCAAgCP,SAAS;YACvCN;QACF,EAAC,GACLJ;IAER,MAAMkB,yBACJhB,QAAQ,eAAe,QACnBc,SACCd,QAAQ,YAAY,gBAAgBU;IAC3C,MAAMO,wBAAwBD,yBAC1BJ,wBAAwBI,0BACxBF;IAEJ,OAAO;QACL,MAAM;QACN,SAAS;YACP,UAAUP;YACV,aAAaI;QACf;QACA,GAAIE,sBAAsBI,wBACtB;YACE,YAAY;gBACV,UAAUJ;gBACV,aAAaI;YACf;QACF,IACA,CAAC,CAAC;IACR;AACF"}

package/dist/es/ai-model/model-adapter/planning.mjs

@@ -1,11 +1,11 @@
-import { createLocateResultAdapter, resolveLocateResultCoordinates } from "../shared/model-locate-result/index.mjs";
+import { createLocateResultCodec, resolveLocateResultCoordinates } from "../shared/model-locate-result/index.mjs";
 import { runCustomPlanning } from "../workflows/planning/custom-planning.mjs";
 import { createDefaultMidscenePlanningProtocol } from "./default-planning-protocol.mjs";
 const defaultReplanningCycleLimit = 20;
 function resolveCustomPlanningDefinition(config) {
     const { coordinates, ...rest } = config;
     const coordinateSystem = resolveLocateResultCoordinates(coordinates);
-    const coordinateNormalizer = createLocateResultAdapter({
+    const coordinateNormalizer = createLocateResultCodec({
         coordinates
     });
     return {
@@ -14,7 +14,7 @@ function resolveCustomPlanningDefinition(config) {
         coordinateNormalizer
     };
 }
-function resolvePlanning(planning, resolvedCustomPlanner, protocolContext) {
+function resolvePlanning(planning, resolvedCustomPlanner, protocolContext, defaultLocateResultCodec) {
     if (planning?.kind === 'custom') {
         if ('function' == typeof planning.planFn) return {
             kind: 'custom',
@@ -34,12 +34,14 @@ function resolvePlanning(planning, resolvedCustomPlanner, protocolContext) {
         };
     }
     const protocolDefinition = planning?.protocol ?? createDefaultMidscenePlanningProtocol;
+    const locateResultCodec = planning?.locateResultFormat === false ? void 0 : planning?.locateResultFormat ? createLocateResultCodec(planning.locateResultFormat) : defaultLocateResultCodec;
     return {
         kind: 'standard',
         cacheEnabled: planning?.cacheEnabled ?? true,
         defaultReplanningCycleLimit: planning?.defaultReplanningCycleLimit ?? defaultReplanningCycleLimit,
         supportsActionDeepLocate: planning?.supportsActionDeepLocate ?? true,
-        protocol: 'function' == typeof protocolDefinition ? protocolDefinition(protocolContext) : protocolDefinition
+        protocol: 'function' == typeof protocolDefinition ? protocolDefinition(protocolContext) : protocolDefinition,
+        locateResultCodec
     };
 }
 export { resolveCustomPlanningDefinition, resolvePlanning };

package/dist/es/ai-model/model-adapter/planning.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/model-adapter/planning.mjs","sources":["../../../../src/ai-model/model-adapter/planning.ts"],"sourcesContent":["import {\n  createLocateResultAdapter,\n  resolveLocateResultCoordinates,\n} from '../shared/model-locate-result';\nimport { runCustomPlanning } from '../workflows/planning/custom-planning';\nimport type {\n  CustomPlanningDefinition,\n  ResolvedCustomPlanningDefinition,\n} from './custom-planning-types';\nimport { createDefaultMidscenePlanningProtocol } from './default-planning-protocol';\nimport type { StandardPlanningProtocolContext } from './planning-protocol';\nimport type { ModelAdapterDefinition, PlanningAdapter } from './types';\n\nconst defaultReplanningCycleLimit = 20;\n\nexport function resolveCustomPlanningDefinition<TParsed>(\n  config: CustomPlanningDefinition<TParsed>,\n): ResolvedCustomPlanningDefinition<TParsed> {\n  const { coordinates, ...rest } = config;\n  const coordinateSystem = resolveLocateResultCoordinates(coordinates);\n  const coordinateNormalizer = createLocateResultAdapter({ coordinates });\n  return {\n    ...rest,\n    coordinateSystem,\n    coordinateNormalizer,\n  };\n}\n\nexport function resolvePlanning(\n  planning: ModelAdapterDefinition['planning'],\n  resolvedCustomPlanner: ResolvedCustomPlanningDefinition | undefined,\n  protocolContext: StandardPlanningProtocolContext,\n): PlanningAdapter {\n  if (planning?.kind === 'custom') {\n    if (typeof planning.planFn === 'function') {\n      return {\n        kind: 'custom',\n        cacheEnabled: planning.cacheEnabled ?? false,\n        defaultReplanningCycleLimit:\n          planning.defaultReplanningCycleLimit ?? defaultReplanningCycleLimit,\n        supportsActionDeepLocate: planning.supportsActionDeepLocate ?? false,\n        planFn: planning.planFn,\n      };\n    }\n\n    if (!resolvedCustomPlanner) {\n      throw new Error('Custom planning planner definition is not resolved');\n    }\n\n    return {\n      kind: 'custom',\n      cacheEnabled: planning.cacheEnabled ?? false,\n      defaultReplanningCycleLimit:\n        planning.defaultReplanningCycleLimit ?? defaultReplanningCycleLimit,\n      supportsActionDeepLocate: planning.supportsActionDeepLocate ?? false,\n      coordinateSystem: resolvedCustomPlanner.coordinateSystem,\n      planFn: (userInstruction, options) =>\n        runCustomPlanning(userInstruction, options, resolvedCustomPlanner),\n    };\n  }\n\n  const protocolDefinition =\n    planning?.protocol ?? createDefaultMidscenePlanningProtocol;\n\n  return {\n    kind: 'standard',\n    cacheEnabled: planning?.cacheEnabled ?? true,\n    defaultReplanningCycleLimit:\n      planning?.defaultReplanningCycleLimit ?? defaultReplanningCycleLimit,\n    supportsActionDeepLocate: planning?.supportsActionDeepLocate ?? true,\n    protocol:\n      typeof protocolDefinition === 'function'\n        ? protocolDefinition(protocolContext)\n        : protocolDefinition,\n  };\n}\n"],"names":["defaultReplanningCycleLimit","resolveCustomPlanningDefinition","config","coordinates","rest","coordinateSystem","resolveLocateResultCoordinates","coordinateNormalizer","createLocateResultAdapter","resolvePlanning","planning","resolvedCustomPlanner","protocolContext","Error","userInstruction","options","runCustomPlanning","protocolDefinition","createDefaultMidscenePlanningProtocol"],"mappings":";;;AAaA,MAAMA,8BAA8B;AAE7B,SAASC,gCACdC,MAAyC;IAEzC,MAAM,EAAEC,WAAW,EAAE,GAAGC,MAAM,GAAGF;IACjC,MAAMG,mBAAmBC,+BAA+BH;IACxD,MAAMI,uBAAuBC,0BAA0B;QAAEL;IAAY;IACrE,OAAO;QACL,GAAGC,IAAI;QACPC;QACAE;IACF;AACF;AAEO,SAASE,gBACdC,QAA4C,EAC5CC,qBAAmE,EACnEC,eAAgD;IAEhD,IAAIF,UAAU,SAAS,UAAU;QAC/B,IAAI,AAA2B,cAA3B,OAAOA,SAAS,MAAM,EACxB,OAAO;YACL,MAAM;YACN,cAAcA,SAAS,YAAY,IAAI;YACvC,6BACEA,SAAS,2BAA2B,IAAIV;YAC1C,0BAA0BU,SAAS,wBAAwB,IAAI;YAC/D,QAAQA,SAAS,MAAM;QACzB;QAGF,IAAI,CAACC,uBACH,MAAM,IAAIE,MAAM;QAGlB,OAAO;YACL,MAAM;YACN,cAAcH,SAAS,YAAY,IAAI;YACvC,6BACEA,SAAS,2BAA2B,IAAIV;YAC1C,0BAA0BU,SAAS,wBAAwB,IAAI;YAC/D,kBAAkBC,sBAAsB,gBAAgB;YACxD,QAAQ,CAACG,iBAAiBC,UACxBC,kBAAkBF,iBAAiBC,SAASJ;QAChD;IACF;IAEA,MAAMM,qBACJP,UAAU,YAAYQ;IAExB,OAAO;QACL,MAAM;QACN,cAAcR,UAAU,gBAAgB;QACxC,6BACEA,UAAU,+BAA+BV;QAC3C,0BAA0BU,UAAU,4BAA4B;QAChE,UACE,AAA8B,cAA9B,OAAOO,qBACHA,mBAAmBL,mBACnBK;IACR;AACF"}
+{"version":3,"file":"ai-model/model-adapter/planning.mjs","sources":["../../../../src/ai-model/model-adapter/planning.ts"],"sourcesContent":["import {\n  createLocateResultCodec,\n  resolveLocateResultCoordinates,\n} from '../shared/model-locate-result';\nimport type { LocateResultCodec } from '../shared/model-locate-result';\nimport { runCustomPlanning } from '../workflows/planning/custom-planning';\nimport type {\n  CustomPlanningDefinition,\n  ResolvedCustomPlanningDefinition,\n} from './custom-planning-types';\nimport { createDefaultMidscenePlanningProtocol } from './default-planning-protocol';\nimport type { StandardPlanningProtocolContext } from './planning-protocol';\nimport type { ModelAdapterDefinition, PlanningAdapter } from './types';\n\nconst defaultReplanningCycleLimit = 20;\n\nexport function resolveCustomPlanningDefinition<TParsed>(\n  config: CustomPlanningDefinition<TParsed>,\n): ResolvedCustomPlanningDefinition<TParsed> {\n  const { coordinates, ...rest } = config;\n  const coordinateSystem = resolveLocateResultCoordinates(coordinates);\n  const coordinateNormalizer = createLocateResultCodec({ coordinates });\n  return {\n    ...rest,\n    coordinateSystem,\n    coordinateNormalizer,\n  };\n}\n\nexport function resolvePlanning(\n  planning: ModelAdapterDefinition['planning'],\n  resolvedCustomPlanner: ResolvedCustomPlanningDefinition | undefined,\n  protocolContext: StandardPlanningProtocolContext,\n  defaultLocateResultCodec?: LocateResultCodec,\n): PlanningAdapter {\n  if (planning?.kind === 'custom') {\n    if (typeof planning.planFn === 'function') {\n      return {\n        kind: 'custom',\n        cacheEnabled: planning.cacheEnabled ?? false,\n        defaultReplanningCycleLimit:\n          planning.defaultReplanningCycleLimit ?? defaultReplanningCycleLimit,\n        supportsActionDeepLocate: planning.supportsActionDeepLocate ?? false,\n        planFn: planning.planFn,\n      };\n    }\n\n    if (!resolvedCustomPlanner) {\n      throw new Error('Custom planning planner definition is not resolved');\n    }\n\n    return {\n      kind: 'custom',\n      cacheEnabled: planning.cacheEnabled ?? false,\n      defaultReplanningCycleLimit:\n        planning.defaultReplanningCycleLimit ?? defaultReplanningCycleLimit,\n      supportsActionDeepLocate: planning.supportsActionDeepLocate ?? false,\n      coordinateSystem: resolvedCustomPlanner.coordinateSystem,\n      planFn: (userInstruction, options) =>\n        runCustomPlanning(userInstruction, options, resolvedCustomPlanner),\n    };\n  }\n\n  const protocolDefinition =\n    planning?.protocol ?? createDefaultMidscenePlanningProtocol;\n  const locateResultCodec =\n    planning?.locateResultFormat === false\n      ? undefined\n      : planning?.locateResultFormat\n        ? createLocateResultCodec(planning.locateResultFormat)\n        : defaultLocateResultCodec;\n\n  return {\n    kind: 'standard',\n    cacheEnabled: planning?.cacheEnabled ?? true,\n    defaultReplanningCycleLimit:\n      planning?.defaultReplanningCycleLimit ?? defaultReplanningCycleLimit,\n    supportsActionDeepLocate: planning?.supportsActionDeepLocate ?? true,\n    protocol:\n      typeof protocolDefinition === 'function'\n        ? protocolDefinition(protocolContext)\n        : protocolDefinition,\n    locateResultCodec,\n  };\n}\n"],"names":["defaultReplanningCycleLimit","resolveCustomPlanningDefinition","config","coordinates","rest","coordinateSystem","resolveLocateResultCoordinates","coordinateNormalizer","createLocateResultCodec","resolvePlanning","planning","resolvedCustomPlanner","protocolContext","defaultLocateResultCodec","Error","userInstruction","options","runCustomPlanning","protocolDefinition","createDefaultMidscenePlanningProtocol","locateResultCodec","undefined"],"mappings":";;;AAcA,MAAMA,8BAA8B;AAE7B,SAASC,gCACdC,MAAyC;IAEzC,MAAM,EAAEC,WAAW,EAAE,GAAGC,MAAM,GAAGF;IACjC,MAAMG,mBAAmBC,+BAA+BH;IACxD,MAAMI,uBAAuBC,wBAAwB;QAAEL;IAAY;IACnE,OAAO;QACL,GAAGC,IAAI;QACPC;QACAE;IACF;AACF;AAEO,SAASE,gBACdC,QAA4C,EAC5CC,qBAAmE,EACnEC,eAAgD,EAChDC,wBAA4C;IAE5C,IAAIH,UAAU,SAAS,UAAU;QAC/B,IAAI,AAA2B,cAA3B,OAAOA,SAAS,MAAM,EACxB,OAAO;YACL,MAAM;YACN,cAAcA,SAAS,YAAY,IAAI;YACvC,6BACEA,SAAS,2BAA2B,IAAIV;YAC1C,0BAA0BU,SAAS,wBAAwB,IAAI;YAC/D,QAAQA,SAAS,MAAM;QACzB;QAGF,IAAI,CAACC,uBACH,MAAM,IAAIG,MAAM;QAGlB,OAAO;YACL,MAAM;YACN,cAAcJ,SAAS,YAAY,IAAI;YACvC,6BACEA,SAAS,2BAA2B,IAAIV;YAC1C,0BAA0BU,SAAS,wBAAwB,IAAI;YAC/D,kBAAkBC,sBAAsB,gBAAgB;YACxD,QAAQ,CAACI,iBAAiBC,UACxBC,kBAAkBF,iBAAiBC,SAASL;QAChD;IACF;IAEA,MAAMO,qBACJR,UAAU,YAAYS;IACxB,MAAMC,oBACJV,UAAU,uBAAuB,QAC7BW,SACAX,UAAU,qBACRF,wBAAwBE,SAAS,kBAAkB,IACnDG;IAER,OAAO;QACL,MAAM;QACN,cAAcH,UAAU,gBAAgB;QACxC,6BACEA,UAAU,+BAA+BV;QAC3C,0BAA0BU,UAAU,4BAA4B;QAChE,UACE,AAA8B,cAA9B,OAAOQ,qBACHA,mBAAmBN,mBACnBM;QACNE;IACF;AACF"}

package/dist/es/ai-model/model-adapter/resolve.mjs

@@ -26,18 +26,22 @@ class ResolvedModelAdapter {
     constructor(config, modelFamily){
         _define_property(this, "jsonParser", void 0);
         _define_property(this, "chatCompletion", void 0);
+        _define_property(this, "acceptBbox2dAlias", void 0);
         _define_property(this, "imagePreprocess", void 0);
         _define_property(this, "planning", void 0);
         _define_property(this, "locate", void 0);
         this.jsonParser = resolveJsonParser(config.jsonParser);
         this.chatCompletion = resolveChatCompletion(config.chatCompletion);
+        this.acceptBbox2dAlias = config.acceptBbox2dAlias ?? false;
         this.imagePreprocess = resolveImagePreprocess(config.imagePreprocess);
         const customPlanner = config.planning?.kind === 'custom' ? config.planning.planner : void 0;
         const resolvedCustomPlanner = customPlanner ? resolveCustomPlanningDefinition(customPlanner) : void 0;
+        this.locate = resolveLocate(config.locate, resolvedCustomPlanner, {
+            jsonParser: this.jsonParser
+        }, this.acceptBbox2dAlias);
         this.planning = resolvePlanning(config.planning, resolvedCustomPlanner, {
             jsonParser: this.jsonParser
-        });
-        this.locate = resolveLocate(config.locate, resolvedCustomPlanner);
+        }, 'standard' === this.locate.kind ? this.locate.element.resultCodec : void 0);
     }
 }
 export { ResolvedModelAdapter };

package/dist/es/ai-model/model-adapter/resolve.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/model-adapter/resolve.mjs","sources":["../../../../src/ai-model/model-adapter/resolve.ts"],"sourcesContent":["import { parseModelResponseJson } from '../shared/json';\nimport { resolveChatCompletion } from './chat-completion';\nimport { resolveLocate } from './locate';\nimport { resolveCustomPlanningDefinition, resolvePlanning } from './planning';\nimport type {\n  ChatCompletionAdapter,\n  ImagePreprocessPolicy,\n  JsonParser,\n  LocateAdapter,\n  ModelAdapter,\n  ModelAdapterDefinition,\n  PlanningAdapter,\n} from './types';\n\nfunction resolveJsonParser(\n  jsonParser: ModelAdapterDefinition['jsonParser'],\n): JsonParser {\n  if (!jsonParser || jsonParser === 'lenient-json') {\n    return parseModelResponseJson;\n  }\n\n  if (typeof jsonParser === 'function') {\n    return jsonParser;\n  }\n\n  throw new Error(`Unknown json parser preset: ${jsonParser}`);\n}\n\nfunction resolveImagePreprocess(\n  imagePreprocess: ModelAdapterDefinition['imagePreprocess'],\n): ImagePreprocessPolicy {\n  return {\n    padBlockSize: imagePreprocess?.padBlockSize,\n  };\n}\n\nexport class ResolvedModelAdapter implements ModelAdapter {\n  readonly jsonParser: JsonParser;\n  readonly chatCompletion: ChatCompletionAdapter;\n  readonly imagePreprocess: ImagePreprocessPolicy;\n  readonly planning: PlanningAdapter;\n  readonly locate: LocateAdapter;\n\n  constructor(config: ModelAdapterDefinition, modelFamily: string) {\n    this.jsonParser = resolveJsonParser(config.jsonParser);\n    this.chatCompletion = resolveChatCompletion(config.chatCompletion);\n    this.imagePreprocess = resolveImagePreprocess(config.imagePreprocess);\n    const customPlanner =\n      config.planning?.kind === 'custom' ? config.planning.planner : undefined;\n    const resolvedCustomPlanner = customPlanner\n      ? resolveCustomPlanningDefinition(customPlanner)\n      : undefined;\n    this.planning = resolvePlanning(config.planning, resolvedCustomPlanner, {\n      jsonParser: this.jsonParser,\n    });\n    this.locate = resolveLocate(config.locate, resolvedCustomPlanner);\n  }\n}\n"],"names":["resolveJsonParser","jsonParser","parseModelResponseJson","Error","resolveImagePreprocess","imagePreprocess","ResolvedModelAdapter","config","modelFamily","resolveChatCompletion","customPlanner","undefined","resolvedCustomPlanner","resolveCustomPlanningDefinition","resolvePlanning","resolveLocate"],"mappings":";;;;;;;;;;;;;;AAcA,SAASA,kBACPC,UAAgD;IAEhD,IAAI,CAACA,cAAcA,AAAe,mBAAfA,YACjB,OAAOC;IAGT,IAAI,AAAsB,cAAtB,OAAOD,YACT,OAAOA;IAGT,MAAM,IAAIE,MAAM,CAAC,4BAA4B,EAAEF,YAAY;AAC7D;AAEA,SAASG,uBACPC,eAA0D;IAE1D,OAAO;QACL,cAAcA,iBAAiB;IACjC;AACF;AAEO,MAAMC;IAOX,YAAYC,MAA8B,EAAEC,WAAmB,CAAE;QANjE,uBAAS,cAAT;QACA,uBAAS,kBAAT;QACA,uBAAS,mBAAT;QACA,uBAAS,YAAT;QACA,uBAAS,UAAT;QAGE,IAAI,CAAC,UAAU,GAAGR,kBAAkBO,OAAO,UAAU;QACrD,IAAI,CAAC,cAAc,GAAGE,sBAAsBF,OAAO,cAAc;QACjE,IAAI,CAAC,eAAe,GAAGH,uBAAuBG,OAAO,eAAe;QACpE,MAAMG,gBACJH,OAAO,QAAQ,EAAE,SAAS,WAAWA,OAAO,QAAQ,CAAC,OAAO,GAAGI;QACjE,MAAMC,wBAAwBF,gBAC1BG,gCAAgCH,iBAChCC;QACJ,IAAI,CAAC,QAAQ,GAAGG,gBAAgBP,OAAO,QAAQ,EAAEK,uBAAuB;YACtE,YAAY,IAAI,CAAC,UAAU;QAC7B;QACA,IAAI,CAAC,MAAM,GAAGG,cAAcR,OAAO,MAAM,EAAEK;IAC7C;AACF"}
+{"version":3,"file":"ai-model/model-adapter/resolve.mjs","sources":["../../../../src/ai-model/model-adapter/resolve.ts"],"sourcesContent":["import { parseModelResponseJson } from '../shared/json';\nimport { resolveChatCompletion } from './chat-completion';\nimport { resolveLocate } from './locate';\nimport { resolveCustomPlanningDefinition, resolvePlanning } from './planning';\nimport type {\n  ChatCompletionAdapter,\n  ImagePreprocessPolicy,\n  JsonParser,\n  LocateAdapter,\n  ModelAdapter,\n  ModelAdapterDefinition,\n  PlanningAdapter,\n} from './types';\n\nfunction resolveJsonParser(\n  jsonParser: ModelAdapterDefinition['jsonParser'],\n): JsonParser {\n  if (!jsonParser || jsonParser === 'lenient-json') {\n    return parseModelResponseJson;\n  }\n\n  if (typeof jsonParser === 'function') {\n    return jsonParser;\n  }\n\n  throw new Error(`Unknown json parser preset: ${jsonParser}`);\n}\n\nfunction resolveImagePreprocess(\n  imagePreprocess: ModelAdapterDefinition['imagePreprocess'],\n): ImagePreprocessPolicy {\n  return {\n    padBlockSize: imagePreprocess?.padBlockSize,\n  };\n}\n\nexport class ResolvedModelAdapter implements ModelAdapter {\n  readonly jsonParser: JsonParser;\n  readonly chatCompletion: ChatCompletionAdapter;\n  readonly acceptBbox2dAlias: boolean;\n  readonly imagePreprocess: ImagePreprocessPolicy;\n  readonly planning: PlanningAdapter;\n  readonly locate: LocateAdapter;\n\n  constructor(config: ModelAdapterDefinition, modelFamily: string) {\n    this.jsonParser = resolveJsonParser(config.jsonParser);\n    this.chatCompletion = resolveChatCompletion(config.chatCompletion);\n    this.acceptBbox2dAlias = config.acceptBbox2dAlias ?? false;\n    this.imagePreprocess = resolveImagePreprocess(config.imagePreprocess);\n    const customPlanner =\n      config.planning?.kind === 'custom' ? config.planning.planner : undefined;\n    const resolvedCustomPlanner = customPlanner\n      ? resolveCustomPlanningDefinition(customPlanner)\n      : undefined;\n    this.locate = resolveLocate(\n      config.locate,\n      resolvedCustomPlanner,\n      {\n        jsonParser: this.jsonParser,\n      },\n      this.acceptBbox2dAlias,\n    );\n    this.planning = resolvePlanning(\n      config.planning,\n      resolvedCustomPlanner,\n      { jsonParser: this.jsonParser },\n      this.locate.kind === 'standard'\n        ? this.locate.element.resultCodec\n        : undefined,\n    );\n  }\n}\n"],"names":["resolveJsonParser","jsonParser","parseModelResponseJson","Error","resolveImagePreprocess","imagePreprocess","ResolvedModelAdapter","config","modelFamily","resolveChatCompletion","customPlanner","undefined","resolvedCustomPlanner","resolveCustomPlanningDefinition","resolveLocate","resolvePlanning"],"mappings":";;;;;;;;;;;;;;AAcA,SAASA,kBACPC,UAAgD;IAEhD,IAAI,CAACA,cAAcA,AAAe,mBAAfA,YACjB,OAAOC;IAGT,IAAI,AAAsB,cAAtB,OAAOD,YACT,OAAOA;IAGT,MAAM,IAAIE,MAAM,CAAC,4BAA4B,EAAEF,YAAY;AAC7D;AAEA,SAASG,uBACPC,eAA0D;IAE1D,OAAO;QACL,cAAcA,iBAAiB;IACjC;AACF;AAEO,MAAMC;IAQX,YAAYC,MAA8B,EAAEC,WAAmB,CAAE;QAPjE,uBAAS,cAAT;QACA,uBAAS,kBAAT;QACA,uBAAS,qBAAT;QACA,uBAAS,mBAAT;QACA,uBAAS,YAAT;QACA,uBAAS,UAAT;QAGE,IAAI,CAAC,UAAU,GAAGR,kBAAkBO,OAAO,UAAU;QACrD,IAAI,CAAC,cAAc,GAAGE,sBAAsBF,OAAO,cAAc;QACjE,IAAI,CAAC,iBAAiB,GAAGA,OAAO,iBAAiB,IAAI;QACrD,IAAI,CAAC,eAAe,GAAGH,uBAAuBG,OAAO,eAAe;QACpE,MAAMG,gBACJH,OAAO,QAAQ,EAAE,SAAS,WAAWA,OAAO,QAAQ,CAAC,OAAO,GAAGI;QACjE,MAAMC,wBAAwBF,gBAC1BG,gCAAgCH,iBAChCC;QACJ,IAAI,CAAC,MAAM,GAAGG,cACZP,OAAO,MAAM,EACbK,uBACA;YACE,YAAY,IAAI,CAAC,UAAU;QAC7B,GACA,IAAI,CAAC,iBAAiB;QAExB,IAAI,CAAC,QAAQ,GAAGG,gBACdR,OAAO,QAAQ,EACfK,uBACA;YAAE,YAAY,IAAI,CAAC,UAAU;QAAC,GAC9B,AAAqB,eAArB,IAAI,CAAC,MAAM,CAAC,IAAI,GACZ,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,WAAW,GAC/BD;IAER;AACF"}

package/dist/es/ai-model/models/deepseek-locate-protocol.mjs

@@ -0,0 +1,87 @@
+import { formatLocateExampleValue } from "../shared/model-locate-result/index.mjs";
+const deepSeekRefBoxPattern = /<(?:｜｜|\|)ref(?:｜｜|\|)>\s*[\s\S]*?\s*<(?:｜｜|\|)\/ref(?:｜｜|\|)>\s*<(?:｜｜|\|)box(?:｜｜|\|)>\s*([\s\S]*?)\s*<(?:｜｜|\|)\/box(?:｜｜|\|)>/g;
+const deepSeekPointPattern = /^\s*<(?:｜｜|\|)point(?:｜｜|\|)>\s*([\s\S]*?)\s*<(?:｜｜|\|)\/point(?:｜｜|\|)>\s*$/;
+const deepSeekElementSystemPromptIntroduction = `## Role:
+You are a GUI click grounding agent.
+
+## Objective:
+- Identify the UI element in the screenshot that matches the user's description.
+- Return the center point of that element.`;
+const buildDeepSeekElementResponseInstructions = (promptSpec)=>`## Output Format:
+Return exactly one ${promptSpec.resultNoun} using the following format, without any explanation or additional text:
+
+<｜｜point｜｜>[${promptSpec.resultValueSchema}]<｜｜/point｜｜>
+
+For example:
+<｜｜point｜｜>[${formatLocateExampleValue(promptSpec.exampleValues[0])}]<｜｜/point｜｜>
+
+Coordinate values must be integers.
+Coordinate requirements: ${promptSpec.resultValueDescription}
+The origin is the top-left of the full screenshot.`;
+const buildDeepSeekSearchAreaResponseInstructions = (promptSpec)=>{
+    const targetExampleValue = formatLocateExampleValue(promptSpec.exampleValues[0]);
+    const referenceExampleValue = formatLocateExampleValue(promptSpec.exampleValues[1] ?? promptSpec.exampleValues[0]);
+    return `## Objective:
+- Identify the target UI element in the screenshot.
+- Identify the reference elements that the description uses to select the target, such as an owner label, row value, column value, nearby text, or relative-position anchor.
+- Return a tight ${promptSpec.resultNoun} for the target first, followed by a tight ${promptSpec.resultNoun} for each reference element.
+
+## Reference Rules:
+- If the description explicitly identifies the target through another visible element, you MUST return that visible element as a reference, even if the target appears unique.
+- For descriptions like "B in the row whose A is X", B is the target and the visible X is a reference.
+- For descriptions like "the icon next to label X", the icon is the target and the visible X is a reference.
+- Do not return unrelated landmarks or alternative target candidates.
+
+## Output Format:
+Return one or more ref-box pairs using the following format, without any explanation or additional text:
+
+<｜｜ref｜｜>target: concise target description<｜｜/ref｜｜><｜｜box｜｜>[${promptSpec.resultValueSchema}]<｜｜/box｜｜>
+<｜｜ref｜｜>reference: concise reference description<｜｜/ref｜｜><｜｜box｜｜>[${promptSpec.resultValueSchema}]<｜｜/box｜｜>
+
+For example:
+If the request is "the edit icon in the row whose project name is Apollo", return:
+<｜｜ref｜｜>target: edit icon<｜｜/ref｜｜><｜｜box｜｜>[${targetExampleValue}]<｜｜/box｜｜>
+<｜｜ref｜｜>reference: Apollo<｜｜/ref｜｜><｜｜box｜｜>[${referenceExampleValue}]<｜｜/box｜｜>
+
+The first ref-box pair must represent the target element. Every remaining pair must represent a reference element needed to identify the target. If no reference element is needed, return only the target pair. Each pair must contain exactly one ${promptSpec.resultNoun}.
+Coordinate values must be integers.
+Coordinate requirements: ${promptSpec.resultValueDescription}
+The origin is the top-left of the full screenshot.`;
+};
+function parseDeepSeekLocateOutput(content) {
+    const match = content.match(deepSeekPointPattern);
+    if (!match) throw new Error('DeepSeek element locate response does not contain a valid <｜｜point｜｜>value<｜｜/point｜｜> result');
+    return {
+        kind: 'located',
+        target: match[1]
+    };
+}
+function parseDeepSeekSearchAreaOutput(content) {
+    const rawBoxes = Array.from(content.matchAll(deepSeekRefBoxPattern), (match)=>match[1]);
+    if (0 === rawBoxes.length) throw new Error('DeepSeek search-area response does not contain a valid <｜｜ref｜｜>label<｜｜/ref｜｜><｜｜box｜｜>[[x1,y1,x2,y2]]<｜｜/box｜｜> result');
+    const [rawBbox, ...rawReferenceBboxes] = rawBoxes;
+    return {
+        kind: 'located',
+        target: rawBbox,
+        ...rawReferenceBboxes.length > 0 ? {
+            references: rawReferenceBboxes
+        } : {}
+    };
+}
+const deepSeekElementLocateProtocol = {
+    systemPromptIntroduction: deepSeekElementSystemPromptIntroduction,
+    buildResponseInstructions: buildDeepSeekElementResponseInstructions,
+    buildUserPrompt: (targetElementDescription)=>`Locate the center point of the following UI element: ${targetElementDescription}`,
+    expectedJsonObjectResponse: false,
+    parseRawResponse: parseDeepSeekLocateOutput
+};
+const deepSeekSearchAreaProtocol = {
+    systemPromptIntroduction: '',
+    buildResponseInstructions: buildDeepSeekSearchAreaResponseInstructions,
+    buildUserPrompt: (sectionDescription)=>`Locate the target and the reference elements needed to distinguish it: ${sectionDescription}`,
+    expectedJsonObjectResponse: false,
+    parseRawResponse: parseDeepSeekSearchAreaOutput
+};
+export { deepSeekElementLocateProtocol, deepSeekSearchAreaProtocol, parseDeepSeekLocateOutput, parseDeepSeekSearchAreaOutput };
+
+//# sourceMappingURL=deepseek-locate-protocol.mjs.map

package/dist/es/ai-model/models/deepseek-locate-protocol.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/models/deepseek-locate-protocol.mjs","sources":["../../../../src/ai-model/models/deepseek-locate-protocol.ts"],"sourcesContent":["import type {\n  ParsedLocateResponse,\n  StandardLocateProtocol,\n} from '../model-adapter/locate-protocol';\nimport {\n  type LocateResultPromptSpec,\n  formatLocateExampleValue,\n} from '../shared/model-locate-result';\n\nconst deepSeekRefBoxPattern =\n  /<(?:｜｜|\\|)ref(?:｜｜|\\|)>\\s*[\\s\\S]*?\\s*<(?:｜｜|\\|)\\/ref(?:｜｜|\\|)>\\s*<(?:｜｜|\\|)box(?:｜｜|\\|)>\\s*([\\s\\S]*?)\\s*<(?:｜｜|\\|)\\/box(?:｜｜|\\|)>/g;\n\nconst deepSeekPointPattern =\n  /^\\s*<(?:｜｜|\\|)point(?:｜｜|\\|)>\\s*([\\s\\S]*?)\\s*<(?:｜｜|\\|)\\/point(?:｜｜|\\|)>\\s*$/;\n\nconst deepSeekElementSystemPromptIntroduction = `## Role:\nYou are a GUI click grounding agent.\n\n## Objective:\n- Identify the UI element in the screenshot that matches the user's description.\n- Return the center point of that element.`;\n\nconst buildDeepSeekElementResponseInstructions = (\n  promptSpec: LocateResultPromptSpec,\n) => `## Output Format:\nReturn exactly one ${promptSpec.resultNoun} using the following format, without any explanation or additional text:\n\n<｜｜point｜｜>[${promptSpec.resultValueSchema}]<｜｜/point｜｜>\n\nFor example:\n<｜｜point｜｜>[${formatLocateExampleValue(promptSpec.exampleValues[0])}]<｜｜/point｜｜>\n\nCoordinate values must be integers.\nCoordinate requirements: ${promptSpec.resultValueDescription}\nThe origin is the top-left of the full screenshot.`;\n\nconst buildDeepSeekSearchAreaResponseInstructions = (\n  promptSpec: LocateResultPromptSpec,\n) => {\n  const targetExampleValue = formatLocateExampleValue(\n    promptSpec.exampleValues[0],\n  );\n  const referenceExampleValue = formatLocateExampleValue(\n    promptSpec.exampleValues[1] ?? promptSpec.exampleValues[0],\n  );\n\n  return `## Objective:\n- Identify the target UI element in the screenshot.\n- Identify the reference elements that the description uses to select the target, such as an owner label, row value, column value, nearby text, or relative-position anchor.\n- Return a tight ${promptSpec.resultNoun} for the target first, followed by a tight ${promptSpec.resultNoun} for each reference element.\n\n## Reference Rules:\n- If the description explicitly identifies the target through another visible element, you MUST return that visible element as a reference, even if the target appears unique.\n- For descriptions like \"B in the row whose A is X\", B is the target and the visible X is a reference.\n- For descriptions like \"the icon next to label X\", the icon is the target and the visible X is a reference.\n- Do not return unrelated landmarks or alternative target candidates.\n\n## Output Format:\nReturn one or more ref-box pairs using the following format, without any explanation or additional text:\n\n<｜｜ref｜｜>target: concise target description<｜｜/ref｜｜><｜｜box｜｜>[${promptSpec.resultValueSchema}]<｜｜/box｜｜>\n<｜｜ref｜｜>reference: concise reference description<｜｜/ref｜｜><｜｜box｜｜>[${promptSpec.resultValueSchema}]<｜｜/box｜｜>\n\nFor example:\nIf the request is \"the edit icon in the row whose project name is Apollo\", return:\n<｜｜ref｜｜>target: edit icon<｜｜/ref｜｜><｜｜box｜｜>[${targetExampleValue}]<｜｜/box｜｜>\n<｜｜ref｜｜>reference: Apollo<｜｜/ref｜｜><｜｜box｜｜>[${referenceExampleValue}]<｜｜/box｜｜>\n\nThe first ref-box pair must represent the target element. Every remaining pair must represent a reference element needed to identify the target. If no reference element is needed, return only the target pair. Each pair must contain exactly one ${promptSpec.resultNoun}.\nCoordinate values must be integers.\nCoordinate requirements: ${promptSpec.resultValueDescription}\nThe origin is the top-left of the full screenshot.`;\n};\n\nexport function parseDeepSeekLocateOutput(\n  content: string,\n): ParsedLocateResponse {\n  const match = content.match(deepSeekPointPattern);\n  if (!match) {\n    throw new Error(\n      'DeepSeek element locate response does not contain a valid <｜｜point｜｜>value<｜｜/point｜｜> result',\n    );\n  }\n\n  return {\n    kind: 'located',\n    target: match[1],\n  };\n}\n\nexport function parseDeepSeekSearchAreaOutput(\n  content: string,\n): ParsedLocateResponse {\n  const rawBoxes = Array.from(\n    content.matchAll(deepSeekRefBoxPattern),\n    (match) => match[1],\n  );\n  if (rawBoxes.length === 0) {\n    throw new Error(\n      'DeepSeek search-area response does not contain a valid <｜｜ref｜｜>label<｜｜/ref｜｜><｜｜box｜｜>[[x1,y1,x2,y2]]<｜｜/box｜｜> result',\n    );\n  }\n\n  const [rawBbox, ...rawReferenceBboxes] = rawBoxes;\n  return {\n    kind: 'located',\n    target: rawBbox,\n    ...(rawReferenceBboxes.length > 0\n      ? { references: rawReferenceBboxes }\n      : {}),\n  };\n}\n\nexport const deepSeekElementLocateProtocol: StandardLocateProtocol = {\n  systemPromptIntroduction: deepSeekElementSystemPromptIntroduction,\n  buildResponseInstructions: buildDeepSeekElementResponseInstructions,\n  buildUserPrompt: (targetElementDescription) =>\n    `Locate the center point of the following UI element: ${targetElementDescription}`,\n  expectedJsonObjectResponse: false,\n  parseRawResponse: parseDeepSeekLocateOutput,\n};\n\nexport const deepSeekSearchAreaProtocol: StandardLocateProtocol = {\n  systemPromptIntroduction: '',\n  buildResponseInstructions: buildDeepSeekSearchAreaResponseInstructions,\n  buildUserPrompt: (sectionDescription) =>\n    `Locate the target and the reference elements needed to distinguish it: ${sectionDescription}`,\n  expectedJsonObjectResponse: false,\n  parseRawResponse: parseDeepSeekSearchAreaOutput,\n};\n"],"names":["deepSeekRefBoxPattern","deepSeekPointPattern","deepSeekElementSystemPromptIntroduction","buildDeepSeekElementResponseInstructions","promptSpec","formatLocateExampleValue","buildDeepSeekSearchAreaResponseInstructions","targetExampleValue","referenceExampleValue","parseDeepSeekLocateOutput","content","match","Error","parseDeepSeekSearchAreaOutput","rawBoxes","Array","rawBbox","rawReferenceBboxes","deepSeekElementLocateProtocol","targetElementDescription","deepSeekSearchAreaProtocol","sectionDescription"],"mappings":";AASA,MAAMA,wBACJ;AAEF,MAAMC,uBACJ;AAEF,MAAMC,0CAA0C,CAAC;;;;;0CAKP,CAAC;AAE3C,MAAMC,2CAA2C,CAC/CC,aACG,CAAC;mBACa,EAAEA,WAAW,UAAU,CAAC;;YAE/B,EAAEA,WAAW,iBAAiB,CAAC;;;YAG/B,EAAEC,yBAAyBD,WAAW,aAAa,CAAC,EAAE,EAAE;;;yBAG3C,EAAEA,WAAW,sBAAsB,CAAC;kDACX,CAAC;AAEnD,MAAME,8CAA8C,CAClDF;IAEA,MAAMG,qBAAqBF,yBACzBD,WAAW,aAAa,CAAC,EAAE;IAE7B,MAAMI,wBAAwBH,yBAC5BD,WAAW,aAAa,CAAC,EAAE,IAAIA,WAAW,aAAa,CAAC,EAAE;IAG5D,OAAO,CAAC;;;iBAGO,EAAEA,WAAW,UAAU,CAAC,2CAA2C,EAAEA,WAAW,UAAU,CAAC;;;;;;;;;;;+DAW7C,EAAEA,WAAW,iBAAiB,CAAC;qEACzB,EAAEA,WAAW,iBAAiB,CAAC;;;;8CAItD,EAAEG,mBAAmB;8CACrB,EAAEC,sBAAsB;;oPAE8K,EAAEJ,WAAW,UAAU,CAAC;;yBAEnP,EAAEA,WAAW,sBAAsB,CAAC;kDACX,CAAC;AACnD;AAEO,SAASK,0BACdC,OAAe;IAEf,MAAMC,QAAQD,QAAQ,KAAK,CAACT;IAC5B,IAAI,CAACU,OACH,MAAM,IAAIC,MACR;IAIJ,OAAO;QACL,MAAM;QACN,QAAQD,KAAK,CAAC,EAAE;IAClB;AACF;AAEO,SAASE,8BACdH,OAAe;IAEf,MAAMI,WAAWC,MAAM,IAAI,CACzBL,QAAQ,QAAQ,CAACV,wBACjB,CAACW,QAAUA,KAAK,CAAC,EAAE;IAErB,IAAIG,AAAoB,MAApBA,SAAS,MAAM,EACjB,MAAM,IAAIF,MACR;IAIJ,MAAM,CAACI,SAAS,GAAGC,mBAAmB,GAAGH;IACzC,OAAO;QACL,MAAM;QACN,QAAQE;QACR,GAAIC,mBAAmB,MAAM,GAAG,IAC5B;YAAE,YAAYA;QAAmB,IACjC,CAAC,CAAC;IACR;AACF;AAEO,MAAMC,gCAAwD;IACnE,0BAA0BhB;IAC1B,2BAA2BC;IAC3B,iBAAiB,CAACgB,2BAChB,CAAC,qDAAqD,EAAEA,0BAA0B;IACpF,4BAA4B;IAC5B,kBAAkBV;AACpB;AAEO,MAAMW,6BAAqD;IAChE,0BAA0B;IAC1B,2BAA2Bd;IAC3B,iBAAiB,CAACe,qBAChB,CAAC,uEAAuE,EAAEA,oBAAoB;IAChG,4BAA4B;IAC5B,kBAAkBR;AACpB"}

package/dist/es/ai-model/models/deepseek.mjs

@@ -0,0 +1,77 @@
+import { createLocateResultValue } from "../shared/model-locate-result/index.mjs";
+import { deepSeekElementLocateProtocol, deepSeekSearchAreaProtocol } from "./deepseek-locate-protocol.mjs";
+const deepSeekPointCoordinates = {
+    shape: 'point',
+    order: 'xy',
+    normalizedBy: 1000
+};
+const deepSeekBboxCoordinates = {
+    shape: 'bbox',
+    order: 'xy',
+    normalizedBy: 1000
+};
+function parseDeepSeekCoordinateValues(input, expectedLength, label) {
+    const coordinateTexts = String(input).match(/\d+/g);
+    if (coordinateTexts?.length !== expectedLength) throw new Error(`DeepSeek ${label} locate result must contain exactly ${expectedLength} positive integers, got ${coordinateTexts?.length ?? 0}`);
+    return coordinateTexts.map(Number);
+}
+function parseDeepSeekPointLocateValue(input) {
+    return createLocateResultValue(deepSeekPointCoordinates, parseDeepSeekCoordinateValues(input, 2, 'point'));
+}
+function parseDeepSeekBboxLocateValue(input) {
+    return createLocateResultValue(deepSeekBboxCoordinates, parseDeepSeekCoordinateValues(input, 4, 'bbox'));
+}
+const buildDeepSeekChatCompletionParams = (input)=>{
+    const { midsceneDefaults, userConfig } = input;
+    const { reasoningEnabled, reasoningEffort } = userConfig;
+    const commonOverrideConfig = {};
+    if (void 0 !== userConfig.temperature) commonOverrideConfig.temperature = userConfig.temperature;
+    if ('none' !== userConfig.responseFormat && input.expectedJsonObjectResponse) commonOverrideConfig.response_format = {
+        type: 'json_object'
+    };
+    const modelSpecificConfig = {};
+    if ('default' !== reasoningEnabled) {
+        modelSpecificConfig.thinking = {
+            type: reasoningEnabled ?? false ? 'enabled' : 'disabled'
+        };
+        if (reasoningEffort) modelSpecificConfig.reasoning_effort = reasoningEffort;
+    }
+    return {
+        config: {
+            ...midsceneDefaults,
+            ...commonOverrideConfig,
+            ...modelSpecificConfig
+        }
+    };
+};
+const deepSeekAdapters = {
+    deepseek: {
+        chatCompletion: {
+            unsupportedUserConfig: [
+                'reasoningBudget'
+            ],
+            buildChatCompletionParams: buildDeepSeekChatCompletionParams,
+            useReasoningAsContentFallback: true,
+            replayRawAssistantMessage: false
+        },
+        locate: {
+            element: {
+                protocol: deepSeekElementLocateProtocol,
+                resultFormat: {
+                    coordinates: deepSeekPointCoordinates,
+                    parseRawLocateValue: parseDeepSeekPointLocateValue
+                }
+            },
+            searchArea: {
+                protocol: deepSeekSearchAreaProtocol,
+                resultFormat: {
+                    coordinates: deepSeekBboxCoordinates,
+                    parseRawLocateValue: parseDeepSeekBboxLocateValue
+                }
+            }
+        }
+    }
+};
+export { deepSeekAdapters };
+
+//# sourceMappingURL=deepseek.mjs.map

package/dist/es/ai-model/models/deepseek.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/models/deepseek.mjs","sources":["../../../../src/ai-model/models/deepseek.ts"],"sourcesContent":["import type { TModelFamily } from '@midscene/shared/env';\nimport type {\n  ChatCompletionCallContext,\n  ChatCompletionParamsResult,\n  ModelAdapterDefinition,\n} from '../model-adapter/types';\nimport {\n  type LocateResultValue,\n  createLocateResultValue,\n} from '../shared/model-locate-result';\nimport {\n  deepSeekElementLocateProtocol,\n  deepSeekSearchAreaProtocol,\n} from './deepseek-locate-protocol';\n\nconst deepSeekPointCoordinates = {\n  shape: 'point',\n  order: 'xy',\n  normalizedBy: 1000,\n} as const;\n\nconst deepSeekBboxCoordinates = {\n  shape: 'bbox',\n  order: 'xy',\n  normalizedBy: 1000,\n} as const;\n\nfunction parseDeepSeekCoordinateValues(\n  input: unknown,\n  expectedLength: number,\n  label: string,\n): number[] {\n  const coordinateTexts = String(input).match(/\\d+/g);\n  if (coordinateTexts?.length !== expectedLength) {\n    throw new Error(\n      `DeepSeek ${label} locate result must contain exactly ${expectedLength} positive integers, got ${coordinateTexts?.length ?? 0}`,\n    );\n  }\n\n  return coordinateTexts.map(Number);\n}\n\nfunction parseDeepSeekPointLocateValue(input: unknown): LocateResultValue {\n  return createLocateResultValue(\n    deepSeekPointCoordinates,\n    parseDeepSeekCoordinateValues(input, 2, 'point'),\n  );\n}\n\nfunction parseDeepSeekBboxLocateValue(input: unknown): LocateResultValue {\n  return createLocateResultValue(\n    deepSeekBboxCoordinates,\n    parseDeepSeekCoordinateValues(input, 4, 'bbox'),\n  );\n}\n\nconst buildDeepSeekChatCompletionParams = (\n  input: ChatCompletionCallContext,\n): ChatCompletionParamsResult => {\n  const { midsceneDefaults, userConfig } = input;\n  const { reasoningEnabled, reasoningEffort } = userConfig;\n  const commonOverrideConfig: Record<string, unknown> = {};\n\n  if (userConfig.temperature !== undefined) {\n    commonOverrideConfig.temperature = userConfig.temperature;\n  }\n\n  if (\n    userConfig.responseFormat !== 'none' &&\n    input.expectedJsonObjectResponse\n  ) {\n    commonOverrideConfig.response_format = { type: 'json_object' };\n  }\n\n  const modelSpecificConfig: Record<string, unknown> = {};\n  if (reasoningEnabled !== 'default') {\n    modelSpecificConfig.thinking = {\n      type: (reasoningEnabled ?? false) ? 'enabled' : 'disabled',\n    };\n    if (reasoningEffort) {\n      modelSpecificConfig.reasoning_effort = reasoningEffort;\n    }\n  }\n\n  return {\n    config: {\n      ...midsceneDefaults,\n      ...commonOverrideConfig,\n      ...modelSpecificConfig,\n    },\n  };\n};\n\nexport const deepSeekAdapters = {\n  deepseek: {\n    chatCompletion: {\n      unsupportedUserConfig: ['reasoningBudget'],\n      buildChatCompletionParams: buildDeepSeekChatCompletionParams,\n      useReasoningAsContentFallback: true,\n      // DeepSeek only requires reasoning_content to be replayed after tool\n      // calls. Midscene planning does not use tool calls, so it is unnecessary.\n      replayRawAssistantMessage: false,\n    },\n    locate: {\n      element: {\n        protocol: deepSeekElementLocateProtocol,\n        resultFormat: {\n          coordinates: deepSeekPointCoordinates,\n          parseRawLocateValue: parseDeepSeekPointLocateValue,\n        },\n      },\n      searchArea: {\n        protocol: deepSeekSearchAreaProtocol,\n        resultFormat: {\n          coordinates: deepSeekBboxCoordinates,\n          parseRawLocateValue: parseDeepSeekBboxLocateValue,\n        },\n      },\n    },\n  },\n} satisfies Pick<Record<TModelFamily, ModelAdapterDefinition>, 'deepseek'>;\n"],"names":["deepSeekPointCoordinates","deepSeekBboxCoordinates","parseDeepSeekCoordinateValues","input","expectedLength","label","coordinateTexts","String","Error","Number","parseDeepSeekPointLocateValue","createLocateResultValue","parseDeepSeekBboxLocateValue","buildDeepSeekChatCompletionParams","midsceneDefaults","userConfig","reasoningEnabled","reasoningEffort","commonOverrideConfig","undefined","modelSpecificConfig","deepSeekAdapters","deepSeekElementLocateProtocol","deepSeekSearchAreaProtocol"],"mappings":";;AAeA,MAAMA,2BAA2B;IAC/B,OAAO;IACP,OAAO;IACP,cAAc;AAChB;AAEA,MAAMC,0BAA0B;IAC9B,OAAO;IACP,OAAO;IACP,cAAc;AAChB;AAEA,SAASC,8BACPC,KAAc,EACdC,cAAsB,EACtBC,KAAa;IAEb,MAAMC,kBAAkBC,OAAOJ,OAAO,KAAK,CAAC;IAC5C,IAAIG,iBAAiB,WAAWF,gBAC9B,MAAM,IAAII,MACR,CAAC,SAAS,EAAEH,MAAM,oCAAoC,EAAED,eAAe,wBAAwB,EAAEE,iBAAiB,UAAU,GAAG;IAInI,OAAOA,gBAAgB,GAAG,CAACG;AAC7B;AAEA,SAASC,8BAA8BP,KAAc;IACnD,OAAOQ,wBACLX,0BACAE,8BAA8BC,OAAO,GAAG;AAE5C;AAEA,SAASS,6BAA6BT,KAAc;IAClD,OAAOQ,wBACLV,yBACAC,8BAA8BC,OAAO,GAAG;AAE5C;AAEA,MAAMU,oCAAoC,CACxCV;IAEA,MAAM,EAAEW,gBAAgB,EAAEC,UAAU,EAAE,GAAGZ;IACzC,MAAM,EAAEa,gBAAgB,EAAEC,eAAe,EAAE,GAAGF;IAC9C,MAAMG,uBAAgD,CAAC;IAEvD,IAAIH,AAA2BI,WAA3BJ,WAAW,WAAW,EACxBG,qBAAqB,WAAW,GAAGH,WAAW,WAAW;IAG3D,IACEA,AAA8B,WAA9BA,WAAW,cAAc,IACzBZ,MAAM,0BAA0B,EAEhCe,qBAAqB,eAAe,GAAG;QAAE,MAAM;IAAc;IAG/D,MAAME,sBAA+C,CAAC;IACtD,IAAIJ,AAAqB,cAArBA,kBAAgC;QAClCI,oBAAoB,QAAQ,GAAG;YAC7B,MAAOJ,oBAAoB,QAAS,YAAY;QAClD;QACA,IAAIC,iBACFG,oBAAoB,gBAAgB,GAAGH;IAE3C;IAEA,OAAO;QACL,QAAQ;YACN,GAAGH,gBAAgB;YACnB,GAAGI,oBAAoB;YACvB,GAAGE,mBAAmB;QACxB;IACF;AACF;AAEO,MAAMC,mBAAmB;IAC9B,UAAU;QACR,gBAAgB;YACd,uBAAuB;gBAAC;aAAkB;YAC1C,2BAA2BR;YAC3B,+BAA+B;YAG/B,2BAA2B;QAC7B;QACA,QAAQ;YACN,SAAS;gBACP,UAAUS;gBACV,cAAc;oBACZ,aAAatB;oBACb,qBAAqBU;gBACvB;YACF;YACA,YAAY;gBACV,UAAUa;gBACV,cAAc;oBACZ,aAAatB;oBACb,qBAAqBW;gBACvB;YACF;QACF;IACF;AACF"}

package/dist/es/ai-model/models/doubao.mjs

@@ -78,9 +78,11 @@ const doubaoVisionAdapter = {
         useReasoningAsContentFallback: true
     },
     locate: {
-        resultAdapter: {
-            coordinates: doubaoBboxCoordinatesMeta,
-            parseRawLocateValue: parseDoubaoRawLocateValue
+        element: {
+            resultFormat: {
+                coordinates: doubaoBboxCoordinatesMeta,
+                parseRawLocateValue: parseDoubaoRawLocateValue
+            }
         }
     }
 };