@aiscene/core 9.0.6 → 9.0.8

package/dist/es/ai-model/models/ui-tars/adapter.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/models/ui-tars/adapter.mjs","sources":["../../../../../src/ai-model/models/ui-tars/adapter.ts"],"sourcesContent":["import { type TModelFamily, UITarsModelVersion } from '@aiscene/shared/env';\nimport { assert } from '@aiscene/shared/utils';\nimport type { ModelAdapterDefinition } from '../../model-adapter/types';\nimport {\n  type LocateResultValue,\n  createLocateResultValue,\n  unwrapCoordinateListLikeInput,\n} from '../../shared/model-locate-result';\nimport { createUiTarsPlanner } from './planning';\n\nconst defaultVlmUiTarsReplanningCycleLimit = 40;\nconst uiTarsBboxCoordinatesMeta = {\n  shape: 'bbox',\n  order: 'xy',\n  normalizedBy: 1000,\n} as const;\nconst uiTarsPointCoordinatesMeta = {\n  shape: 'point',\n  order: 'xy',\n  normalizedBy: 1000,\n} as const;\n\n// UI-TARS has not received active updates for a long time, so this parser is\n// intentionally kept separate from Doubao even though the current logic is the\n// same. This avoids coupling UI-TARS behavior to future Doubao adapter changes.\nfunction parseUiTarsRawLocateValue(input: unknown): LocateResultValue {\n  const bbox = unwrapCoordinateListLikeInput(input as any);\n  if (typeof bbox === 'string') {\n    assert(\n      /^(\\d+)\\s(\\d+)\\s(\\d+)\\s(\\d+)$/.test(bbox.trim()),\n      `invalid bbox data string for ui-tars mode: ${bbox}`,\n    );\n    const splitted = bbox.split(' ');\n    if (splitted.length === 4) {\n      return createLocateResultValue(uiTarsBboxCoordinatesMeta, [\n        Number(splitted[0]),\n        Number(splitted[1]),\n        Number(splitted[2]),\n        Number(splitted[3]),\n      ]);\n    }\n    throw new Error(`invalid bbox data string for ui-tars mode: ${bbox}`);\n  }\n\n  let bboxList: number[] = [];\n  if (Array.isArray(bbox) && typeof bbox[0] === 'string') {\n    bbox.forEach((item) => {\n      if (typeof item === 'string' && item.includes(',')) {\n        const [x, y] = item.split(',');\n        bboxList.push(Number(x.trim()), Number(y.trim()));\n      } else if (typeof item === 'string' && item.includes(' ')) {\n        const [x, y] = item.split(' ');\n        bboxList.push(Number(x.trim()), Number(y.trim()));\n      } else {\n        bboxList.push(Number(item));\n      }\n    });\n  } else {\n    bboxList = bbox as number[];\n  }\n\n  if (bboxList.length === 4 || bboxList.length === 5) {\n    return createLocateResultValue(uiTarsBboxCoordinatesMeta, [\n      bboxList[0],\n      bboxList[1],\n      bboxList[2],\n      bboxList[3],\n    ]);\n  }\n\n  if (\n    bboxList.length === 6 ||\n    bboxList.length === 2 ||\n    bboxList.length === 3 ||\n    bboxList.length === 7\n  ) {\n    return createLocateResultValue(uiTarsPointCoordinatesMeta, [\n      bboxList[0],\n      bboxList[1],\n    ]);\n  }\n\n  if (bbox.length === 8) {\n    return createLocateResultValue(uiTarsBboxCoordinatesMeta, [\n      bboxList[0],\n      bboxList[1],\n      bboxList[4],\n      bboxList[5],\n    ]);\n  }\n\n  const msg = `invalid bbox data for ui-tars mode: ${JSON.stringify(bbox)} `;\n  throw new Error(msg);\n}\n\nfunction createUiTarsAdapter(\n  uiTarsModelVersion: UITarsModelVersion,\n): ModelAdapterDefinition {\n  return {\n    chatCompletion: {\n      unsupportedUserConfig: [\n        'reasoningEnabled',\n        'reasoningEffort',\n        'reasoningBudget',\n      ],\n      buildChatCompletionParams: ({ midsceneDefaults, userConfig }) => {\n        const commonOverrideConfig: Record<string, unknown> = {};\n\n        if (userConfig.temperature !== undefined) {\n          commonOverrideConfig.temperature = userConfig.temperature;\n        }\n\n        return {\n          config: {\n            ...midsceneDefaults,\n            ...commonOverrideConfig,\n          },\n        };\n      },\n    },\n    planning: {\n      kind: 'custom',\n      cacheEnabled: false,\n      defaultReplanningCycleLimit: defaultVlmUiTarsReplanningCycleLimit,\n      planner: createUiTarsPlanner(uiTarsModelVersion),\n    },\n    locate: {\n      element: {\n        resultFormat: {\n          coordinates: uiTarsBboxCoordinatesMeta,\n          parseRawLocateValue: parseUiTarsRawLocateValue,\n        },\n      },\n    },\n  };\n}\n\nconst uiTarsDoubao15Adapter = createUiTarsAdapter(\n  UITarsModelVersion.DOUBAO_1_5_20B,\n);\n\nexport const uiTarsAdapters = {\n  'vlm-ui-tars': createUiTarsAdapter(UITarsModelVersion.V1_0),\n  'vlm-ui-tars-doubao': uiTarsDoubao15Adapter,\n  'vlm-ui-tars-doubao-1.5': uiTarsDoubao15Adapter,\n} satisfies Pick<\n  Record<TModelFamily, ModelAdapterDefinition>,\n  'vlm-ui-tars' | 'vlm-ui-tars-doubao' | 'vlm-ui-tars-doubao-1.5'\n>;\n"],"names":["defaultVlmUiTarsReplanningCycleLimit","uiTarsBboxCoordinatesMeta","uiTarsPointCoordinatesMeta","parseUiTarsRawLocateValue","input","bbox","unwrapCoordinateListLikeInput","assert","splitted","createLocateResultValue","Number","Error","bboxList","Array","item","x","y","msg","JSON","createUiTarsAdapter","uiTarsModelVersion","midsceneDefaults","userConfig","commonOverrideConfig","undefined","createUiTarsPlanner","uiTarsDoubao15Adapter","UITarsModelVersion","uiTarsAdapters"],"mappings":";;;;AAUA,MAAMA,uCAAuC;AAC7C,MAAMC,4BAA4B;IAChC,OAAO;IACP,OAAO;IACP,cAAc;AAChB;AACA,MAAMC,6BAA6B;IACjC,OAAO;IACP,OAAO;IACP,cAAc;AAChB;AAKA,SAASC,0BAA0BC,KAAc;IAC/C,MAAMC,OAAOC,8BAA8BF;IAC3C,IAAI,AAAgB,YAAhB,OAAOC,MAAmB;QAC5BE,OACE,+BAA+B,IAAI,CAACF,KAAK,IAAI,KAC7C,CAAC,2CAA2C,EAAEA,MAAM;QAEtD,MAAMG,WAAWH,KAAK,KAAK,CAAC;QAC5B,IAAIG,AAAoB,MAApBA,SAAS,MAAM,EACjB,OAAOC,wBAAwBR,2BAA2B;YACxDS,OAAOF,QAAQ,CAAC,EAAE;YAClBE,OAAOF,QAAQ,CAAC,EAAE;YAClBE,OAAOF,QAAQ,CAAC,EAAE;YAClBE,OAAOF,QAAQ,CAAC,EAAE;SACnB;QAEH,MAAM,IAAIG,MAAM,CAAC,2CAA2C,EAAEN,MAAM;IACtE;IAEA,IAAIO,WAAqB,EAAE;IAC3B,IAAIC,MAAM,OAAO,CAACR,SAAS,AAAmB,YAAnB,OAAOA,IAAI,CAAC,EAAE,EACvCA,KAAK,OAAO,CAAC,CAACS;QACZ,IAAI,AAAgB,YAAhB,OAAOA,QAAqBA,KAAK,QAAQ,CAAC,MAAM;YAClD,MAAM,CAACC,GAAGC,EAAE,GAAGF,KAAK,KAAK,CAAC;YAC1BF,SAAS,IAAI,CAACF,OAAOK,EAAE,IAAI,KAAKL,OAAOM,EAAE,IAAI;QAC/C,OAAO,IAAI,AAAgB,YAAhB,OAAOF,QAAqBA,KAAK,QAAQ,CAAC,MAAM;YACzD,MAAM,CAACC,GAAGC,EAAE,GAAGF,KAAK,KAAK,CAAC;YAC1BF,SAAS,IAAI,CAACF,OAAOK,EAAE,IAAI,KAAKL,OAAOM,EAAE,IAAI;QAC/C,OACEJ,SAAS,IAAI,CAACF,OAAOI;IAEzB;SAEAF,WAAWP;IAGb,IAAIO,AAAoB,MAApBA,SAAS,MAAM,IAAUA,AAAoB,MAApBA,SAAS,MAAM,EAC1C,OAAOH,wBAAwBR,2BAA2B;QACxDW,QAAQ,CAAC,EAAE;QACXA,QAAQ,CAAC,EAAE;QACXA,QAAQ,CAAC,EAAE;QACXA,QAAQ,CAAC,EAAE;KACZ;IAGH,IACEA,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,EAEf,OAAOH,wBAAwBP,4BAA4B;QACzDU,QAAQ,CAAC,EAAE;QACXA,QAAQ,CAAC,EAAE;KACZ;IAGH,IAAIP,AAAgB,MAAhBA,KAAK,MAAM,EACb,OAAOI,wBAAwBR,2BAA2B;QACxDW,QAAQ,CAAC,EAAE;QACXA,QAAQ,CAAC,EAAE;QACXA,QAAQ,CAAC,EAAE;QACXA,QAAQ,CAAC,EAAE;KACZ;IAGH,MAAMK,MAAM,CAAC,oCAAoC,EAAEC,KAAK,SAAS,CAACb,MAAM,CAAC,CAAC;IAC1E,MAAM,IAAIM,MAAMM;AAClB;AAEA,SAASE,oBACPC,kBAAsC;IAEtC,OAAO;QACL,gBAAgB;YACd,uBAAuB;gBACrB;gBACA;gBACA;aACD;YACD,2BAA2B,CAAC,EAAEC,gBAAgB,EAAEC,UAAU,EAAE;gBAC1D,MAAMC,uBAAgD,CAAC;gBAEvD,IAAID,AAA2BE,WAA3BF,WAAW,WAAW,EACxBC,qBAAqB,WAAW,GAAGD,WAAW,WAAW;gBAG3D,OAAO;oBACL,QAAQ;wBACN,GAAGD,gBAAgB;wBACnB,GAAGE,oBAAoB;oBACzB;gBACF;YACF;QACF;QACA,UAAU;YACR,MAAM;YACN,cAAc;YACd,6BAA6BvB;YAC7B,SAASyB,oBAAoBL;QAC/B;QACA,QAAQ;YACN,SAAS;gBACP,cAAc;oBACZ,aAAanB;oBACb,qBAAqBE;gBACvB;YACF;QACF;IACF;AACF;AAEA,MAAMuB,wBAAwBP,oBAC5BQ,mBAAmB,cAAc;AAG5B,MAAMC,iBAAiB;IAC5B,eAAeT,oBAAoBQ,mBAAmB,IAAI;IAC1D,sBAAsBD;IACtB,0BAA0BA;AAC5B"}

package/dist/es/ai-model/models/ui-tars/parser.mjs

@@ -0,0 +1,40 @@
+import { getDebug } from "@aiscene/shared/logger";
+import { actionParser } from "@ui-tars/action-parser";
+const debug = getDebug('ui-tars-planning');
+function parseUiTarsPlanningResponse(rawResponse, shotSize, uiTarsModelVersion) {
+    const convertedText = convertBboxToCoordinates(rawResponse);
+    const parseResult = actionParser({
+        prediction: convertedText,
+        factor: [
+            1000,
+            1000
+        ],
+        screenContext: {
+            width: shotSize.width,
+            height: shotSize.height
+        },
+        modelVer: uiTarsModelVersion
+    });
+    debug('ui-tars modelVer', uiTarsModelVersion, ', parsed', JSON.stringify(parseResult.parsed));
+    return {
+        rawResponse,
+        actions: parseResult.parsed
+    };
+}
+function convertBboxToCoordinates(text) {
+    const pattern = /<bbox>(\d+)\s+(\d+)\s+(\d+)\s+(\d+)<\/bbox>/g;
+    function replaceMatch(match, x1, y1, x2, y2) {
+        const x1Num = Number.parseInt(x1, 10);
+        const y1Num = Number.parseInt(y1, 10);
+        const x2Num = Number.parseInt(x2, 10);
+        const y2Num = Number.parseInt(y2, 10);
+        const x = Math.floor((x1Num + x2Num) / 2);
+        const y = Math.floor((y1Num + y2Num) / 2);
+        return `(${x},${y})`;
+    }
+    const cleanedText = text.replace(/\[EOS\]/g, '').replace(/```(?:[a-zA-Z0-9_-]+)?/g, '');
+    return cleanedText.replace(pattern, replaceMatch).trim();
+}
+export { parseUiTarsPlanningResponse };
+
+//# sourceMappingURL=parser.mjs.map

package/dist/es/ai-model/models/ui-tars/parser.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/models/ui-tars/parser.mjs","sources":["../../../../../src/ai-model/models/ui-tars/parser.ts"],"sourcesContent":["import type { Size } from '@/types';\nimport type { UITarsModelVersion } from '@aiscene/shared/env';\nimport { getDebug } from '@aiscene/shared/logger';\nimport { actionParser } from '@ui-tars/action-parser';\n\ntype ActionType =\n  | 'click'\n  | 'left_double'\n  | 'right_single'\n  | 'drag'\n  | 'type'\n  | 'hotkey'\n  | 'finished'\n  | 'scroll'\n  | 'wait';\n\nconst debug = getDebug('ui-tars-planning');\n\nexport interface UiTarsParsedPlanningResponse {\n  rawResponse: string;\n  actions: ReturnType<typeof actionParser>['parsed'];\n}\n\nexport function parseUiTarsPlanningResponse(\n  rawResponse: string,\n  shotSize: Size,\n  uiTarsModelVersion: UITarsModelVersion,\n): UiTarsParsedPlanningResponse {\n  const convertedText = convertBboxToCoordinates(rawResponse);\n  const parseResult = actionParser({\n    prediction: convertedText,\n    factor: [1000, 1000],\n    screenContext: {\n      width: shotSize.width,\n      height: shotSize.height,\n    },\n    modelVer: uiTarsModelVersion,\n  });\n\n  debug(\n    'ui-tars modelVer',\n    uiTarsModelVersion,\n    ', parsed',\n    JSON.stringify(parseResult.parsed),\n  );\n\n  return {\n    rawResponse,\n    actions: parseResult.parsed,\n  };\n}\n\n/**\n * Converts bounding box notation to coordinate points.\n */\nfunction convertBboxToCoordinates(text: string): string {\n  const pattern = /<bbox>(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)<\\/bbox>/g;\n\n  function replaceMatch(\n    match: string,\n    x1: string,\n    y1: string,\n    x2: string,\n    y2: string,\n  ): string {\n    const x1Num = Number.parseInt(x1, 10);\n    const y1Num = Number.parseInt(y1, 10);\n    const x2Num = Number.parseInt(x2, 10);\n    const y2Num = Number.parseInt(y2, 10);\n\n    const x = Math.floor((x1Num + x2Num) / 2);\n    const y = Math.floor((y1Num + y2Num) / 2);\n\n    return `(${x},${y})`;\n  }\n\n  const cleanedText = text\n    .replace(/\\[EOS\\]/g, '')\n    .replace(/```(?:[a-zA-Z0-9_-]+)?/g, '');\n  return cleanedText.replace(pattern, replaceMatch).trim();\n}\n\ninterface BaseAction {\n  action_type: ActionType;\n  action_inputs: Record<string, any>;\n  reflection: string | null;\n  thought: string | null;\n}\n\ninterface ClickAction extends BaseAction {\n  action_type: 'click';\n  action_inputs: {\n    start_box: string;\n  };\n}\n\ninterface DragAction extends BaseAction {\n  action_type: 'drag';\n  action_inputs: {\n    start_box: string;\n    end_box: string;\n  };\n}\n\ninterface WaitAction extends BaseAction {\n  action_type: 'wait';\n  action_inputs: {\n    time: string;\n  };\n}\n\ninterface LeftDoubleAction extends BaseAction {\n  action_type: 'left_double';\n  action_inputs: {\n    start_box: string;\n  };\n}\n\ninterface RightSingleAction extends BaseAction {\n  action_type: 'right_single';\n  action_inputs: {\n    start_box: string;\n  };\n}\n\ninterface TypeAction extends BaseAction {\n  action_type: 'type';\n  action_inputs: {\n    content: string;\n  };\n}\n\ninterface HotkeyAction extends BaseAction {\n  action_type: 'hotkey';\n  action_inputs: {\n    key: string;\n  };\n}\n\ninterface ScrollAction extends BaseAction {\n  action_type: 'scroll';\n  action_inputs: {\n    direction: 'up' | 'down';\n  };\n}\n\ninterface FinishedAction extends BaseAction {\n  action_type: 'finished';\n  action_inputs: {\n    content?: string;\n  };\n}\n\nexport type Action =\n  | ClickAction\n  | LeftDoubleAction\n  | RightSingleAction\n  | DragAction\n  | TypeAction\n  | HotkeyAction\n  | ScrollAction\n  | FinishedAction\n  | WaitAction;\n"],"names":["debug","getDebug","parseUiTarsPlanningResponse","rawResponse","shotSize","uiTarsModelVersion","convertedText","convertBboxToCoordinates","parseResult","actionParser","JSON","text","pattern","replaceMatch","match","x1","y1","x2","y2","x1Num","Number","y1Num","x2Num","y2Num","x","Math","y","cleanedText"],"mappings":";;AAgBA,MAAMA,QAAQC,SAAS;AAOhB,SAASC,4BACdC,WAAmB,EACnBC,QAAc,EACdC,kBAAsC;IAEtC,MAAMC,gBAAgBC,yBAAyBJ;IAC/C,MAAMK,cAAcC,aAAa;QAC/B,YAAYH;QACZ,QAAQ;YAAC;YAAM;SAAK;QACpB,eAAe;YACb,OAAOF,SAAS,KAAK;YACrB,QAAQA,SAAS,MAAM;QACzB;QACA,UAAUC;IACZ;IAEAL,MACE,oBACAK,oBACA,YACAK,KAAK,SAAS,CAACF,YAAY,MAAM;IAGnC,OAAO;QACLL;QACA,SAASK,YAAY,MAAM;IAC7B;AACF;AAKA,SAASD,yBAAyBI,IAAY;IAC5C,MAAMC,UAAU;IAEhB,SAASC,aACPC,KAAa,EACbC,EAAU,EACVC,EAAU,EACVC,EAAU,EACVC,EAAU;QAEV,MAAMC,QAAQC,OAAO,QAAQ,CAACL,IAAI;QAClC,MAAMM,QAAQD,OAAO,QAAQ,CAACJ,IAAI;QAClC,MAAMM,QAAQF,OAAO,QAAQ,CAACH,IAAI;QAClC,MAAMM,QAAQH,OAAO,QAAQ,CAACF,IAAI;QAElC,MAAMM,IAAIC,KAAK,KAAK,CAAEN,AAAAA,CAAAA,QAAQG,KAAI,IAAK;QACvC,MAAMI,IAAID,KAAK,KAAK,CAAEJ,AAAAA,CAAAA,QAAQE,KAAI,IAAK;QAEvC,OAAO,CAAC,CAAC,EAAEC,EAAE,CAAC,EAAEE,EAAE,CAAC,CAAC;IACtB;IAEA,MAAMC,cAAchB,KACjB,OAAO,CAAC,YAAY,IACpB,OAAO,CAAC,2BAA2B;IACtC,OAAOgB,YAAY,OAAO,CAACf,SAASC,cAAc,IAAI;AACxD"}

package/dist/es/ai-model/models/ui-tars/planning.mjs

@@ -0,0 +1,25 @@
+import { transformUiTarsActions } from "./actions.mjs";
+import { parseUiTarsPlanningResponse } from "./parser.mjs";
+import { getSummary, getUiTarsPlanningPrompt } from "./prompt.mjs";
+function createUiTarsPlanner(uiTarsModelVersion) {
+    return {
+        messages: {
+            systemPromptPlacement: 'user-message',
+            buildSystemPrompt: getUiTarsPlanningPrompt,
+            buildUserInstruction: (instruction)=>`<user_instruction>${instruction}</user_instruction>`,
+            buildAssistantContent: (_parsedResponse, rawResponse)=>getSummary(rawResponse)
+        },
+        coordinates: {
+            shape: 'point',
+            order: 'xy',
+            normalizedBy: 1
+        },
+        parseResponse: (rawResponse, { options })=>parseUiTarsPlanningResponse(rawResponse, options.context.shotSize, uiTarsModelVersion),
+        transformActions: (parsedPlanningResponse)=>transformUiTarsActions(parsedPlanningResponse),
+        shouldContinuePlanning: (_parsedResponse, actions)=>actions.every((action)=>'Finished' !== action.type),
+        buildResponseLog: (_parsedResponse, rawResponse)=>getSummary(rawResponse)
+    };
+}
+export { createUiTarsPlanner };
+
+//# sourceMappingURL=planning.mjs.map

package/dist/es/ai-model/models/ui-tars/planning.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/models/ui-tars/planning.mjs","sources":["../../../../../src/ai-model/models/ui-tars/planning.ts"],"sourcesContent":["import type { UITarsModelVersion } from '@aiscene/shared/env';\nimport type { CustomPlanningDefinition } from '../../model-adapter/custom-planning-types';\nimport { transformUiTarsActions } from './actions';\nimport {\n  type UiTarsParsedPlanningResponse,\n  parseUiTarsPlanningResponse,\n} from './parser';\nimport { getSummary, getUiTarsPlanningPrompt } from './prompt';\n\nexport function createUiTarsPlanner(\n  uiTarsModelVersion: UITarsModelVersion,\n): CustomPlanningDefinition<UiTarsParsedPlanningResponse> {\n  return {\n    messages: {\n      systemPromptPlacement: 'user-message',\n      buildSystemPrompt: getUiTarsPlanningPrompt,\n      buildUserInstruction: (instruction) =>\n        `<user_instruction>${instruction}</user_instruction>`,\n      buildAssistantContent: (_parsedResponse, rawResponse) =>\n        getSummary(rawResponse),\n    },\n    coordinates: { shape: 'point', order: 'xy', normalizedBy: 1 },\n    parseResponse: (rawResponse, { options }) => {\n      return parseUiTarsPlanningResponse(\n        rawResponse,\n        options.context.shotSize,\n        uiTarsModelVersion,\n      );\n    },\n    transformActions: (parsedPlanningResponse) => {\n      return transformUiTarsActions(parsedPlanningResponse);\n    },\n    shouldContinuePlanning: (_parsedResponse, actions) =>\n      actions.every((action) => action.type !== 'Finished'),\n    buildResponseLog: (_parsedResponse, rawResponse) => getSummary(rawResponse),\n  };\n}\n"],"names":["createUiTarsPlanner","uiTarsModelVersion","getUiTarsPlanningPrompt","instruction","_parsedResponse","rawResponse","getSummary","options","parseUiTarsPlanningResponse","parsedPlanningResponse","transformUiTarsActions","actions","action"],"mappings":";;;AASO,SAASA,oBACdC,kBAAsC;IAEtC,OAAO;QACL,UAAU;YACR,uBAAuB;YACvB,mBAAmBC;YACnB,sBAAsB,CAACC,cACrB,CAAC,kBAAkB,EAAEA,YAAY,mBAAmB,CAAC;YACvD,uBAAuB,CAACC,iBAAiBC,cACvCC,WAAWD;QACf;QACA,aAAa;YAAE,OAAO;YAAS,OAAO;YAAM,cAAc;QAAE;QAC5D,eAAe,CAACA,aAAa,EAAEE,OAAO,EAAE,GAC/BC,4BACLH,aACAE,QAAQ,OAAO,CAAC,QAAQ,EACxBN;QAGJ,kBAAkB,CAACQ,yBACVC,uBAAuBD;QAEhC,wBAAwB,CAACL,iBAAiBO,UACxCA,QAAQ,KAAK,CAAC,CAACC,SAAWA,AAAgB,eAAhBA,OAAO,IAAI;QACvC,kBAAkB,CAACR,iBAAiBC,cAAgBC,WAAWD;IACjE;AACF"}

package/dist/es/ai-model/models/ui-tars/prompt.mjs

@@ -0,0 +1,36 @@
+import { getPreferredLanguage } from "@aiscene/shared/env";
+function getUiTarsPlanningPrompt() {
+    const preferredLanguage = getPreferredLanguage();
+    return `
+You are a GUI agent. You are given a task and your action history, with screenshots. You need to perform the next action to complete the task. 
+
+## Output Format
+\`\`\`
+Thought: ...
+Action: ...
+\`\`\`
+
+## Action Space
+
+click(start_box='[x1, y1, x2, y2]')
+left_double(start_box='[x1, y1, x2, y2]')
+right_single(start_box='[x1, y1, x2, y2]')
+drag(start_box='[x1, y1, x2, y2]', end_box='[x3, y3, x4, y4]')
+hotkey(key='')
+type(content='xxx') # Use escape characters \\', \\\", and \\n in content part to ensure we can parse the content in normal python string format. If you want to submit your input, use \\n at the end of content. 
+scroll(start_box='[x1, y1, x2, y2]', direction='down or up or right or left')
+wait() #Sleep for 5s and take a screenshot to check for any changes.
+finished(content='xxx') # Use escape characters \\', \\", and \\n in content part to ensure we can parse the content in normal python string format.
+
+
+## Note
+- Use ${preferredLanguage} in \`Thought\` part.
+- Write a small plan and finally summarize your next action (with its target element) in one sentence in \`Thought\` part.
+
+## User Instruction
+`;
+}
+const getSummary = (prediction)=>prediction.replace(/Reflection:[\s\S]*?(?=Action_Summary:|Action:|$)/g, '').trim();
+export { getSummary, getUiTarsPlanningPrompt };
+
+//# sourceMappingURL=prompt.mjs.map

package/dist/es/ai-model/models/ui-tars/prompt.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/models/ui-tars/prompt.mjs","sources":["../../../../../src/ai-model/models/ui-tars/prompt.ts"],"sourcesContent":["import { getPreferredLanguage } from '@aiscene/shared/env';\n\nexport function getUiTarsPlanningPrompt(): string {\n  const preferredLanguage = getPreferredLanguage();\n\n  return `\nYou are a GUI agent. You are given a task and your action history, with screenshots. You need to perform the next action to complete the task. \n\n## Output Format\n\\`\\`\\`\nThought: ...\nAction: ...\n\\`\\`\\`\n\n## Action Space\n\nclick(start_box='[x1, y1, x2, y2]')\nleft_double(start_box='[x1, y1, x2, y2]')\nright_single(start_box='[x1, y1, x2, y2]')\ndrag(start_box='[x1, y1, x2, y2]', end_box='[x3, y3, x4, y4]')\nhotkey(key='')\ntype(content='xxx') # Use escape characters \\\\', \\\\\\\", and \\\\n in content part to ensure we can parse the content in normal python string format. If you want to submit your input, use \\\\n at the end of content. \nscroll(start_box='[x1, y1, x2, y2]', direction='down or up or right or left')\nwait() #Sleep for 5s and take a screenshot to check for any changes.\nfinished(content='xxx') # Use escape characters \\\\', \\\\\", and \\\\n in content part to ensure we can parse the content in normal python string format.\n\n\n## Note\n- Use ${preferredLanguage} in \\`Thought\\` part.\n- Write a small plan and finally summarize your next action (with its target element) in one sentence in \\`Thought\\` part.\n\n## User Instruction\n`;\n}\n\nexport const getSummary = (prediction: string) =>\n  prediction\n    .replace(/Reflection:[\\s\\S]*?(?=Action_Summary:|Action:|$)/g, '')\n    .trim();\n"],"names":["getUiTarsPlanningPrompt","preferredLanguage","getPreferredLanguage","getSummary","prediction"],"mappings":";AAEO,SAASA;IACd,MAAMC,oBAAoBC;IAE1B,OAAO,CAAC;;;;;;;;;;;;;;;;;;;;;;;MAuBJ,EAAED,kBAAkB;;;;AAI1B,CAAC;AACD;AAEO,MAAME,aAAa,CAACC,aACzBA,WACG,OAAO,CAAC,qDAAqD,IAC7D,IAAI"}

package/dist/es/ai-model/models/utils/intent.mjs

@@ -0,0 +1,6 @@
+function isLocateIntent(intent) {
+    return 'default' === intent;
+}
+export { isLocateIntent };
+
+//# sourceMappingURL=intent.mjs.map

package/dist/es/ai-model/models/utils/intent.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/models/utils/intent.mjs","sources":["../../../../../src/ai-model/models/utils/intent.ts"],"sourcesContent":["import type { TIntent } from '@aiscene/shared/env';\n\nexport function isLocateIntent(intent?: TIntent): boolean {\n  return intent === 'default';\n}\n"],"names":["isLocateIntent","intent"],"mappings":"AAEO,SAASA,eAAeC,MAAgB;IAC7C,OAAOA,AAAW,cAAXA;AACT"}

package/dist/es/ai-model/prompt/insight/index.mjs

@@ -0,0 +1,3 @@
+import { buildInsightSystemPrompt } from "./system-prompt.mjs";
+import { buildTypeQueryDemandValue, extractDataQueryPrompt } from "./user-prompt.mjs";
+export { buildInsightSystemPrompt, buildTypeQueryDemandValue, extractDataQueryPrompt };

package/dist/es/ai-model/prompt/insight/system-prompt.mjs

@@ -0,0 +1,104 @@
+import { getPreferredLanguage } from "@aiscene/shared/env";
+function buildInsightContextPrompt({ screenshotIncluded, referenceImagesIncluded }) {
+    const contextPrompts = [
+        "The user will give you data requirements in <DATA_DEMAND>. You need to understand the user's requirements and extract the data satisfying the <DATA_DEMAND>."
+    ];
+    if (screenshotIncluded) contextPrompts.push('The user will provide a current screenshot to evaluate, and may provide its contents. Base your answer on the current screenshot and its contents when provided. Treat them as the primary source of truth for what is currently visible or true.');
+    else contextPrompts.push('The user will not provide a current screenshot. Use only the supplied page contents and other inputs, and do not infer unsupported visual details.');
+    if (referenceImagesIncluded) {
+        const referenceImagesPrompt = 'Reference images are supporting context only unless <DATA_DEMAND> explicitly asks for comparison, matching, or reasoning about them.';
+        contextPrompts.push(screenshotIncluded ? `${referenceImagesPrompt} Do not conclude that something exists in the current screenshot solely because it appears in a reference image; when they conflict, trust the current screenshot and its contents.` : `${referenceImagesPrompt} Do not treat reference images as direct evidence of the current state unless the demand explicitly asks you to use them that way.`);
+    }
+    return contextPrompts.join('\n\n');
+}
+function buildInsightSystemPrompt(options) {
+    const preferredLanguage = getPreferredLanguage();
+    const screenshotIncluded = options.screenshotIncluded ?? true;
+    const referenceImagesIncluded = options.referenceImagesIncluded ?? false;
+    const { responsePrefix, dataOutput } = options.insightProtocol;
+    const contextPrompt = buildInsightContextPrompt({
+        screenshotIncluded,
+        referenceImagesIncluded
+    });
+    return `
+You are a versatile professional in software UI design and testing. Your outstanding contributions will impact the user experience of billions of users.
+
+${contextPrompt}
+
+If a key specifies a JSON data type (such as Number, String, Boolean, Object, Array), ensure the returned value strictly matches that data type.
+
+When DATA_DEMAND is a JSON object, the keys in your response must exactly match the keys in DATA_DEMAND. Do not rename, translate, or substitute any key.${dataOutput.rules ? `\n\n${dataOutput.rules}` : ''}
+
+
+Return in the following XML format:
+${responsePrefix ? responsePrefix : ''}
+<observation>brief evidence observed for the extraction, less than 300 words. Use ${preferredLanguage} in this field.</observation>
+${dataOutput.placeholder}
+<errors>optional error messages as JSON array, e.g., ["error1", "error2"]</errors>
+
+# Example 1
+For example, if the DATA_DEMAND is:
+
+<DATA_DEMAND>
+{
+  "name": "name shows on the left panel, string",
+  "age": "age shows on the right panel, number",
+  "isAdmin": "if the user is admin, boolean"
+}
+</DATA_DEMAND>
+
+By viewing the screenshot and page contents, you can extract the following data:
+
+${responsePrefix ? responsePrefix : ''}
+<observation>According to the screenshot, i can see ...</observation>
+${dataOutput.buildExample(`{
+  "name": "John",
+  "age": 30,
+  "isAdmin": true
+}`.trim())}
+
+# Example 2
+If the DATA_DEMAND is:
+
+<DATA_DEMAND>
+the todo items list, string[]
+</DATA_DEMAND>
+
+By viewing the screenshot and page contents, you can extract the following data:
+
+${responsePrefix ? responsePrefix : ''}
+<observation>According to the screenshot, i can see ...</observation>
+${dataOutput.buildExample('["todo 1", "todo 2", "todo 3"]')}
+
+# Example 3
+If the DATA_DEMAND is:
+
+<DATA_DEMAND>
+the page title, string
+</DATA_DEMAND>
+
+By viewing the screenshot and page contents, you can extract the following data:
+
+${responsePrefix ? responsePrefix : ''}
+<observation>According to the screenshot, i can see ...</observation>
+${dataOutput.buildExample('"todo list"')}
+
+# Example 4
+If the DATA_DEMAND is:
+
+<DATA_DEMAND>
+{
+  "StatementIsTruthy": "Boolean, is it currently the SMS page?"
+}
+</DATA_DEMAND>
+
+By viewing the screenshot and page contents, you can extract the following data:
+
+${responsePrefix ? responsePrefix : ''}
+<observation>According to the screenshot, i can see ...</observation>
+${dataOutput.buildExample('{ "StatementIsTruthy": true }')}
+`;
+}
+export { buildInsightSystemPrompt };
+
+//# sourceMappingURL=system-prompt.mjs.map

package/dist/es/ai-model/prompt/insight/system-prompt.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/prompt/insight/system-prompt.mjs","sources":["../../../../../src/ai-model/prompt/insight/system-prompt.ts"],"sourcesContent":["import { getPreferredLanguage } from '@aiscene/shared/env';\nimport type { InsightProtocol } from '../../model-adapter/insight-protocol';\n\nfunction buildInsightContextPrompt({\n  screenshotIncluded,\n  referenceImagesIncluded,\n}: {\n  screenshotIncluded: boolean;\n  referenceImagesIncluded: boolean;\n}) {\n  const contextPrompts = [\n    \"The user will give you data requirements in <DATA_DEMAND>. You need to understand the user's requirements and extract the data satisfying the <DATA_DEMAND>.\",\n  ];\n\n  if (screenshotIncluded) {\n    contextPrompts.push(\n      'The user will provide a current screenshot to evaluate, and may provide its contents. Base your answer on the current screenshot and its contents when provided. Treat them as the primary source of truth for what is currently visible or true.',\n    );\n  } else {\n    contextPrompts.push(\n      'The user will not provide a current screenshot. Use only the supplied page contents and other inputs, and do not infer unsupported visual details.',\n    );\n  }\n\n  if (referenceImagesIncluded) {\n    const referenceImagesPrompt =\n      'Reference images are supporting context only unless <DATA_DEMAND> explicitly asks for comparison, matching, or reasoning about them.';\n    contextPrompts.push(\n      screenshotIncluded\n        ? `${referenceImagesPrompt} Do not conclude that something exists in the current screenshot solely because it appears in a reference image; when they conflict, trust the current screenshot and its contents.`\n        : `${referenceImagesPrompt} Do not treat reference images as direct evidence of the current state unless the demand explicitly asks you to use them that way.`,\n    );\n  }\n\n  return contextPrompts.join('\\n\\n');\n}\n\nexport function buildInsightSystemPrompt(options: {\n  screenshotIncluded?: boolean;\n  referenceImagesIncluded?: boolean;\n  insightProtocol: InsightProtocol;\n}) {\n  const preferredLanguage = getPreferredLanguage();\n  const screenshotIncluded = options.screenshotIncluded ?? true;\n  const referenceImagesIncluded = options.referenceImagesIncluded ?? false;\n  const { responsePrefix, dataOutput } = options.insightProtocol;\n\n  const contextPrompt = buildInsightContextPrompt({\n    screenshotIncluded,\n    referenceImagesIncluded,\n  });\n\n  return `\nYou are a versatile professional in software UI design and testing. Your outstanding contributions will impact the user experience of billions of users.\n\n${contextPrompt}\n\nIf a key specifies a JSON data type (such as Number, String, Boolean, Object, Array), ensure the returned value strictly matches that data type.\n\nWhen DATA_DEMAND is a JSON object, the keys in your response must exactly match the keys in DATA_DEMAND. Do not rename, translate, or substitute any key.${dataOutput.rules ? `\\n\\n${dataOutput.rules}` : ''}\n\n\nReturn in the following XML format:\n${responsePrefix ? responsePrefix : ''}\n<observation>brief evidence observed for the extraction, less than 300 words. Use ${preferredLanguage} in this field.</observation>\n${dataOutput.placeholder}\n<errors>optional error messages as JSON array, e.g., [\"error1\", \"error2\"]</errors>\n\n# Example 1\nFor example, if the DATA_DEMAND is:\n\n<DATA_DEMAND>\n{\n  \"name\": \"name shows on the left panel, string\",\n  \"age\": \"age shows on the right panel, number\",\n  \"isAdmin\": \"if the user is admin, boolean\"\n}\n</DATA_DEMAND>\n\nBy viewing the screenshot and page contents, you can extract the following data:\n\n${responsePrefix ? responsePrefix : ''}\n<observation>According to the screenshot, i can see ...</observation>\n${dataOutput.buildExample(\n  `{\n  \"name\": \"John\",\n  \"age\": 30,\n  \"isAdmin\": true\n}`.trim(),\n)}\n\n# Example 2\nIf the DATA_DEMAND is:\n\n<DATA_DEMAND>\nthe todo items list, string[]\n</DATA_DEMAND>\n\nBy viewing the screenshot and page contents, you can extract the following data:\n\n${responsePrefix ? responsePrefix : ''}\n<observation>According to the screenshot, i can see ...</observation>\n${dataOutput.buildExample('[\"todo 1\", \"todo 2\", \"todo 3\"]')}\n\n# Example 3\nIf the DATA_DEMAND is:\n\n<DATA_DEMAND>\nthe page title, string\n</DATA_DEMAND>\n\nBy viewing the screenshot and page contents, you can extract the following data:\n\n${responsePrefix ? responsePrefix : ''}\n<observation>According to the screenshot, i can see ...</observation>\n${dataOutput.buildExample('\"todo list\"')}\n\n# Example 4\nIf the DATA_DEMAND is:\n\n<DATA_DEMAND>\n{\n  \"StatementIsTruthy\": \"Boolean, is it currently the SMS page?\"\n}\n</DATA_DEMAND>\n\nBy viewing the screenshot and page contents, you can extract the following data:\n\n${responsePrefix ? responsePrefix : ''}\n<observation>According to the screenshot, i can see ...</observation>\n${dataOutput.buildExample('{ \"StatementIsTruthy\": true }')}\n`;\n}\n"],"names":["buildInsightContextPrompt","screenshotIncluded","referenceImagesIncluded","contextPrompts","referenceImagesPrompt","buildInsightSystemPrompt","options","preferredLanguage","getPreferredLanguage","responsePrefix","dataOutput","contextPrompt"],"mappings":";AAGA,SAASA,0BAA0B,EACjCC,kBAAkB,EAClBC,uBAAuB,EAIxB;IACC,MAAMC,iBAAiB;QACrB;KACD;IAED,IAAIF,oBACFE,eAAe,IAAI,CACjB;SAGFA,eAAe,IAAI,CACjB;IAIJ,IAAID,yBAAyB;QAC3B,MAAME,wBACJ;QACFD,eAAe,IAAI,CACjBF,qBACI,GAAGG,sBAAsB,mLAAmL,CAAC,GAC7M,GAAGA,sBAAsB,kIAAkI,CAAC;IAEpK;IAEA,OAAOD,eAAe,IAAI,CAAC;AAC7B;AAEO,SAASE,yBAAyBC,OAIxC;IACC,MAAMC,oBAAoBC;IAC1B,MAAMP,qBAAqBK,QAAQ,kBAAkB,IAAI;IACzD,MAAMJ,0BAA0BI,QAAQ,uBAAuB,IAAI;IACnE,MAAM,EAAEG,cAAc,EAAEC,UAAU,EAAE,GAAGJ,QAAQ,eAAe;IAE9D,MAAMK,gBAAgBX,0BAA0B;QAC9CC;QACAC;IACF;IAEA,OAAO,CAAC;;;AAGV,EAAES,cAAc;;;;yJAIyI,EAAED,WAAW,KAAK,GAAG,CAAC,IAAI,EAAEA,WAAW,KAAK,EAAE,GAAG,GAAG;;;;AAI7M,EAAED,iBAAiBA,iBAAiB,GAAG;kFAC2C,EAAEF,kBAAkB;AACtG,EAAEG,WAAW,WAAW,CAAC;;;;;;;;;;;;;;;;AAgBzB,EAAED,iBAAiBA,iBAAiB,GAAG;;AAEvC,EAAEC,WAAW,YAAY,CACvB,CAAC;;;;CAIF,CAAC,CAAC,IAAI,IACL;;;;;;;;;;;AAWF,EAAED,iBAAiBA,iBAAiB,GAAG;;AAEvC,EAAEC,WAAW,YAAY,CAAC,kCAAkC;;;;;;;;;;;AAW5D,EAAED,iBAAiBA,iBAAiB,GAAG;;AAEvC,EAAEC,WAAW,YAAY,CAAC,eAAe;;;;;;;;;;;;;AAazC,EAAED,iBAAiBA,iBAAiB,GAAG;;AAEvC,EAAEC,WAAW,YAAY,CAAC,iCAAiC;AAC3D,CAAC;AACD"}

package/dist/es/ai-model/prompt/insight/user-prompt.mjs

@@ -0,0 +1,23 @@
+function buildTypeQueryDemandValue(type, demand) {
+    const currentScreenshotConstraint = 'based on the current screenshot and its contents if provided, unless the user explicitly asks to compare with reference images';
+    if ('Assert' === type) return `Boolean, ${currentScreenshotConstraint}, whether the following statement is true: ${demand}`;
+    if ('WaitFor' === type) return `Boolean, the user wants to do some 'wait for' operation. ${currentScreenshotConstraint}, please check whether the following statement is true: ${demand}`;
+    return `${type}, ${currentScreenshotConstraint}, ${demand}`;
+}
+const extractDataQueryPrompt = (pageDescription, dataQuery, context)=>{
+    const dataQueryText = 'string' == typeof dataQuery ? dataQuery : JSON.stringify(dataQuery, null, 2);
+    const trimmedContext = context?.trim();
+    const contextSection = trimmedContext ? `\n<CONTEXT>\n${trimmedContext}\n</CONTEXT>\n` : '';
+    return `
+<PageDescription>
+${pageDescription}
+</PageDescription>
+${contextSection}
+<DATA_DEMAND>
+${dataQueryText}
+</DATA_DEMAND>
+  `;
+};
+export { buildTypeQueryDemandValue, extractDataQueryPrompt };
+
+//# sourceMappingURL=user-prompt.mjs.map

package/dist/es/ai-model/prompt/insight/user-prompt.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/prompt/insight/user-prompt.mjs","sources":["../../../../../src/ai-model/prompt/insight/user-prompt.ts"],"sourcesContent":["import type { ServiceExtractParam } from '@/types';\n\nexport function buildTypeQueryDemandValue(\n  type: 'Boolean' | 'Number' | 'String' | 'Assert' | 'WaitFor',\n  demand: ServiceExtractParam,\n) {\n  const currentScreenshotConstraint =\n    'based on the current screenshot and its contents if provided, unless the user explicitly asks to compare with reference images';\n\n  if (type === 'Assert') {\n    return `Boolean, ${currentScreenshotConstraint}, whether the following statement is true: ${demand}`;\n  }\n\n  if (type === 'WaitFor') {\n    return `Boolean, the user wants to do some 'wait for' operation. ${currentScreenshotConstraint}, please check whether the following statement is true: ${demand}`;\n  }\n\n  return `${type}, ${currentScreenshotConstraint}, ${demand}`;\n}\n\nexport const extractDataQueryPrompt = (\n  pageDescription: string,\n  dataQuery: string | Record<string, string>,\n  context?: string,\n) => {\n  const dataQueryText =\n    typeof dataQuery === 'string'\n      ? dataQuery\n      : JSON.stringify(dataQuery, null, 2);\n\n  const trimmedContext = context?.trim();\n  const contextSection = trimmedContext\n    ? `\\n<CONTEXT>\\n${trimmedContext}\\n</CONTEXT>\\n`\n    : '';\n\n  return `\n<PageDescription>\n${pageDescription}\n</PageDescription>\n${contextSection}\n<DATA_DEMAND>\n${dataQueryText}\n</DATA_DEMAND>\n  `;\n};\n"],"names":["buildTypeQueryDemandValue","type","demand","currentScreenshotConstraint","extractDataQueryPrompt","pageDescription","dataQuery","context","dataQueryText","JSON","trimmedContext","contextSection"],"mappings":"AAEO,SAASA,0BACdC,IAA4D,EAC5DC,MAA2B;IAE3B,MAAMC,8BACJ;IAEF,IAAIF,AAAS,aAATA,MACF,OAAO,CAAC,SAAS,EAAEE,4BAA4B,2CAA2C,EAAED,QAAQ;IAGtG,IAAID,AAAS,cAATA,MACF,OAAO,CAAC,yDAAyD,EAAEE,4BAA4B,wDAAwD,EAAED,QAAQ;IAGnK,OAAO,GAAGD,KAAK,EAAE,EAAEE,4BAA4B,EAAE,EAAED,QAAQ;AAC7D;AAEO,MAAME,yBAAyB,CACpCC,iBACAC,WACAC;IAEA,MAAMC,gBACJ,AAAqB,YAArB,OAAOF,YACHA,YACAG,KAAK,SAAS,CAACH,WAAW,MAAM;IAEtC,MAAMI,iBAAiBH,SAAS;IAChC,MAAMI,iBAAiBD,iBACnB,CAAC,aAAa,EAAEA,eAAe,cAAc,CAAC,GAC9C;IAEJ,OAAO,CAAC;;AAEV,EAAEL,gBAAgB;;AAElB,EAAEM,eAAe;;AAEjB,EAAEH,cAAc;;EAEd,CAAC;AACH"}

package/dist/es/ai-model/prompt/locate/element-system-prompt.mjs

@@ -0,0 +1,12 @@
+import { locateGroundingRules } from "./grounding-rules.mjs";
+function buildElementLocateSystemPrompt({ systemPromptIntroduction, responseInstructions }) {
+    return `
+${systemPromptIntroduction}
+
+${locateGroundingRules()}
+
+${responseInstructions}`;
+}
+export { buildElementLocateSystemPrompt };
+
+//# sourceMappingURL=element-system-prompt.mjs.map

package/dist/es/ai-model/prompt/locate/element-system-prompt.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/prompt/locate/element-system-prompt.mjs","sources":["../../../../../src/ai-model/prompt/locate/element-system-prompt.ts"],"sourcesContent":["import { locateGroundingRules } from './grounding-rules';\n\nexport function buildElementLocateSystemPrompt({\n  systemPromptIntroduction,\n  responseInstructions,\n}: {\n  systemPromptIntroduction: string;\n  responseInstructions: string;\n}) {\n  return `\n${systemPromptIntroduction}\n\n${locateGroundingRules()}\n\n${responseInstructions}`;\n}\n"],"names":["buildElementLocateSystemPrompt","systemPromptIntroduction","responseInstructions","locateGroundingRules"],"mappings":";AAEO,SAASA,+BAA+B,EAC7CC,wBAAwB,EACxBC,oBAAoB,EAIrB;IACC,OAAO,CAAC;AACV,EAAED,yBAAyB;;AAE3B,EAAEE,uBAAuB;;AAEzB,EAAED,sBAAsB;AACxB"}

package/dist/es/ai-model/prompt/locate/grounding-rules.mjs

@@ -0,0 +1,13 @@
+function locateGroundingRules() {
+    return `## Important Notes for Locating Elements:
+- First identify the target primitive from the user's description, then locate that primitive only. Treat labels, owners, rows, columns, and surrounding text as context unless the description says they are the target.
+- If the target itself is visible text, link text, status text, table cell text, or header text, return only the tight visible text region, not the entire control, row, sentence, or container.
+- If a text or link target wraps across multiple lines, do not return one large box covering the whole wrapped text. Return a tight box around a distinctive visible segment of the target text; for CJK link labels, the first 2-4 visible characters are enough when unique.
+- If the target is an input/select/filter field body, current value area, or blank field region, return that field/control body or value region. Do not retarget to a trailing search icon, dropdown arrow, clear button, or nearby table header.
+- If the target is an icon, arrow, checkbox, radio, or accessory control, return only that glyph/control region. Do not return adjacent owner text.
+- If the target is a tiny icon/control among adjacent similar icons, use the described local order or relative position within that group and return only that glyph/control.
+- If the same text appears in multiple regions, obey the described owner region first, such as filter bar vs table header.`;
+}
+export { locateGroundingRules };
+
+//# sourceMappingURL=grounding-rules.mjs.map

package/dist/es/ai-model/prompt/locate/grounding-rules.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/prompt/locate/grounding-rules.mjs","sources":["../../../../../src/ai-model/prompt/locate/grounding-rules.ts"],"sourcesContent":["export function locateGroundingRules() {\n  return `## Important Notes for Locating Elements:\n- First identify the target primitive from the user's description, then locate that primitive only. Treat labels, owners, rows, columns, and surrounding text as context unless the description says they are the target.\n- If the target itself is visible text, link text, status text, table cell text, or header text, return only the tight visible text region, not the entire control, row, sentence, or container.\n- If a text or link target wraps across multiple lines, do not return one large box covering the whole wrapped text. Return a tight box around a distinctive visible segment of the target text; for CJK link labels, the first 2-4 visible characters are enough when unique.\n- If the target is an input/select/filter field body, current value area, or blank field region, return that field/control body or value region. Do not retarget to a trailing search icon, dropdown arrow, clear button, or nearby table header.\n- If the target is an icon, arrow, checkbox, radio, or accessory control, return only that glyph/control region. Do not return adjacent owner text.\n- If the target is a tiny icon/control among adjacent similar icons, use the described local order or relative position within that group and return only that glyph/control.\n- If the same text appears in multiple regions, obey the described owner region first, such as filter bar vs table header.`;\n}\n"],"names":["locateGroundingRules"],"mappings":"AAAO,SAASA;IACd,OAAO,CAAC;;;;;;;0HAOgH,CAAC;AAC3H"}

package/dist/es/ai-model/prompt/locate/index.mjs

@@ -0,0 +1,4 @@
+import { buildElementLocateSystemPrompt } from "./element-system-prompt.mjs";
+import { locateGroundingRules } from "./grounding-rules.mjs";
+import { buildSearchAreaLocateSystemPrompt } from "./search-area-system-prompt.mjs";
+export { buildElementLocateSystemPrompt, buildSearchAreaLocateSystemPrompt, locateGroundingRules };

package/dist/es/ai-model/prompt/locate/search-area-system-prompt.mjs

@@ -0,0 +1,9 @@
+function buildSearchAreaLocateSystemPrompt({ systemPromptIntroduction, responseInstructions }) {
+    return `
+${systemPromptIntroduction}
+
+${responseInstructions}`;
+}
+export { buildSearchAreaLocateSystemPrompt };
+
+//# sourceMappingURL=search-area-system-prompt.mjs.map

package/dist/es/ai-model/prompt/locate/search-area-system-prompt.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/prompt/locate/search-area-system-prompt.mjs","sources":["../../../../../src/ai-model/prompt/locate/search-area-system-prompt.ts"],"sourcesContent":["export function buildSearchAreaLocateSystemPrompt({\n  systemPromptIntroduction,\n  responseInstructions,\n}: {\n  systemPromptIntroduction: string;\n  responseInstructions: string;\n}) {\n  return `\n${systemPromptIntroduction}\n\n${responseInstructions}`;\n}\n"],"names":["buildSearchAreaLocateSystemPrompt","systemPromptIntroduction","responseInstructions"],"mappings":"AAAO,SAASA,kCAAkC,EAChDC,wBAAwB,EACxBC,oBAAoB,EAIrB;IACC,OAAO,CAAC;AACV,EAAED,yBAAyB;;AAE3B,EAAEC,sBAAsB;AACxB"}

package/dist/es/ai-model/prompt/planning/action-output-example.mjs

@@ -0,0 +1,52 @@
+import { findAllMidsceneLocatorField } from "../../../common.mjs";
+import { actionInputParamSchema, actionTapParamSchema } from "../../../device/index.mjs";
+const injectLocateResultIntoSample = (sample, locateFields, promptSpec, locateResultExampleIndex)=>{
+    const result = {
+        ...sample
+    };
+    let exampleIndex = locateResultExampleIndex;
+    for (const field of locateFields)if (result[field] && 'object' == typeof result[field] && result[field].prompt) {
+        result[field] = {
+            ...result[field],
+            [promptSpec.resultKey]: promptSpec.exampleValues[exampleIndex % promptSpec.exampleValues.length]
+        };
+        exampleIndex++;
+    }
+    return result;
+};
+const buildActionOutputExample = (action, { locatePromptSpec, locateResultExampleIndex = 0, buildActionOutput })=>{
+    if (!action.sample || 'object' != typeof action.sample) return;
+    const locateFields = findAllMidsceneLocatorField(action.paramSchema);
+    const sampleWithLocateResult = locatePromptSpec ? injectLocateResultIntoSample(action.sample, locateFields, locatePromptSpec, locateResultExampleIndex) : action.sample;
+    return buildActionOutput({
+        actionName: action.name,
+        param: sampleWithLocateResult,
+        locateFields,
+        locateResultKey: locatePromptSpec?.resultKey
+    });
+};
+const createSampleTapAction = (prompt)=>{
+    const sample = {
+        locate: {
+            prompt
+        }
+    };
+    return {
+        name: 'Tap',
+        paramSchema: actionTapParamSchema,
+        sample
+    };
+};
+const createSampleInputAction = (value)=>{
+    const sample = {
+        value
+    };
+    return {
+        name: 'Input',
+        paramSchema: actionInputParamSchema,
+        sample
+    };
+};
+export { buildActionOutputExample, createSampleInputAction, createSampleTapAction };
+
+//# sourceMappingURL=action-output-example.mjs.map

package/dist/es/ai-model/prompt/planning/action-output-example.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/prompt/planning/action-output-example.mjs","sources":["../../../../../src/ai-model/prompt/planning/action-output-example.ts"],"sourcesContent":["import { findAllMidsceneLocatorField } from '@/common';\nimport { actionInputParamSchema, actionTapParamSchema } from '@/device';\nimport type { DeviceAction } from '@/types';\nimport type { z } from 'zod';\nimport type { PlanningActionOutputProtocol } from '../../model-adapter/planning-protocol';\nimport type { LocateResultPromptSpec } from '../../shared/model-locate-result';\n\nexport type ActionOutputExampleDefinition = Pick<\n  DeviceAction<any>,\n  'name' | 'paramSchema' | 'sample'\n>;\n\nconst injectLocateResultIntoSample = (\n  sample: Record<string, any>,\n  locateFields: string[],\n  promptSpec: LocateResultPromptSpec,\n  locateResultExampleIndex: number,\n): Record<string, any> => {\n  const result = { ...sample };\n  let exampleIndex = locateResultExampleIndex;\n\n  for (const field of locateFields) {\n    if (\n      result[field] &&\n      typeof result[field] === 'object' &&\n      result[field].prompt\n    ) {\n      result[field] = {\n        ...result[field],\n        [promptSpec.resultKey]:\n          promptSpec.exampleValues[\n            exampleIndex % promptSpec.exampleValues.length\n          ],\n      };\n      exampleIndex++;\n    }\n  }\n\n  return result;\n};\n\nexport const buildActionOutputExample = (\n  action: ActionOutputExampleDefinition,\n  {\n    locatePromptSpec,\n    locateResultExampleIndex = 0,\n    buildActionOutput,\n  }: {\n    locatePromptSpec?: LocateResultPromptSpec;\n    /**\n     * Selects the locate result example value injected into this action.\n     * Keep the same index for the same example across multi-turn prompt\n     * variants, and use different indexes for different examples to avoid\n     * reusing identical locate coordinates.\n     */\n    locateResultExampleIndex?: number;\n    buildActionOutput: PlanningActionOutputProtocol['buildActionOutput'];\n  },\n) => {\n  if (!action.sample || typeof action.sample !== 'object') {\n    return undefined;\n  }\n\n  const locateFields = findAllMidsceneLocatorField(action.paramSchema);\n  const sampleWithLocateResult = locatePromptSpec\n    ? injectLocateResultIntoSample(\n        action.sample,\n        locateFields,\n        locatePromptSpec,\n        locateResultExampleIndex,\n      )\n    : action.sample;\n\n  return buildActionOutput({\n    actionName: action.name,\n    param: sampleWithLocateResult,\n    locateFields,\n    locateResultKey: locatePromptSpec?.resultKey,\n  });\n};\n\nexport const createSampleTapAction = (\n  prompt: string,\n): ActionOutputExampleDefinition => {\n  const sample: z.input<typeof actionTapParamSchema> = {\n    locate: { prompt },\n  };\n  return {\n    name: 'Tap',\n    paramSchema: actionTapParamSchema,\n    sample,\n  };\n};\n\nexport const createSampleInputAction = (\n  value: string,\n): ActionOutputExampleDefinition => {\n  const sample: z.input<typeof actionInputParamSchema> = { value };\n  return {\n    name: 'Input',\n    paramSchema: actionInputParamSchema,\n    sample,\n  };\n};\n"],"names":["injectLocateResultIntoSample","sample","locateFields","promptSpec","locateResultExampleIndex","result","exampleIndex","field","buildActionOutputExample","action","locatePromptSpec","buildActionOutput","findAllMidsceneLocatorField","sampleWithLocateResult","createSampleTapAction","prompt","actionTapParamSchema","createSampleInputAction","value","actionInputParamSchema"],"mappings":";;AAYA,MAAMA,+BAA+B,CACnCC,QACAC,cACAC,YACAC;IAEA,MAAMC,SAAS;QAAE,GAAGJ,MAAM;IAAC;IAC3B,IAAIK,eAAeF;IAEnB,KAAK,MAAMG,SAASL,aAClB,IACEG,MAAM,CAACE,MAAM,IACb,AAAyB,YAAzB,OAAOF,MAAM,CAACE,MAAM,IACpBF,MAAM,CAACE,MAAM,CAAC,MAAM,EACpB;QACAF,MAAM,CAACE,MAAM,GAAG;YACd,GAAGF,MAAM,CAACE,MAAM;YAChB,CAACJ,WAAW,SAAS,CAAC,EACpBA,WAAW,aAAa,CACtBG,eAAeH,WAAW,aAAa,CAAC,MAAM,CAC/C;QACL;QACAG;IACF;IAGF,OAAOD;AACT;AAEO,MAAMG,2BAA2B,CACtCC,QACA,EACEC,gBAAgB,EAChBN,2BAA2B,CAAC,EAC5BO,iBAAiB,EAWlB;IAED,IAAI,CAACF,OAAO,MAAM,IAAI,AAAyB,YAAzB,OAAOA,OAAO,MAAM,EACxC;IAGF,MAAMP,eAAeU,4BAA4BH,OAAO,WAAW;IACnE,MAAMI,yBAAyBH,mBAC3BV,6BACES,OAAO,MAAM,EACbP,cACAQ,kBACAN,4BAEFK,OAAO,MAAM;IAEjB,OAAOE,kBAAkB;QACvB,YAAYF,OAAO,IAAI;QACvB,OAAOI;QACPX;QACA,iBAAiBQ,kBAAkB;IACrC;AACF;AAEO,MAAMI,wBAAwB,CACnCC;IAEA,MAAMd,SAA+C;QACnD,QAAQ;YAAEc;QAAO;IACnB;IACA,OAAO;QACL,MAAM;QACN,aAAaC;QACbf;IACF;AACF;AAEO,MAAMgB,0BAA0B,CACrCC;IAEA,MAAMjB,SAAiD;QAAEiB;IAAM;IAC/D,OAAO;QACL,MAAM;QACN,aAAaC;QACblB;IACF;AACF"}

package/dist/es/ai-model/prompt/planning/action-space-description.mjs

@@ -0,0 +1,28 @@
+import js_yaml from "js-yaml";
+import { buildActionOutputExample } from "./action-output-example.mjs";
+const serializeActionDescriptions = (actionDescriptions, format)=>{
+    if ('jsonl' === format) return actionDescriptions.map((description)=>JSON.stringify(description)).join('\n');
+    return js_yaml.dump(actionDescriptions, {
+        indent: 2,
+        lineWidth: -1,
+        noRefs: true
+    }).trim();
+};
+const buildPlanningActionSpaceDescription = ({ actionSpace, locatePromptSpec, planningProtocol })=>{
+    const locateFieldDescription = planningProtocol.actionSpaceProtocol.buildLocateFieldDescription(locatePromptSpec);
+    const actionDescriptions = actionSpace.map((action)=>{
+        const actionOutputExample = planningProtocol.actionSpaceProtocol.includeActionOutputExample ? buildActionOutputExample(action, {
+            locatePromptSpec,
+            buildActionOutput: planningProtocol.actionOutputProtocol.buildActionOutput
+        }) : void 0;
+        return planningProtocol.actionSpaceProtocol.buildActionDescription({
+            action,
+            locateFieldDescription,
+            actionOutputExample
+        });
+    });
+    return serializeActionDescriptions(actionDescriptions, planningProtocol.actionSpaceProtocol.format);
+};
+export { buildPlanningActionSpaceDescription, serializeActionDescriptions };
+
+//# sourceMappingURL=action-space-description.mjs.map

package/dist/es/ai-model/prompt/planning/action-space-description.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/prompt/planning/action-space-description.mjs","sources":["../../../../../src/ai-model/prompt/planning/action-space-description.ts"],"sourcesContent":["import type { DeviceAction } from '@/types';\nimport yaml from 'js-yaml';\nimport type {\n  PlanningActionSpaceFormat,\n  StandardPlanningProtocol,\n} from '../../model-adapter/planning-protocol';\nimport type { LocateResultPromptSpec } from '../../shared/model-locate-result';\nimport { buildActionOutputExample } from './action-output-example';\n\ntype BuildPlanningActionSpaceDescriptionInput = {\n  actionSpace: DeviceAction<any>[];\n  locatePromptSpec?: LocateResultPromptSpec;\n  planningProtocol: StandardPlanningProtocol;\n};\n\nexport const serializeActionDescriptions = (\n  actionDescriptions: unknown[],\n  format: PlanningActionSpaceFormat,\n) => {\n  if (format === 'jsonl') {\n    return actionDescriptions\n      .map((description) => JSON.stringify(description))\n      .join('\\n');\n  }\n\n  return yaml\n    .dump(actionDescriptions, {\n      indent: 2,\n      lineWidth: -1,\n      noRefs: true,\n    })\n    .trim();\n};\n\nexport const buildPlanningActionSpaceDescription = ({\n  actionSpace,\n  locatePromptSpec,\n  planningProtocol,\n}: BuildPlanningActionSpaceDescriptionInput) => {\n  const locateFieldDescription =\n    planningProtocol.actionSpaceProtocol.buildLocateFieldDescription(\n      locatePromptSpec,\n    );\n  const actionDescriptions = actionSpace.map((action) => {\n    const actionOutputExample = planningProtocol.actionSpaceProtocol\n      .includeActionOutputExample\n      ? buildActionOutputExample(action, {\n          locatePromptSpec,\n          buildActionOutput:\n            planningProtocol.actionOutputProtocol.buildActionOutput,\n        })\n      : undefined;\n    return planningProtocol.actionSpaceProtocol.buildActionDescription({\n      action,\n      locateFieldDescription,\n      actionOutputExample,\n    });\n  });\n\n  return serializeActionDescriptions(\n    actionDescriptions,\n    planningProtocol.actionSpaceProtocol.format,\n  );\n};\n"],"names":["serializeActionDescriptions","actionDescriptions","format","description","JSON","yaml","buildPlanningActionSpaceDescription","actionSpace","locatePromptSpec","planningProtocol","locateFieldDescription","action","actionOutputExample","buildActionOutputExample","undefined"],"mappings":";;AAeO,MAAMA,8BAA8B,CACzCC,oBACAC;IAEA,IAAIA,AAAW,YAAXA,QACF,OAAOD,mBACJ,GAAG,CAAC,CAACE,cAAgBC,KAAK,SAAS,CAACD,cACpC,IAAI,CAAC;IAGV,OAAOE,QAAAA,IACA,CAACJ,oBAAoB;QACxB,QAAQ;QACR,WAAW;QACX,QAAQ;IACV,GACC,IAAI;AACT;AAEO,MAAMK,sCAAsC,CAAC,EAClDC,WAAW,EACXC,gBAAgB,EAChBC,gBAAgB,EACyB;IACzC,MAAMC,yBACJD,iBAAiB,mBAAmB,CAAC,2BAA2B,CAC9DD;IAEJ,MAAMP,qBAAqBM,YAAY,GAAG,CAAC,CAACI;QAC1C,MAAMC,sBAAsBH,iBAAiB,mBAAmB,CAC7D,0BAA0B,GACzBI,yBAAyBF,QAAQ;YAC/BH;YACA,mBACEC,iBAAiB,oBAAoB,CAAC,iBAAiB;QAC3D,KACAK;QACJ,OAAOL,iBAAiB,mBAAmB,CAAC,sBAAsB,CAAC;YACjEE;YACAD;YACAE;QACF;IACF;IAEA,OAAOZ,4BACLC,oBACAQ,iBAAiB,mBAAmB,CAAC,MAAM;AAE/C"}

package/dist/es/ai-model/prompt/planning/index.mjs

@@ -0,0 +1,6 @@
+import { buildStandardPlanningSystemPrompt } from "./system-prompt.mjs";
+import { buildPlanningActionSpaceDescription, serializeActionDescriptions } from "./action-space-description.mjs";
+import { buildActionOutputExample, createSampleInputAction, createSampleTapAction } from "./action-output-example.mjs";
+import { buildPlanningResponseExample } from "./planning-response-example.mjs";
+import { buildSubGoalsText } from "./sub-goals-text.mjs";
+export { buildActionOutputExample, buildPlanningActionSpaceDescription, buildPlanningResponseExample, buildStandardPlanningSystemPrompt, buildSubGoalsText, createSampleInputAction, createSampleTapAction, serializeActionDescriptions };