@aiscene/core 9.0.6 → 9.0.8

package/dist/es/ai-model/model-adapter/default-planning-protocol.mjs

@@ -0,0 +1,124 @@
+import { getZodDescription, getZodTypeName, isMidsceneLocatorField } from "@aiscene/shared/zod-schema-utils";
+import { getZodDefaultValue } from "../shared/action-schema.mjs";
+import { assertJsonObject } from "../shared/json.mjs";
+import { formatLocateExampleValue } from "../shared/model-locate-result/index.mjs";
+import { extractXMLTag } from "../shared/xml.mjs";
+const buildActionDescription = ({ action, locateFieldDescription, actionOutputExample })=>{
+    const actionDescription = {
+        type: action.name,
+        description: action.description || "No description provided"
+    };
+    if (action.paramSchema) {
+        const schema = action.paramSchema;
+        const isZodObject = schema._def?.typeName === 'ZodObject';
+        if (isZodObject && schema.shape) {
+            const param = {};
+            for (const [key, field] of Object.entries(schema.shape))if (field && 'object' == typeof field) {
+                const isOptional = 'function' == typeof field.isOptional && field.isOptional();
+                const typeDescription = isMidsceneLocatorField(field) ? locateFieldDescription : getZodTypeName(field);
+                const description = getZodDescription(field);
+                const defaultValue = getZodDefaultValue(field);
+                const hasDefault = void 0 !== defaultValue;
+                const paramDescription = {
+                    type: typeDescription
+                };
+                if (isOptional) paramDescription.optional = true;
+                if (description) paramDescription.description = description;
+                if (hasDefault) paramDescription.default = defaultValue;
+                param[key] = paramDescription;
+            }
+            if (Object.keys(param).length > 0) actionDescription.param = param;
+        } else {
+            const typeName = getZodTypeName(schema);
+            const description = getZodDescription(schema);
+            const paramDescription = {
+                type: typeName,
+                instruction: 'Pass the value directly, not as an object.'
+            };
+            if (description) paramDescription.description = description;
+            actionDescription.param = paramDescription;
+        }
+    }
+    if (actionOutputExample) actionDescription.sample = actionOutputExample;
+    return actionDescription;
+};
+const buildLocateFieldDescription = (promptSpec)=>{
+    if (promptSpec) return `{ prompt: string, ${promptSpec.resultKey}: ${promptSpec.resultValueSchema} /* ${promptSpec.resultValueDescription} */ }`;
+    return "{ prompt: string /* description of the target element */ }";
+};
+const serializeActionParam = (param, locateFields, locateResultKey)=>{
+    const locatorObjects = new WeakSet();
+    for (const field of locateFields){
+        const value = param[field];
+        if (value && 'object' == typeof value) locatorObjects.add(value);
+    }
+    const originalJson = JSON.stringify(param);
+    let markerPrefix = '__MIDSCENE_LOCATE_RESULT_EXAMPLE_';
+    while(originalJson.includes(markerPrefix))markerPrefix = `_${markerPrefix}`;
+    const replacements = new Map();
+    const serializedParam = JSON.stringify(param, function(key, value) {
+        if (locatorObjects.has(this) && key === locateResultKey) {
+            const marker = `${markerPrefix}${replacements.size}__`;
+            replacements.set(marker, formatLocateExampleValue(value));
+            return marker;
+        }
+        return value;
+    }, 2);
+    return [
+        ...replacements
+    ].reduce((result, [marker, locateResultValue])=>result.replace(JSON.stringify(marker), locateResultValue), serializedParam);
+};
+const buildPlanningActionOutput = ({ actionName, param, locateFields, locateResultKey })=>`<action-type>${actionName}</action-type>
+<action-param-json>
+${locateFields && locateResultKey ? serializeActionParam(param, locateFields, locateResultKey) : JSON.stringify(param, null, 2)}
+</action-param-json>`;
+const createMidscenePlanningActionOutputParser = (jsonParser)=>(content)=>{
+        const actionType = extractXMLTag(content, 'action-type');
+        const actionParamStr = extractXMLTag(content, 'action-param-json');
+        if (!actionType || 'null' === actionType.toLowerCase()) return null;
+        const type = actionType.split('<')[0].trim();
+        let param;
+        if (actionParamStr) try {
+            param = jsonParser(actionParamStr, {
+                source: 'planning-action-param',
+                preserveStringValueKeys: 'input' === type.toLowerCase() ? [
+                    'value'
+                ] : void 0
+            });
+            assertJsonObject(param);
+        } catch (error) {
+            throw new Error(`Failed to parse action-param-json: ${error}`);
+        }
+        return {
+            type,
+            ...void 0 !== param ? {
+                param
+            } : {}
+        };
+    };
+const createDefaultMidscenePlanningProtocol = ({ jsonParser })=>{
+    const parseActionOutput = createMidscenePlanningActionOutputParser(jsonParser);
+    return {
+        actionSpaceProtocol: {
+            title: 'Supporting actions list',
+            format: 'yaml',
+            includeActionOutputExample: true,
+            buildLocateFieldDescription,
+            buildActionDescription
+        },
+        actionOutputProtocol: {
+            actionOutputTagNames: [
+                'action-type',
+                'action-param-json'
+            ],
+            actionOutputRules: "- Use the <action-type> and <action-param-json> tags to output the action to be executed.\n- The value inside <action-type> MUST exactly match the 'type' field of one action in the Supporting actions list. 'complete' is NOT a valid action-type.\n- Parameter names are strict. Use EXACTLY the field names listed for the selected action. Do NOT invent alias fields. If the selected action provides a \"sample\" field, use the XML structure shown in that sample as the exact format for the action output.",
+            actionOutputPlaceholder: "<action-type>...</action-type>\n<action-param-json>...</action-param-json>",
+            buildActionOutput: buildPlanningActionOutput,
+            parseActionOutput,
+            parseRawLocateParameter: (value)=>value
+        }
+    };
+};
+export { buildActionDescription, buildLocateFieldDescription, buildPlanningActionOutput, createDefaultMidscenePlanningProtocol, createMidscenePlanningActionOutputParser };
+
+//# sourceMappingURL=default-planning-protocol.mjs.map

package/dist/es/ai-model/model-adapter/default-planning-protocol.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/model-adapter/default-planning-protocol.mjs","sources":["../../../../src/ai-model/model-adapter/default-planning-protocol.ts"],"sourcesContent":["import {\n  getZodDescription,\n  getZodTypeName,\n  isMidsceneLocatorField,\n} from '@aiscene/shared/zod-schema-utils';\nimport type { z } from 'zod';\nimport { getZodDefaultValue } from '../shared/action-schema';\nimport { type JsonParser, assertJsonObject } from '../shared/json';\nimport {\n  type LocateResultPromptSpec,\n  formatLocateExampleValue,\n} from '../shared/model-locate-result';\nimport { extractXMLTag } from '../shared/xml';\nimport type {\n  ParsedPlanningLocateParameter,\n  PlanningActionDescriptionBuildInput,\n  PlanningActionOutputBuildInput,\n  StandardPlanningProtocolFactory,\n} from './planning-protocol';\n\ntype ActionParamDescription = {\n  type: string;\n  optional?: true;\n  description?: string;\n  default?: unknown;\n  instruction?: string;\n};\n\ntype ActionDescription = {\n  type: string;\n  description: string;\n  param?: Record<string, ActionParamDescription> | ActionParamDescription;\n  sample?: string;\n};\n\nexport const buildActionDescription = ({\n  action,\n  locateFieldDescription,\n  actionOutputExample,\n}: PlanningActionDescriptionBuildInput) => {\n  const actionDescription: ActionDescription = {\n    type: action.name,\n    description: action.description || 'No description provided',\n  };\n\n  if (action.paramSchema) {\n    const schema = action.paramSchema as {\n      _def?: { typeName?: string };\n      shape?: Record<string, unknown>;\n    };\n    const isZodObject = schema._def?.typeName === 'ZodObject';\n\n    if (isZodObject && schema.shape) {\n      const param: Record<string, ActionParamDescription> = {};\n\n      for (const [key, field] of Object.entries(schema.shape)) {\n        if (field && typeof field === 'object') {\n          const isOptional =\n            typeof (field as { isOptional?: () => boolean }).isOptional ===\n              'function' &&\n            (field as { isOptional: () => boolean }).isOptional();\n          // Deliberately do not resolve locators nested in ZodUnion here:\n          // findAllMidsceneLocatorField does not recognize them either, so\n          // describing them as locators would diverge from the execution path.\n          const typeDescription = isMidsceneLocatorField(field)\n            ? locateFieldDescription\n            : getZodTypeName(field);\n          const description = getZodDescription(field as z.ZodTypeAny);\n          const defaultValue = getZodDefaultValue(field);\n          const hasDefault = defaultValue !== undefined;\n\n          const paramDescription: ActionParamDescription = {\n            type: typeDescription,\n          };\n          if (isOptional) {\n            paramDescription.optional = true;\n          }\n          if (description) {\n            paramDescription.description = description;\n          }\n          if (hasDefault) {\n            paramDescription.default = defaultValue;\n          }\n\n          param[key] = paramDescription;\n        }\n      }\n\n      if (Object.keys(param).length > 0) {\n        actionDescription.param = param;\n      }\n    } else {\n      const typeName = getZodTypeName(schema);\n      const description = getZodDescription(schema as z.ZodTypeAny);\n      const paramDescription: ActionParamDescription = {\n        type: typeName,\n        instruction: 'Pass the value directly, not as an object.',\n      };\n      if (description) {\n        paramDescription.description = description;\n      }\n      actionDescription.param = paramDescription;\n    }\n  }\n\n  if (actionOutputExample) {\n    actionDescription.sample = actionOutputExample;\n  }\n\n  return actionDescription;\n};\n\nexport const buildLocateFieldDescription = (\n  promptSpec?: LocateResultPromptSpec,\n) => {\n  if (promptSpec) {\n    return `{ prompt: string, ${promptSpec.resultKey}: ${promptSpec.resultValueSchema} /* ${promptSpec.resultValueDescription} */ }`;\n  }\n  return '{ prompt: string /* description of the target element */ }';\n};\n\nconst serializeActionParam = (\n  param: Record<string, unknown>,\n  locateFields: string[],\n  locateResultKey: string,\n) => {\n  const locatorObjects = new WeakSet<object>();\n  for (const field of locateFields) {\n    const value = param[field];\n    if (value && typeof value === 'object') {\n      locatorObjects.add(value);\n    }\n  }\n\n  const originalJson = JSON.stringify(param);\n  // Ensure generated markers cannot collide with strings already present in the sample.\n  let markerPrefix = '__MIDSCENE_LOCATE_RESULT_EXAMPLE_';\n  while (originalJson.includes(markerPrefix)) {\n    markerPrefix = `_${markerPrefix}`;\n  }\n\n  const replacements = new Map<string, string>();\n  const serializedParam = JSON.stringify(\n    param,\n    function (this: object, key, value) {\n      if (locatorObjects.has(this) && key === locateResultKey) {\n        const marker = `${markerPrefix}${replacements.size}__`;\n        replacements.set(marker, formatLocateExampleValue(value));\n        return marker;\n      }\n      return value;\n    },\n    2,\n  );\n\n  return [...replacements].reduce(\n    (result, [marker, locateResultValue]) =>\n      result.replace(JSON.stringify(marker), locateResultValue),\n    serializedParam,\n  );\n};\n\nexport const buildPlanningActionOutput = ({\n  actionName,\n  param,\n  locateFields,\n  locateResultKey,\n}: PlanningActionOutputBuildInput) => `<action-type>${actionName}</action-type>\n<action-param-json>\n${\n  // Keep locate result arrays on one line when serializing action examples.\n  locateFields && locateResultKey\n    ? serializeActionParam(param, locateFields, locateResultKey)\n    : JSON.stringify(param, null, 2)\n}\n</action-param-json>`;\n\nexport const createMidscenePlanningActionOutputParser =\n  (jsonParser: JsonParser) => (content: string) => {\n    const actionType = extractXMLTag(content, 'action-type');\n    const actionParamStr = extractXMLTag(content, 'action-param-json');\n\n    if (!actionType || actionType.toLowerCase() === 'null') {\n      return null;\n    }\n\n    // Strip any trailing XML tags that leaked into the action type.\n    const type = actionType.split('<')[0].trim();\n    let param: any = undefined;\n\n    if (actionParamStr) {\n      try {\n        param = jsonParser(actionParamStr, {\n          source: 'planning-action-param',\n          preserveStringValueKeys:\n            type.toLowerCase() === 'input' ? ['value'] : undefined,\n        });\n        assertJsonObject(param);\n      } catch (error) {\n        throw new Error(`Failed to parse action-param-json: ${error}`);\n      }\n    }\n\n    return {\n      type,\n      ...(param !== undefined ? { param } : {}),\n    };\n  };\n\nexport const createDefaultMidscenePlanningProtocol: StandardPlanningProtocolFactory =\n  ({ jsonParser }) => {\n    const parseActionOutput =\n      createMidscenePlanningActionOutputParser(jsonParser);\n\n    return {\n      actionSpaceProtocol: {\n        title: 'Supporting actions list',\n        format: 'yaml',\n        includeActionOutputExample: true,\n        buildLocateFieldDescription,\n        buildActionDescription,\n      },\n      actionOutputProtocol: {\n        actionOutputTagNames: ['action-type', 'action-param-json'],\n        actionOutputRules: [\n          '- Use the <action-type> and <action-param-json> tags to output the action to be executed.',\n          \"- The value inside <action-type> MUST exactly match the 'type' field of one action in the Supporting actions list. 'complete' is NOT a valid action-type.\",\n          '- Parameter names are strict. Use EXACTLY the field names listed for the selected action. Do NOT invent alias fields. If the selected action provides a \"sample\" field, use the XML structure shown in that sample as the exact format for the action output.',\n        ].join('\\n'),\n        actionOutputPlaceholder: [\n          '<action-type>...</action-type>',\n          '<action-param-json>...</action-param-json>',\n        ].join('\\n'),\n        buildActionOutput: buildPlanningActionOutput,\n        parseActionOutput,\n        parseRawLocateParameter: (value) =>\n          value as ParsedPlanningLocateParameter,\n      },\n    };\n  };\n"],"names":["buildActionDescription","action","locateFieldDescription","actionOutputExample","actionDescription","schema","isZodObject","param","key","field","Object","isOptional","typeDescription","isMidsceneLocatorField","getZodTypeName","description","getZodDescription","defaultValue","getZodDefaultValue","hasDefault","undefined","paramDescription","typeName","buildLocateFieldDescription","promptSpec","serializeActionParam","locateFields","locateResultKey","locatorObjects","WeakSet","value","originalJson","JSON","markerPrefix","replacements","Map","serializedParam","marker","formatLocateExampleValue","result","locateResultValue","buildPlanningActionOutput","actionName","createMidscenePlanningActionOutputParser","jsonParser","content","actionType","extractXMLTag","actionParamStr","type","assertJsonObject","error","Error","createDefaultMidscenePlanningProtocol","parseActionOutput"],"mappings":";;;;;AAmCO,MAAMA,yBAAyB,CAAC,EACrCC,MAAM,EACNC,sBAAsB,EACtBC,mBAAmB,EACiB;IACpC,MAAMC,oBAAuC;QAC3C,MAAMH,OAAO,IAAI;QACjB,aAAaA,OAAO,WAAW,IAAI;IACrC;IAEA,IAAIA,OAAO,WAAW,EAAE;QACtB,MAAMI,SAASJ,OAAO,WAAW;QAIjC,MAAMK,cAAcD,OAAO,IAAI,EAAE,aAAa;QAE9C,IAAIC,eAAeD,OAAO,KAAK,EAAE;YAC/B,MAAME,QAAgD,CAAC;YAEvD,KAAK,MAAM,CAACC,KAAKC,MAAM,IAAIC,OAAO,OAAO,CAACL,OAAO,KAAK,EACpD,IAAII,SAAS,AAAiB,YAAjB,OAAOA,OAAoB;gBACtC,MAAME,aACJ,AACE,cADF,OAAQF,MAAyC,UAAU,IAE1DA,MAAwC,UAAU;gBAIrD,MAAMG,kBAAkBC,uBAAuBJ,SAC3CP,yBACAY,eAAeL;gBACnB,MAAMM,cAAcC,kBAAkBP;gBACtC,MAAMQ,eAAeC,mBAAmBT;gBACxC,MAAMU,aAAaF,AAAiBG,WAAjBH;gBAEnB,MAAMI,mBAA2C;oBAC/C,MAAMT;gBACR;gBACA,IAAID,YACFU,iBAAiB,QAAQ,GAAG;gBAE9B,IAAIN,aACFM,iBAAiB,WAAW,GAAGN;gBAEjC,IAAII,YACFE,iBAAiB,OAAO,GAAGJ;gBAG7BV,KAAK,CAACC,IAAI,GAAGa;YACf;YAGF,IAAIX,OAAO,IAAI,CAACH,OAAO,MAAM,GAAG,GAC9BH,kBAAkB,KAAK,GAAGG;QAE9B,OAAO;YACL,MAAMe,WAAWR,eAAeT;YAChC,MAAMU,cAAcC,kBAAkBX;YACtC,MAAMgB,mBAA2C;gBAC/C,MAAMC;gBACN,aAAa;YACf;YACA,IAAIP,aACFM,iBAAiB,WAAW,GAAGN;YAEjCX,kBAAkB,KAAK,GAAGiB;QAC5B;IACF;IAEA,IAAIlB,qBACFC,kBAAkB,MAAM,GAAGD;IAG7B,OAAOC;AACT;AAEO,MAAMmB,8BAA8B,CACzCC;IAEA,IAAIA,YACF,OAAO,CAAC,kBAAkB,EAAEA,WAAW,SAAS,CAAC,EAAE,EAAEA,WAAW,iBAAiB,CAAC,IAAI,EAAEA,WAAW,sBAAsB,CAAC,KAAK,CAAC;IAElI,OAAO;AACT;AAEA,MAAMC,uBAAuB,CAC3BlB,OACAmB,cACAC;IAEA,MAAMC,iBAAiB,IAAIC;IAC3B,KAAK,MAAMpB,SAASiB,aAAc;QAChC,MAAMI,QAAQvB,KAAK,CAACE,MAAM;QAC1B,IAAIqB,SAAS,AAAiB,YAAjB,OAAOA,OAClBF,eAAe,GAAG,CAACE;IAEvB;IAEA,MAAMC,eAAeC,KAAK,SAAS,CAACzB;IAEpC,IAAI0B,eAAe;IACnB,MAAOF,aAAa,QAAQ,CAACE,cAC3BA,eAAe,CAAC,CAAC,EAAEA,cAAc;IAGnC,MAAMC,eAAe,IAAIC;IACzB,MAAMC,kBAAkBJ,KAAK,SAAS,CACpCzB,OACA,SAAwBC,GAAG,EAAEsB,KAAK;QAChC,IAAIF,eAAe,GAAG,CAAC,IAAI,KAAKpB,QAAQmB,iBAAiB;YACvD,MAAMU,SAAS,GAAGJ,eAAeC,aAAa,IAAI,CAAC,EAAE,CAAC;YACtDA,aAAa,GAAG,CAACG,QAAQC,yBAAyBR;YAClD,OAAOO;QACT;QACA,OAAOP;IACT,GACA;IAGF,OAAO;WAAII;KAAa,CAAC,MAAM,CAC7B,CAACK,QAAQ,CAACF,QAAQG,kBAAkB,GAClCD,OAAO,OAAO,CAACP,KAAK,SAAS,CAACK,SAASG,oBACzCJ;AAEJ;AAEO,MAAMK,4BAA4B,CAAC,EACxCC,UAAU,EACVnC,KAAK,EACLmB,YAAY,EACZC,eAAe,EACgB,GAAK,CAAC,aAAa,EAAEe,WAAW;;AAEjE,EAEEhB,gBAAgBC,kBACZF,qBAAqBlB,OAAOmB,cAAcC,mBAC1CK,KAAK,SAAS,CAACzB,OAAO,MAAM,GACjC;oBACmB,CAAC;AAEd,MAAMoC,2CACX,CAACC,aAA2B,CAACC;QAC3B,MAAMC,aAAaC,cAAcF,SAAS;QAC1C,MAAMG,iBAAiBD,cAAcF,SAAS;QAE9C,IAAI,CAACC,cAAcA,AAA6B,WAA7BA,WAAW,WAAW,IACvC,OAAO;QAIT,MAAMG,OAAOH,WAAW,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI;QAC1C,IAAIvC;QAEJ,IAAIyC,gBACF,IAAI;YACFzC,QAAQqC,WAAWI,gBAAgB;gBACjC,QAAQ;gBACR,yBACEC,AAAuB,YAAvBA,KAAK,WAAW,KAAiB;oBAAC;iBAAQ,GAAG7B;YACjD;YACA8B,iBAAiB3C;QACnB,EAAE,OAAO4C,OAAO;YACd,MAAM,IAAIC,MAAM,CAAC,mCAAmC,EAAED,OAAO;QAC/D;QAGF,OAAO;YACLF;YACA,GAAI1C,AAAUa,WAAVb,QAAsB;gBAAEA;YAAM,IAAI,CAAC,CAAC;QAC1C;IACF;AAEK,MAAM8C,wCACX,CAAC,EAAET,UAAU,EAAE;IACb,MAAMU,oBACJX,yCAAyCC;IAE3C,OAAO;QACL,qBAAqB;YACnB,OAAO;YACP,QAAQ;YACR,4BAA4B;YAC5BrB;YACAvB;QACF;QACA,sBAAsB;YACpB,sBAAsB;gBAAC;gBAAe;aAAoB;YAC1D,mBAAmB;YAKnB,yBAAyB;YAIzB,mBAAmByC;YACnBa;YACA,yBAAyB,CAACxB,QACxBA;QACJ;IACF;AACF"}

package/dist/es/ai-model/model-adapter/image-preprocess.mjs

@@ -0,0 +1,27 @@
+import { paddingToMatchBlockByBase64 } from "@aiscene/shared/img";
+async function prepareModelImage(options) {
+    const { imageBase64, width, height, policy } = options;
+    let preparedImageBase64 = imageBase64;
+    let modelWidth = width;
+    let modelHeight = height;
+    if (void 0 !== policy.padBlockSize) {
+        const paddedResult = await paddingToMatchBlockByBase64(imageBase64, policy.padBlockSize);
+        preparedImageBase64 = paddedResult.imageBase64;
+        modelWidth = paddedResult.width;
+        modelHeight = paddedResult.height;
+    }
+    return {
+        imageBase64: preparedImageBase64,
+        preparedSize: {
+            width: modelWidth,
+            height: modelHeight
+        },
+        contentSize: {
+            width,
+            height
+        }
+    };
+}
+export { prepareModelImage };
+
+//# sourceMappingURL=image-preprocess.mjs.map

package/dist/es/ai-model/model-adapter/image-preprocess.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/model-adapter/image-preprocess.mjs","sources":["../../../../src/ai-model/model-adapter/image-preprocess.ts"],"sourcesContent":["import { paddingToMatchBlockByBase64 } from '@aiscene/shared/img';\n\nexport interface ImagePreprocessPolicy {\n  padBlockSize?: number;\n}\n\nexport interface PreparedModelImage {\n  imageBase64: string;\n  /**\n   * Size of the image sent to the model after preprocessing. This can be larger\n   * than the original screenshot when padding is applied to satisfy model block\n   * size requirements.\n   */\n  preparedSize: {\n    width: number;\n    height: number;\n  };\n  /**\n   * Size of the real screenshot content inside the prepared image. Pixel bboxes\n   * are parsed against `preparedSize`, then clipped to `contentSize` so padding\n   * added for the model is not treated as valid UI content.\n   */\n  contentSize: {\n    width: number;\n    height: number;\n  };\n}\n\nexport async function prepareModelImage(options: {\n  imageBase64: string;\n  width: number;\n  height: number;\n  policy: ImagePreprocessPolicy;\n}): Promise<PreparedModelImage> {\n  const { imageBase64, width, height, policy } = options;\n  let preparedImageBase64 = imageBase64;\n  let modelWidth = width;\n  let modelHeight = height;\n\n  if (policy.padBlockSize !== undefined) {\n    const paddedResult = await paddingToMatchBlockByBase64(\n      imageBase64,\n      policy.padBlockSize,\n    );\n    preparedImageBase64 = paddedResult.imageBase64;\n    modelWidth = paddedResult.width;\n    modelHeight = paddedResult.height;\n  }\n\n  return {\n    imageBase64: preparedImageBase64,\n    preparedSize: {\n      width: modelWidth,\n      height: modelHeight,\n    },\n    contentSize: {\n      width,\n      height,\n    },\n  };\n}\n"],"names":["prepareModelImage","options","imageBase64","width","height","policy","preparedImageBase64","modelWidth","modelHeight","undefined","paddedResult","paddingToMatchBlockByBase64"],"mappings":";AA4BO,eAAeA,kBAAkBC,OAKvC;IACC,MAAM,EAAEC,WAAW,EAAEC,KAAK,EAAEC,MAAM,EAAEC,MAAM,EAAE,GAAGJ;IAC/C,IAAIK,sBAAsBJ;IAC1B,IAAIK,aAAaJ;IACjB,IAAIK,cAAcJ;IAElB,IAAIC,AAAwBI,WAAxBJ,OAAO,YAAY,EAAgB;QACrC,MAAMK,eAAe,MAAMC,4BACzBT,aACAG,OAAO,YAAY;QAErBC,sBAAsBI,aAAa,WAAW;QAC9CH,aAAaG,aAAa,KAAK;QAC/BF,cAAcE,aAAa,MAAM;IACnC;IAEA,OAAO;QACL,aAAaJ;QACb,cAAc;YACZ,OAAOC;YACP,QAAQC;QACV;QACA,aAAa;YACXL;YACAC;QACF;IACF;AACF"}

package/dist/es/ai-model/model-adapter/index.mjs

@@ -0,0 +1,2 @@
+import { ResolvedModelAdapter } from "./resolve.mjs";
+export { ResolvedModelAdapter };

package/dist/es/ai-model/model-adapter/insight.mjs

@@ -0,0 +1,13 @@
+import { createDefaultInsightProtocol } from "./default-insight-protocol.mjs";
+function resolveInsightProtocol(definition, context) {
+    const protocolDefinition = definition ?? createDefaultInsightProtocol;
+    return 'function' == typeof protocolDefinition ? protocolDefinition(context) : protocolDefinition;
+}
+function resolveInsight(insight, context) {
+    return {
+        protocol: resolveInsightProtocol(insight?.protocol, context)
+    };
+}
+export { resolveInsight, resolveInsightProtocol };
+
+//# sourceMappingURL=insight.mjs.map

package/dist/es/ai-model/model-adapter/insight.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/model-adapter/insight.mjs","sources":["../../../../src/ai-model/model-adapter/insight.ts"],"sourcesContent":["import { createDefaultInsightProtocol } from './default-insight-protocol';\nimport type {\n  InsightAdapter,\n  InsightDefinition,\n  InsightProtocol,\n  InsightProtocolContext,\n  InsightProtocolDefinition,\n} from './insight-protocol';\n\nexport function resolveInsightProtocol(\n  definition: InsightProtocolDefinition | undefined,\n  context: InsightProtocolContext,\n): InsightProtocol {\n  const protocolDefinition = definition ?? createDefaultInsightProtocol;\n  return typeof protocolDefinition === 'function'\n    ? protocolDefinition(context)\n    : protocolDefinition;\n}\n\nexport function resolveInsight(\n  insight: InsightDefinition | undefined,\n  context: InsightProtocolContext,\n): InsightAdapter {\n  return {\n    protocol: resolveInsightProtocol(insight?.protocol, context),\n  };\n}\n"],"names":["resolveInsightProtocol","definition","context","protocolDefinition","createDefaultInsightProtocol","resolveInsight","insight"],"mappings":";AASO,SAASA,uBACdC,UAAiD,EACjDC,OAA+B;IAE/B,MAAMC,qBAAqBF,cAAcG;IACzC,OAAO,AAA8B,cAA9B,OAAOD,qBACVA,mBAAmBD,WACnBC;AACN;AAEO,SAASE,eACdC,OAAsC,EACtCJ,OAA+B;IAE/B,OAAO;QACL,UAAUF,uBAAuBM,SAAS,UAAUJ;IACtD;AACF"}

package/dist/es/ai-model/model-adapter/locate.mjs

@@ -0,0 +1,53 @@
+import { createLocateResultCodec } from "../shared/model-locate-result/factory.mjs";
+import { resolvePlanningTapLocator } from "../workflows/grounding/planning-action-locate.mjs";
+import { createDefaultElementProtocol, createDefaultSearchAreaProtocol } from "./default-locate-protocol.mjs";
+const defaultLocateResultFormatDefinition = {
+    coordinates: {
+        shape: 'bbox',
+        order: 'xy',
+        normalizedBy: 1000
+    }
+};
+const resolveLocateProtocolDefinition = (protocolDefinition, protocolContext)=>'function' == typeof protocolDefinition ? protocolDefinition(protocolContext) : protocolDefinition;
+function resolveLocate(locate, resolvedCustomPlanner, protocolContext, acceptBbox2dAlias = false) {
+    if (locate?.kind === 'custom') {
+        let locateFn = locate.locateFn;
+        if (!locateFn) {
+            const planningTapLocator = locate.planningTapLocator;
+            if (!planningTapLocator) throw new Error('Custom locate definition requires either locateFn or planningTapLocator');
+            if (!resolvedCustomPlanner) throw new Error('Custom planning tap locator requires a custom planning planner definition');
+            locateFn = resolvePlanningTapLocator(planningTapLocator, resolvedCustomPlanner);
+        }
+        return {
+            kind: 'custom',
+            locateFn
+        };
+    }
+    const elementProtocol = resolveLocateProtocolDefinition(locate?.element?.protocol ?? ((context)=>createDefaultElementProtocol(context, {
+            acceptBbox2dAlias
+        })), protocolContext);
+    const elementResultFormat = locate?.element?.resultFormat ?? defaultLocateResultFormatDefinition;
+    const elementResultCodec = createLocateResultCodec(elementResultFormat);
+    const searchAreaProtocol = locate?.searchArea === false ? void 0 : resolveLocateProtocolDefinition(locate?.searchArea?.protocol ?? ((context)=>createDefaultSearchAreaProtocol(context, {
+            acceptBbox2dAlias
+        })), protocolContext);
+    const searchAreaResultFormat = locate?.searchArea === false ? void 0 : locate?.searchArea?.resultFormat ?? elementResultFormat;
+    const searchAreaResultCodec = searchAreaResultFormat ? createLocateResultCodec(searchAreaResultFormat) : void 0;
+    return {
+        kind: 'standard',
+        userMessageContentOrder: locate?.userMessageContentOrder ?? 'image-first',
+        element: {
+            protocol: elementProtocol,
+            resultCodec: elementResultCodec
+        },
+        ...searchAreaProtocol && searchAreaResultCodec ? {
+            searchArea: {
+                protocol: searchAreaProtocol,
+                resultCodec: searchAreaResultCodec
+            }
+        } : {}
+    };
+}
+export { resolveLocate };
+
+//# sourceMappingURL=locate.mjs.map

package/dist/es/ai-model/model-adapter/locate.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/model-adapter/locate.mjs","sources":["../../../../src/ai-model/model-adapter/locate.ts"],"sourcesContent":["import { createLocateResultCodec } from '../shared/model-locate-result/factory';\nimport type { LocateResultFormatDefinition } from '../shared/model-locate-result/types';\nimport { resolvePlanningTapLocator } from '../workflows/grounding/planning-action-locate';\nimport type { ResolvedCustomPlanningDefinition } from './custom-planning-types';\nimport {\n  createDefaultElementProtocol,\n  createDefaultSearchAreaProtocol,\n} from './default-locate-protocol';\nimport type {\n  StandardLocateProtocolContext,\n  StandardLocateProtocolDefinition,\n} from './locate-protocol';\nimport type { LocateAdapter, ModelAdapterDefinition } from './types';\n\nconst defaultLocateResultFormatDefinition: LocateResultFormatDefinition = {\n  coordinates: { shape: 'bbox', order: 'xy', normalizedBy: 1000 },\n};\n\nconst resolveLocateProtocolDefinition = (\n  protocolDefinition: StandardLocateProtocolDefinition,\n  protocolContext: StandardLocateProtocolContext,\n) =>\n  typeof protocolDefinition === 'function'\n    ? protocolDefinition(protocolContext)\n    : protocolDefinition;\n\nexport function resolveLocate(\n  locate: ModelAdapterDefinition['locate'],\n  resolvedCustomPlanner: ResolvedCustomPlanningDefinition | undefined,\n  protocolContext: StandardLocateProtocolContext,\n  acceptBbox2dAlias = false,\n): LocateAdapter {\n  if (locate?.kind === 'custom') {\n    let locateFn = locate.locateFn;\n\n    if (!locateFn) {\n      const planningTapLocator = locate.planningTapLocator;\n\n      if (!planningTapLocator) {\n        throw new Error(\n          'Custom locate definition requires either locateFn or planningTapLocator',\n        );\n      }\n\n      if (!resolvedCustomPlanner) {\n        throw new Error(\n          'Custom planning tap locator requires a custom planning planner definition',\n        );\n      }\n      locateFn = resolvePlanningTapLocator(\n        planningTapLocator,\n        resolvedCustomPlanner,\n      );\n    }\n\n    return {\n      kind: 'custom',\n      locateFn,\n    };\n  }\n\n  const elementProtocol = resolveLocateProtocolDefinition(\n    locate?.element?.protocol ??\n      ((context) =>\n        createDefaultElementProtocol(context, { acceptBbox2dAlias })),\n    protocolContext,\n  );\n  const elementResultFormat =\n    locate?.element?.resultFormat ?? defaultLocateResultFormatDefinition;\n  const elementResultCodec = createLocateResultCodec(elementResultFormat);\n  const searchAreaProtocol =\n    locate?.searchArea === false\n      ? undefined\n      : resolveLocateProtocolDefinition(\n          locate?.searchArea?.protocol ??\n            ((context) =>\n              createDefaultSearchAreaProtocol(context, {\n                acceptBbox2dAlias,\n              })),\n          protocolContext,\n        );\n  const searchAreaResultFormat =\n    locate?.searchArea === false\n      ? undefined\n      : (locate?.searchArea?.resultFormat ?? elementResultFormat);\n  const searchAreaResultCodec = searchAreaResultFormat\n    ? createLocateResultCodec(searchAreaResultFormat)\n    : undefined;\n\n  return {\n    kind: 'standard',\n    userMessageContentOrder: locate?.userMessageContentOrder ?? 'image-first',\n    element: {\n      protocol: elementProtocol,\n      resultCodec: elementResultCodec,\n    },\n    ...(searchAreaProtocol && searchAreaResultCodec\n      ? {\n          searchArea: {\n            protocol: searchAreaProtocol,\n            resultCodec: searchAreaResultCodec,\n          },\n        }\n      : {}),\n  };\n}\n"],"names":["defaultLocateResultFormatDefinition","resolveLocateProtocolDefinition","protocolDefinition","protocolContext","resolveLocate","locate","resolvedCustomPlanner","acceptBbox2dAlias","locateFn","planningTapLocator","Error","resolvePlanningTapLocator","elementProtocol","context","createDefaultElementProtocol","elementResultFormat","elementResultCodec","createLocateResultCodec","searchAreaProtocol","undefined","createDefaultSearchAreaProtocol","searchAreaResultFormat","searchAreaResultCodec"],"mappings":";;;AAcA,MAAMA,sCAAoE;IACxE,aAAa;QAAE,OAAO;QAAQ,OAAO;QAAM,cAAc;IAAK;AAChE;AAEA,MAAMC,kCAAkC,CACtCC,oBACAC,kBAEA,AAA8B,cAA9B,OAAOD,qBACHA,mBAAmBC,mBACnBD;AAEC,SAASE,cACdC,MAAwC,EACxCC,qBAAmE,EACnEH,eAA8C,EAC9CI,oBAAoB,KAAK;IAEzB,IAAIF,QAAQ,SAAS,UAAU;QAC7B,IAAIG,WAAWH,OAAO,QAAQ;QAE9B,IAAI,CAACG,UAAU;YACb,MAAMC,qBAAqBJ,OAAO,kBAAkB;YAEpD,IAAI,CAACI,oBACH,MAAM,IAAIC,MACR;YAIJ,IAAI,CAACJ,uBACH,MAAM,IAAII,MACR;YAGJF,WAAWG,0BACTF,oBACAH;QAEJ;QAEA,OAAO;YACL,MAAM;YACNE;QACF;IACF;IAEA,MAAMI,kBAAkBX,gCACtBI,QAAQ,SAAS,YACb,EAAAQ,UACAC,6BAA6BD,SAAS;YAAEN;QAAkB,EAAC,GAC/DJ;IAEF,MAAMY,sBACJV,QAAQ,SAAS,gBAAgBL;IACnC,MAAMgB,qBAAqBC,wBAAwBF;IACnD,MAAMG,qBACJb,QAAQ,eAAe,QACnBc,SACAlB,gCACEI,QAAQ,YAAY,YAChB,EAAAQ,UACAO,gCAAgCP,SAAS;YACvCN;QACF,EAAC,GACLJ;IAER,MAAMkB,yBACJhB,QAAQ,eAAe,QACnBc,SACCd,QAAQ,YAAY,gBAAgBU;IAC3C,MAAMO,wBAAwBD,yBAC1BJ,wBAAwBI,0BACxBF;IAEJ,OAAO;QACL,MAAM;QACN,yBAAyBd,QAAQ,2BAA2B;QAC5D,SAAS;YACP,UAAUO;YACV,aAAaI;QACf;QACA,GAAIE,sBAAsBI,wBACtB;YACE,YAAY;gBACV,UAAUJ;gBACV,aAAaI;YACf;QACF,IACA,CAAC,CAAC;IACR;AACF"}

package/dist/es/ai-model/model-adapter/planning.mjs

@@ -0,0 +1,49 @@
+import { createLocateResultCodec, resolveLocateResultCoordinates } from "../shared/model-locate-result/index.mjs";
+import { runCustomPlanning } from "../workflows/planning/custom-planning.mjs";
+import { createDefaultMidscenePlanningProtocol } from "./default-planning-protocol.mjs";
+const defaultReplanningCycleLimit = 20;
+function resolveCustomPlanningDefinition(config) {
+    const { coordinates, ...rest } = config;
+    const coordinateSystem = resolveLocateResultCoordinates(coordinates);
+    const coordinateNormalizer = createLocateResultCodec({
+        coordinates
+    });
+    return {
+        ...rest,
+        coordinateSystem,
+        coordinateNormalizer
+    };
+}
+function resolvePlanning(planning, resolvedCustomPlanner, protocolContext, defaultLocateResultCodec) {
+    if (planning?.kind === 'custom') {
+        if ('function' == typeof planning.planFn) return {
+            kind: 'custom',
+            cacheEnabled: planning.cacheEnabled ?? false,
+            defaultReplanningCycleLimit: planning.defaultReplanningCycleLimit ?? defaultReplanningCycleLimit,
+            supportsActionDeepLocate: planning.supportsActionDeepLocate ?? false,
+            planFn: planning.planFn
+        };
+        if (!resolvedCustomPlanner) throw new Error('Custom planning planner definition is not resolved');
+        return {
+            kind: 'custom',
+            cacheEnabled: planning.cacheEnabled ?? false,
+            defaultReplanningCycleLimit: planning.defaultReplanningCycleLimit ?? defaultReplanningCycleLimit,
+            supportsActionDeepLocate: planning.supportsActionDeepLocate ?? false,
+            coordinateSystem: resolvedCustomPlanner.coordinateSystem,
+            planFn: (userInstruction, options)=>runCustomPlanning(userInstruction, options, resolvedCustomPlanner)
+        };
+    }
+    const protocolDefinition = planning?.protocol ?? createDefaultMidscenePlanningProtocol;
+    const locateResultCodec = planning?.locateResultFormat === false ? void 0 : planning?.locateResultFormat ? createLocateResultCodec(planning.locateResultFormat) : defaultLocateResultCodec;
+    return {
+        kind: 'standard',
+        cacheEnabled: planning?.cacheEnabled ?? true,
+        defaultReplanningCycleLimit: planning?.defaultReplanningCycleLimit ?? defaultReplanningCycleLimit,
+        supportsActionDeepLocate: planning?.supportsActionDeepLocate ?? true,
+        protocol: 'function' == typeof protocolDefinition ? protocolDefinition(protocolContext) : protocolDefinition,
+        locateResultCodec
+    };
+}
+export { resolveCustomPlanningDefinition, resolvePlanning };
+
+//# sourceMappingURL=planning.mjs.map

package/dist/es/ai-model/model-adapter/planning.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/model-adapter/planning.mjs","sources":["../../../../src/ai-model/model-adapter/planning.ts"],"sourcesContent":["import {\n  createLocateResultCodec,\n  resolveLocateResultCoordinates,\n} from '../shared/model-locate-result';\nimport type { LocateResultCodec } from '../shared/model-locate-result';\nimport { runCustomPlanning } from '../workflows/planning/custom-planning';\nimport type {\n  CustomPlanningDefinition,\n  ResolvedCustomPlanningDefinition,\n} from './custom-planning-types';\nimport { createDefaultMidscenePlanningProtocol } from './default-planning-protocol';\nimport type { StandardPlanningProtocolContext } from './planning-protocol';\nimport type { ModelAdapterDefinition, PlanningAdapter } from './types';\n\nconst defaultReplanningCycleLimit = 20;\n\nexport function resolveCustomPlanningDefinition<TParsed>(\n  config: CustomPlanningDefinition<TParsed>,\n): ResolvedCustomPlanningDefinition<TParsed> {\n  const { coordinates, ...rest } = config;\n  const coordinateSystem = resolveLocateResultCoordinates(coordinates);\n  const coordinateNormalizer = createLocateResultCodec({ coordinates });\n  return {\n    ...rest,\n    coordinateSystem,\n    coordinateNormalizer,\n  };\n}\n\nexport function resolvePlanning(\n  planning: ModelAdapterDefinition['planning'],\n  resolvedCustomPlanner: ResolvedCustomPlanningDefinition | undefined,\n  protocolContext: StandardPlanningProtocolContext,\n  defaultLocateResultCodec?: LocateResultCodec,\n): PlanningAdapter {\n  if (planning?.kind === 'custom') {\n    if (typeof planning.planFn === 'function') {\n      return {\n        kind: 'custom',\n        cacheEnabled: planning.cacheEnabled ?? false,\n        defaultReplanningCycleLimit:\n          planning.defaultReplanningCycleLimit ?? defaultReplanningCycleLimit,\n        supportsActionDeepLocate: planning.supportsActionDeepLocate ?? false,\n        planFn: planning.planFn,\n      };\n    }\n\n    if (!resolvedCustomPlanner) {\n      throw new Error('Custom planning planner definition is not resolved');\n    }\n\n    return {\n      kind: 'custom',\n      cacheEnabled: planning.cacheEnabled ?? false,\n      defaultReplanningCycleLimit:\n        planning.defaultReplanningCycleLimit ?? defaultReplanningCycleLimit,\n      supportsActionDeepLocate: planning.supportsActionDeepLocate ?? false,\n      coordinateSystem: resolvedCustomPlanner.coordinateSystem,\n      planFn: (userInstruction, options) =>\n        runCustomPlanning(userInstruction, options, resolvedCustomPlanner),\n    };\n  }\n\n  const protocolDefinition =\n    planning?.protocol ?? createDefaultMidscenePlanningProtocol;\n  const locateResultCodec =\n    planning?.locateResultFormat === false\n      ? undefined\n      : planning?.locateResultFormat\n        ? createLocateResultCodec(planning.locateResultFormat)\n        : defaultLocateResultCodec;\n\n  return {\n    kind: 'standard',\n    cacheEnabled: planning?.cacheEnabled ?? true,\n    defaultReplanningCycleLimit:\n      planning?.defaultReplanningCycleLimit ?? defaultReplanningCycleLimit,\n    supportsActionDeepLocate: planning?.supportsActionDeepLocate ?? true,\n    protocol:\n      typeof protocolDefinition === 'function'\n        ? protocolDefinition(protocolContext)\n        : protocolDefinition,\n    locateResultCodec,\n  };\n}\n"],"names":["defaultReplanningCycleLimit","resolveCustomPlanningDefinition","config","coordinates","rest","coordinateSystem","resolveLocateResultCoordinates","coordinateNormalizer","createLocateResultCodec","resolvePlanning","planning","resolvedCustomPlanner","protocolContext","defaultLocateResultCodec","Error","userInstruction","options","runCustomPlanning","protocolDefinition","createDefaultMidscenePlanningProtocol","locateResultCodec","undefined"],"mappings":";;;AAcA,MAAMA,8BAA8B;AAE7B,SAASC,gCACdC,MAAyC;IAEzC,MAAM,EAAEC,WAAW,EAAE,GAAGC,MAAM,GAAGF;IACjC,MAAMG,mBAAmBC,+BAA+BH;IACxD,MAAMI,uBAAuBC,wBAAwB;QAAEL;IAAY;IACnE,OAAO;QACL,GAAGC,IAAI;QACPC;QACAE;IACF;AACF;AAEO,SAASE,gBACdC,QAA4C,EAC5CC,qBAAmE,EACnEC,eAAgD,EAChDC,wBAA4C;IAE5C,IAAIH,UAAU,SAAS,UAAU;QAC/B,IAAI,AAA2B,cAA3B,OAAOA,SAAS,MAAM,EACxB,OAAO;YACL,MAAM;YACN,cAAcA,SAAS,YAAY,IAAI;YACvC,6BACEA,SAAS,2BAA2B,IAAIV;YAC1C,0BAA0BU,SAAS,wBAAwB,IAAI;YAC/D,QAAQA,SAAS,MAAM;QACzB;QAGF,IAAI,CAACC,uBACH,MAAM,IAAIG,MAAM;QAGlB,OAAO;YACL,MAAM;YACN,cAAcJ,SAAS,YAAY,IAAI;YACvC,6BACEA,SAAS,2BAA2B,IAAIV;YAC1C,0BAA0BU,SAAS,wBAAwB,IAAI;YAC/D,kBAAkBC,sBAAsB,gBAAgB;YACxD,QAAQ,CAACI,iBAAiBC,UACxBC,kBAAkBF,iBAAiBC,SAASL;QAChD;IACF;IAEA,MAAMO,qBACJR,UAAU,YAAYS;IACxB,MAAMC,oBACJV,UAAU,uBAAuB,QAC7BW,SACAX,UAAU,qBACRF,wBAAwBE,SAAS,kBAAkB,IACnDG;IAER,OAAO;QACL,MAAM;QACN,cAAcH,UAAU,gBAAgB;QACxC,6BACEA,UAAU,+BAA+BV;QAC3C,0BAA0BU,UAAU,4BAA4B;QAChE,UACE,AAA8B,cAA9B,OAAOQ,qBACHA,mBAAmBN,mBACnBM;QACNE;IACF;AACF"}

package/dist/es/ai-model/model-adapter/resolve.mjs

@@ -0,0 +1,54 @@
+import { parseModelResponseJson } from "../shared/json.mjs";
+import { resolveChatCompletion } from "./chat-completion.mjs";
+import { resolveInsight } from "./insight.mjs";
+import { resolveLocate } from "./locate.mjs";
+import { resolveCustomPlanningDefinition, resolvePlanning } from "./planning.mjs";
+function _define_property(obj, key, value) {
+    if (key in obj) Object.defineProperty(obj, key, {
+        value: value,
+        enumerable: true,
+        configurable: true,
+        writable: true
+    });
+    else obj[key] = value;
+    return obj;
+}
+function resolveJsonParser(jsonParser) {
+    if (!jsonParser || 'lenient-json' === jsonParser) return parseModelResponseJson;
+    if ('function' == typeof jsonParser) return jsonParser;
+    throw new Error(`Unknown json parser preset: ${jsonParser}`);
+}
+function resolveImagePreprocess(imagePreprocess) {
+    return {
+        padBlockSize: imagePreprocess?.padBlockSize
+    };
+}
+class ResolvedModelAdapter {
+    constructor(config, modelFamily){
+        _define_property(this, "jsonParser", void 0);
+        _define_property(this, "chatCompletion", void 0);
+        _define_property(this, "acceptBbox2dAlias", void 0);
+        _define_property(this, "imagePreprocess", void 0);
+        _define_property(this, "insight", void 0);
+        _define_property(this, "planning", void 0);
+        _define_property(this, "locate", void 0);
+        this.jsonParser = resolveJsonParser(config.jsonParser);
+        this.chatCompletion = resolveChatCompletion(config.chatCompletion);
+        this.acceptBbox2dAlias = config.acceptBbox2dAlias ?? false;
+        this.imagePreprocess = resolveImagePreprocess(config.imagePreprocess);
+        this.insight = resolveInsight(config.insight, {
+            jsonParser: this.jsonParser
+        });
+        const customPlanner = config.planning?.kind === 'custom' ? config.planning.planner : void 0;
+        const resolvedCustomPlanner = customPlanner ? resolveCustomPlanningDefinition(customPlanner) : void 0;
+        this.locate = resolveLocate(config.locate, resolvedCustomPlanner, {
+            jsonParser: this.jsonParser
+        }, this.acceptBbox2dAlias);
+        this.planning = resolvePlanning(config.planning, resolvedCustomPlanner, {
+            jsonParser: this.jsonParser
+        }, 'standard' === this.locate.kind ? this.locate.element.resultCodec : void 0);
+    }
+}
+export { ResolvedModelAdapter };
+
+//# sourceMappingURL=resolve.mjs.map

package/dist/es/ai-model/model-adapter/resolve.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/model-adapter/resolve.mjs","sources":["../../../../src/ai-model/model-adapter/resolve.ts"],"sourcesContent":["import { parseModelResponseJson } from '../shared/json';\nimport { resolveChatCompletion } from './chat-completion';\nimport { resolveInsight } from './insight';\nimport type { InsightAdapter } from './insight-protocol';\nimport { resolveLocate } from './locate';\nimport { resolveCustomPlanningDefinition, resolvePlanning } from './planning';\nimport type {\n  ChatCompletionAdapter,\n  ImagePreprocessPolicy,\n  JsonParser,\n  LocateAdapter,\n  ModelAdapter,\n  ModelAdapterDefinition,\n  PlanningAdapter,\n} from './types';\n\nfunction resolveJsonParser(\n  jsonParser: ModelAdapterDefinition['jsonParser'],\n): JsonParser {\n  if (!jsonParser || jsonParser === 'lenient-json') {\n    return parseModelResponseJson;\n  }\n\n  if (typeof jsonParser === 'function') {\n    return jsonParser;\n  }\n\n  throw new Error(`Unknown json parser preset: ${jsonParser}`);\n}\n\nfunction resolveImagePreprocess(\n  imagePreprocess: ModelAdapterDefinition['imagePreprocess'],\n): ImagePreprocessPolicy {\n  return {\n    padBlockSize: imagePreprocess?.padBlockSize,\n  };\n}\n\nexport class ResolvedModelAdapter implements ModelAdapter {\n  readonly jsonParser: JsonParser;\n  readonly chatCompletion: ChatCompletionAdapter;\n  readonly acceptBbox2dAlias: boolean;\n  readonly imagePreprocess: ImagePreprocessPolicy;\n  readonly insight: InsightAdapter;\n  readonly planning: PlanningAdapter;\n  readonly locate: LocateAdapter;\n\n  constructor(config: ModelAdapterDefinition, modelFamily: string) {\n    this.jsonParser = resolveJsonParser(config.jsonParser);\n    this.chatCompletion = resolveChatCompletion(config.chatCompletion);\n    this.acceptBbox2dAlias = config.acceptBbox2dAlias ?? false;\n    this.imagePreprocess = resolveImagePreprocess(config.imagePreprocess);\n    this.insight = resolveInsight(config.insight, {\n      jsonParser: this.jsonParser,\n    });\n    const customPlanner =\n      config.planning?.kind === 'custom' ? config.planning.planner : undefined;\n    const resolvedCustomPlanner = customPlanner\n      ? resolveCustomPlanningDefinition(customPlanner)\n      : undefined;\n    this.locate = resolveLocate(\n      config.locate,\n      resolvedCustomPlanner,\n      {\n        jsonParser: this.jsonParser,\n      },\n      this.acceptBbox2dAlias,\n    );\n    this.planning = resolvePlanning(\n      config.planning,\n      resolvedCustomPlanner,\n      { jsonParser: this.jsonParser },\n      this.locate.kind === 'standard'\n        ? this.locate.element.resultCodec\n        : undefined,\n    );\n  }\n}\n"],"names":["resolveJsonParser","jsonParser","parseModelResponseJson","Error","resolveImagePreprocess","imagePreprocess","ResolvedModelAdapter","config","modelFamily","resolveChatCompletion","resolveInsight","customPlanner","undefined","resolvedCustomPlanner","resolveCustomPlanningDefinition","resolveLocate","resolvePlanning"],"mappings":";;;;;;;;;;;;;;;AAgBA,SAASA,kBACPC,UAAgD;IAEhD,IAAI,CAACA,cAAcA,AAAe,mBAAfA,YACjB,OAAOC;IAGT,IAAI,AAAsB,cAAtB,OAAOD,YACT,OAAOA;IAGT,MAAM,IAAIE,MAAM,CAAC,4BAA4B,EAAEF,YAAY;AAC7D;AAEA,SAASG,uBACPC,eAA0D;IAE1D,OAAO;QACL,cAAcA,iBAAiB;IACjC;AACF;AAEO,MAAMC;IASX,YAAYC,MAA8B,EAAEC,WAAmB,CAAE;QARjE,uBAAS,cAAT;QACA,uBAAS,kBAAT;QACA,uBAAS,qBAAT;QACA,uBAAS,mBAAT;QACA,uBAAS,WAAT;QACA,uBAAS,YAAT;QACA,uBAAS,UAAT;QAGE,IAAI,CAAC,UAAU,GAAGR,kBAAkBO,OAAO,UAAU;QACrD,IAAI,CAAC,cAAc,GAAGE,sBAAsBF,OAAO,cAAc;QACjE,IAAI,CAAC,iBAAiB,GAAGA,OAAO,iBAAiB,IAAI;QACrD,IAAI,CAAC,eAAe,GAAGH,uBAAuBG,OAAO,eAAe;QACpE,IAAI,CAAC,OAAO,GAAGG,eAAeH,OAAO,OAAO,EAAE;YAC5C,YAAY,IAAI,CAAC,UAAU;QAC7B;QACA,MAAMI,gBACJJ,OAAO,QAAQ,EAAE,SAAS,WAAWA,OAAO,QAAQ,CAAC,OAAO,GAAGK;QACjE,MAAMC,wBAAwBF,gBAC1BG,gCAAgCH,iBAChCC;QACJ,IAAI,CAAC,MAAM,GAAGG,cACZR,OAAO,MAAM,EACbM,uBACA;YACE,YAAY,IAAI,CAAC,UAAU;QAC7B,GACA,IAAI,CAAC,iBAAiB;QAExB,IAAI,CAAC,QAAQ,GAAGG,gBACdT,OAAO,QAAQ,EACfM,uBACA;YAAE,YAAY,IAAI,CAAC,UAAU;QAAC,GAC9B,AAAqB,eAArB,IAAI,CAAC,MAAM,CAAC,IAAI,GACZ,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,WAAW,GAC/BD;IAER;AACF"}

package/dist/es/ai-model/models/auto-glm/actions.mjs

@@ -0,0 +1,205 @@
+import { getDebug } from "@aiscene/shared/logger";
+const debug = getDebug('auto-glm-actions');
+const BACK_BUTTON_NAMES = [
+    'AndroidBackButton',
+    'HarmonyBackButton'
+];
+const HOME_BUTTON_NAMES = [
+    'AndroidHomeButton',
+    'HarmonyHomeButton'
+];
+function findActionName(actionSpace, knownNames, defaultName) {
+    if (!actionSpace) return defaultName;
+    const match = actionSpace.find((a)=>knownNames.includes(a.name));
+    return match ? match.name : defaultName;
+}
+function transformAutoGLMAction(action, { actionSpace, coordinateDistanceToPixels }) {
+    try {
+        switch(action._metadata){
+            case 'finish':
+                {
+                    const finishAction = action;
+                    debug('Transform finish action:', finishAction);
+                    return [
+                        {
+                            type: 'Finished',
+                            param: {},
+                            thought: finishAction.message
+                        }
+                    ];
+                }
+            case 'do':
+                {
+                    const doAction = action;
+                    switch(doAction.action){
+                        case 'Tap':
+                            {
+                                const tapAction = doAction;
+                                debug('Transform Tap action:', tapAction);
+                                return [
+                                    {
+                                        type: 'Tap',
+                                        param: {
+                                            locate: {
+                                                point: tapAction.element,
+                                                prompt: ''
+                                            }
+                                        }
+                                    }
+                                ];
+                            }
+                        case 'Double Tap':
+                            {
+                                const doubleTapAction = doAction;
+                                debug('Transform Double Tap action:', doubleTapAction);
+                                return [
+                                    {
+                                        type: 'DoubleClick',
+                                        param: {
+                                            locate: {
+                                                point: doubleTapAction.element,
+                                                prompt: ''
+                                            }
+                                        }
+                                    }
+                                ];
+                            }
+                        case 'Type':
+                            {
+                                const typeAction = doAction;
+                                debug('Transform Type action:', typeAction);
+                                return [
+                                    {
+                                        type: 'Input',
+                                        param: {
+                                            value: typeAction.text
+                                        }
+                                    }
+                                ];
+                            }
+                        case 'Swipe':
+                            {
+                                const swipeAction = doAction;
+                                debug('Transform Swipe action:', swipeAction);
+                                const deltaX = swipeAction.end[0] - swipeAction.start[0];
+                                const deltaY = swipeAction.end[1] - swipeAction.start[1];
+                                let direction;
+                                let distance;
+                                const absDeltaX = Math.abs(deltaX);
+                                const absDeltaY = Math.abs(deltaY);
+                                if (absDeltaY > absDeltaX) {
+                                    distance = coordinateDistanceToPixels(deltaY, 'y');
+                                    direction = deltaY > 0 ? 'up' : 'down';
+                                } else {
+                                    distance = coordinateDistanceToPixels(deltaX, 'x');
+                                    direction = deltaX > 0 ? 'left' : 'right';
+                                }
+                                debug(`Calculate swipe direction: ${direction}, distance: ${distance}`);
+                                return [
+                                    {
+                                        type: 'Scroll',
+                                        param: {
+                                            locate: {
+                                                point: swipeAction.start,
+                                                prompt: ''
+                                            },
+                                            distance,
+                                            direction
+                                        },
+                                        thought: swipeAction.think || ''
+                                    }
+                                ];
+                            }
+                        case 'Long Press':
+                            {
+                                const longPressAction = doAction;
+                                debug('Transform Long Press action:', longPressAction);
+                                return [
+                                    {
+                                        type: 'LongPress',
+                                        param: {
+                                            locate: {
+                                                point: longPressAction.element,
+                                                prompt: ''
+                                            }
+                                        },
+                                        thought: longPressAction.think || ''
+                                    }
+                                ];
+                            }
+                        case 'Back':
+                            {
+                                const backAction = doAction;
+                                debug('Transform Back action:', backAction);
+                                return [
+                                    {
+                                        type: findActionName(actionSpace, BACK_BUTTON_NAMES, 'AndroidBackButton'),
+                                        param: {},
+                                        thought: backAction.think || ''
+                                    }
+                                ];
+                            }
+                        case 'Home':
+                            {
+                                const homeAction = doAction;
+                                debug('Transform Home action:', homeAction);
+                                return [
+                                    {
+                                        type: findActionName(actionSpace, HOME_BUTTON_NAMES, 'AndroidHomeButton'),
+                                        param: {},
+                                        thought: homeAction.think || ''
+                                    }
+                                ];
+                            }
+                        case 'Wait':
+                            {
+                                const waitAction = doAction;
+                                debug('Transform Wait action:', waitAction);
+                                return [
+                                    {
+                                        type: 'Sleep',
+                                        param: {
+                                            timeMs: waitAction.durationMs
+                                        },
+                                        thought: waitAction.think || ''
+                                    }
+                                ];
+                            }
+                        case 'Launch':
+                            {
+                                const launchAction = doAction;
+                                debug('Transform Launch action:', launchAction);
+                                return [
+                                    {
+                                        type: 'Launch',
+                                        param: {
+                                            uri: launchAction.app
+                                        },
+                                        thought: launchAction.think || ''
+                                    }
+                                ];
+                            }
+                        case 'Interact':
+                            throw new Error('Action "Interact" from auto-glm is not supported in the current implementation.');
+                        case 'Call_API':
+                            throw new Error('Action "Call_API" from auto-glm is not supported in the current implementation.');
+                        case 'Take_over':
+                            throw new Error('Action "Take_over" from auto-glm is not supported in the current implementation.');
+                        case 'Note':
+                            throw new Error('Action "Note" from auto-glm is not supported in the current implementation.');
+                        default:
+                            throw new Error(`Unknown do() action type: ${doAction.action}`);
+                    }
+                }
+            default:
+                throw new Error(`Unknown action metadata: ${action._metadata}`);
+        }
+    } catch (error) {
+        const errorMessage = error instanceof Error ? error.message : String(error);
+        debug('Transform error:', errorMessage);
+        throw new Error(`Failed to transform action: ${errorMessage}`);
+    }
+}
+export { transformAutoGLMAction };
+
+//# sourceMappingURL=actions.mjs.map