@aiscene/core 9.0.6 → 9.0.8

package/dist/es/ai-model/shared/json.mjs

@@ -0,0 +1,63 @@
+import { jsonrepair } from "jsonrepair";
+function extractJSONFromCodeBlock(response) {
+    try {
+        const jsonMatch = response.match(/^\s*(\{[\s\S]*\})\s*$/);
+        if (jsonMatch) return jsonMatch[1];
+        const codeBlockMatch = response.match(/```(?:json)?\s*(\{[\s\S]*?\})\s*```/);
+        if (codeBlockMatch) return codeBlockMatch[1];
+        const jsonLikeMatch = response.match(/\{[\s\S]*\}/);
+        if (jsonLikeMatch) return jsonLikeMatch[0];
+    } catch  {}
+    return response;
+}
+function trimParsedJsonStrings(obj, context = {}) {
+    if (null == obj) return obj;
+    if (Array.isArray(obj)) return obj.map((item)=>trimParsedJsonStrings(item, context));
+    if ('object' == typeof obj) {
+        const normalized = {};
+        for (const [key, value] of Object.entries(obj)){
+            const trimmedKey = key.trim();
+            const preserveStringValue = context.preserveStringValueKeys?.includes(trimmedKey) ?? false;
+            const normalizedValue = 'string' == typeof value ? preserveStringValue ? value : value.trim() : trimParsedJsonStrings(value, context);
+            normalized[trimmedKey] = normalizedValue;
+        }
+        return normalized;
+    }
+    if ('string' == typeof obj) return obj.trim();
+    return obj;
+}
+function repairKnownJsonIssues(jsonBlock, _rawResponse) {
+    return jsonBlock;
+}
+function assertJsonObject(parsed) {
+    if (!parsed || 'object' != typeof parsed || Array.isArray(parsed)) {
+        const parsedType = Array.isArray(parsed) ? 'array' : null === parsed ? 'null' : typeof parsed;
+        throw new Error(`LLM response is valid JSON but does not match the expected schema. Expected to be a JSON object, got ${parsedType}: ${JSON.stringify(parsed)}.`);
+    }
+}
+function parseJsonWithRepair(jsonStr) {
+    try {
+        return JSON.parse(jsonStr);
+    } catch  {
+        return JSON.parse(jsonrepair(jsonStr));
+    }
+}
+function parseModelResponseJson(raw, context) {
+    const cleanJsonString = extractJSONFromCodeBlock(raw);
+    let parsedObj;
+    try {
+        parsedObj = parseJsonWithRepair(cleanJsonString);
+    } catch (e1) {
+        const code = repairKnownJsonIssues(cleanJsonString, raw);
+        if (code === cleanJsonString) throw new Error(`failed to parse LLM response into JSON. Error - ${String(e1)}. Response - \n ${raw}`);
+        try {
+            parsedObj = parseJsonWithRepair(code);
+        } catch (e2) {
+            throw new Error(`failed to parse LLM response into JSON. First error - ${String(e1)}. Second error - ${String(e2)}. Response - \n ${raw}`);
+        }
+    }
+    return trimParsedJsonStrings(parsedObj, context);
+}
+export { assertJsonObject, extractJSONFromCodeBlock, parseModelResponseJson };
+
+//# sourceMappingURL=json.mjs.map

package/dist/es/ai-model/shared/json.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/shared/json.mjs","sources":["../../../../src/ai-model/shared/json.ts"],"sourcesContent":["import { jsonrepair } from 'jsonrepair';\n\n/**\n * Extract the JSON portion from a model response.\n *\n * This mainly handles fenced JSON like ```json { ... } ``` by removing the\n * fence. If that does not produce a JSON object, it falls back to a greedy regex\n * that extracts from the first \"{\" to the last \"}\". If that still fails, the\n * original response is returned.\n *\n * Expected model responses are JSON objects, possibly wrapped in markdown\n * fences. Natural language mixed with JSON, or arrays like\n * [{\"type\":\"Tap\"}, {\"type\":\"Hover\"}], are outside the supported contract and\n * are not reliably recoverable.\n *\n * This extractor is also used by extraction responses that can be any\n * JSON value. In those cases, arrays/strings/numbers have no object braces and\n * pass through unchanged.\n */\nexport function extractJSONFromCodeBlock(response: string) {\n  try {\n    // First, try to match a JSON object directly in the response\n    const jsonMatch = response.match(/^\\s*(\\{[\\s\\S]*\\})\\s*$/);\n    if (jsonMatch) {\n      return jsonMatch[1];\n    }\n\n    // If no direct JSON object is found, try to extract JSON from a code block\n    const codeBlockMatch = response.match(\n      /```(?:json)?\\s*(\\{[\\s\\S]*?\\})\\s*```/,\n    );\n    if (codeBlockMatch) {\n      return codeBlockMatch[1];\n    }\n\n    // If no code block is found, try to find a JSON-like structure in the text\n    const jsonLikeMatch = response.match(/\\{[\\s\\S]*\\}/);\n    if (jsonLikeMatch) {\n      return jsonLikeMatch[0];\n    }\n  } catch {}\n  // If no JSON-like structure is found, return the original response\n  return response;\n}\n\nexport type JsonParserSource =\n  | 'generic-object'\n  | 'planning-action-param'\n  | 'locate'\n  | 'section-locator';\n\nexport interface JsonParserContext {\n  source: JsonParserSource;\n  preserveStringValueKeys?: string[];\n}\n\nexport type JsonParser = (raw: string, context?: JsonParserContext) => unknown;\n\n/**\n * Trim whitespace in parsed JSON by normalizing:\n * 1. All object keys (e.g., \" prompt \" -> \"prompt\")\n * 2. All string values (e.g., \" Tap \" -> \"Tap\")\n * This handles LLM output that may include leading/trailing spaces.\n */\nfunction trimParsedJsonStrings(\n  obj: any,\n  context: Pick<JsonParserContext, 'preserveStringValueKeys'> = {},\n): any {\n  // Handle null and undefined\n  if (obj === null || obj === undefined) {\n    return obj;\n  }\n\n  // Handle arrays - recursively normalize each element\n  if (Array.isArray(obj)) {\n    return obj.map((item) => trimParsedJsonStrings(item, context));\n  }\n\n  // Handle objects\n  if (typeof obj === 'object') {\n    const normalized: any = {};\n\n    for (const [key, value] of Object.entries(obj)) {\n      // Trim the key to remove leading/trailing spaces\n      const trimmedKey = key.trim();\n      const preserveStringValue =\n        context.preserveStringValueKeys?.includes(trimmedKey) ?? false;\n\n      const normalizedValue =\n        typeof value === 'string'\n          ? preserveStringValue\n            ? value\n            : value.trim()\n          : trimParsedJsonStrings(value, context);\n\n      normalized[trimmedKey] = normalizedValue;\n    }\n\n    return normalized;\n  }\n\n  // Handle primitive strings\n  if (typeof obj === 'string') {\n    return obj.trim();\n  }\n\n  // Return other primitives as-is\n  return obj;\n}\n\nfunction repairKnownJsonIssues(\n  jsonBlock: string,\n  _rawResponse: string,\n): string {\n  // TODO: Add project-specific repairs that jsonrepair cannot handle.\n  return jsonBlock;\n}\n\nexport function assertJsonObject(\n  parsed: unknown,\n): asserts parsed is Record<string, unknown> {\n  if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {\n    const parsedType = Array.isArray(parsed)\n      ? 'array'\n      : parsed === null\n        ? 'null'\n        : typeof parsed;\n    throw new Error(\n      `LLM response is valid JSON but does not match the expected schema. Expected to be a JSON object, got ${parsedType}: ${JSON.stringify(parsed)}.`,\n    );\n  }\n}\n\nfunction parseJsonWithRepair(jsonStr: string) {\n  try {\n    return JSON.parse(jsonStr);\n  } catch {\n    return JSON.parse(jsonrepair(jsonStr));\n  }\n}\n\nexport function parseModelResponseJson(\n  raw: string,\n  context?: JsonParserContext,\n) {\n  const cleanJsonString = extractJSONFromCodeBlock(raw);\n\n  let parsedObj: unknown;\n\n  try {\n    parsedObj = parseJsonWithRepair(cleanJsonString);\n  } catch (e1) {\n    const code = repairKnownJsonIssues(cleanJsonString, raw);\n    if (code === cleanJsonString) {\n      throw new Error(\n        `failed to parse LLM response into JSON. Error - ${String(\n          e1,\n        )}. Response - \\n ${raw}`,\n      );\n    }\n\n    try {\n      parsedObj = parseJsonWithRepair(code);\n    } catch (e2) {\n      throw new Error(\n        `failed to parse LLM response into JSON. First error - ${String(\n          e1,\n        )}. Second error - ${String(e2)}. Response - \\n ${raw}`,\n      );\n    }\n  }\n\n  return trimParsedJsonStrings(parsedObj, context);\n}\n"],"names":["extractJSONFromCodeBlock","response","jsonMatch","codeBlockMatch","jsonLikeMatch","trimParsedJsonStrings","obj","context","Array","item","normalized","key","value","Object","trimmedKey","preserveStringValue","normalizedValue","repairKnownJsonIssues","jsonBlock","_rawResponse","assertJsonObject","parsed","parsedType","Error","JSON","parseJsonWithRepair","jsonStr","jsonrepair","parseModelResponseJson","raw","cleanJsonString","parsedObj","e1","code","String","e2"],"mappings":";AAmBO,SAASA,yBAAyBC,QAAgB;IACvD,IAAI;QAEF,MAAMC,YAAYD,SAAS,KAAK,CAAC;QACjC,IAAIC,WACF,OAAOA,SAAS,CAAC,EAAE;QAIrB,MAAMC,iBAAiBF,SAAS,KAAK,CACnC;QAEF,IAAIE,gBACF,OAAOA,cAAc,CAAC,EAAE;QAI1B,MAAMC,gBAAgBH,SAAS,KAAK,CAAC;QACrC,IAAIG,eACF,OAAOA,aAAa,CAAC,EAAE;IAE3B,EAAE,OAAM,CAAC;IAET,OAAOH;AACT;AAqBA,SAASI,sBACPC,GAAQ,EACRC,UAA8D,CAAC,CAAC;IAGhE,IAAID,QAAAA,KACF,OAAOA;IAIT,IAAIE,MAAM,OAAO,CAACF,MAChB,OAAOA,IAAI,GAAG,CAAC,CAACG,OAASJ,sBAAsBI,MAAMF;IAIvD,IAAI,AAAe,YAAf,OAAOD,KAAkB;QAC3B,MAAMI,aAAkB,CAAC;QAEzB,KAAK,MAAM,CAACC,KAAKC,MAAM,IAAIC,OAAO,OAAO,CAACP,KAAM;YAE9C,MAAMQ,aAAaH,IAAI,IAAI;YAC3B,MAAMI,sBACJR,QAAQ,uBAAuB,EAAE,SAASO,eAAe;YAE3D,MAAME,kBACJ,AAAiB,YAAjB,OAAOJ,QACHG,sBACEH,QACAA,MAAM,IAAI,KACZP,sBAAsBO,OAAOL;YAEnCG,UAAU,CAACI,WAAW,GAAGE;QAC3B;QAEA,OAAON;IACT;IAGA,IAAI,AAAe,YAAf,OAAOJ,KACT,OAAOA,IAAI,IAAI;IAIjB,OAAOA;AACT;AAEA,SAASW,sBACPC,SAAiB,EACjBC,YAAoB;IAGpB,OAAOD;AACT;AAEO,SAASE,iBACdC,MAAe;IAEf,IAAI,CAACA,UAAU,AAAkB,YAAlB,OAAOA,UAAuBb,MAAM,OAAO,CAACa,SAAS;QAClE,MAAMC,aAAad,MAAM,OAAO,CAACa,UAC7B,UACAA,AAAW,SAAXA,SACE,SACA,OAAOA;QACb,MAAM,IAAIE,MACR,CAAC,qGAAqG,EAAED,WAAW,EAAE,EAAEE,KAAK,SAAS,CAACH,QAAQ,CAAC,CAAC;IAEpJ;AACF;AAEA,SAASI,oBAAoBC,OAAe;IAC1C,IAAI;QACF,OAAOF,KAAK,KAAK,CAACE;IACpB,EAAE,OAAM;QACN,OAAOF,KAAK,KAAK,CAACG,WAAWD;IAC/B;AACF;AAEO,SAASE,uBACdC,GAAW,EACXtB,OAA2B;IAE3B,MAAMuB,kBAAkB9B,yBAAyB6B;IAEjD,IAAIE;IAEJ,IAAI;QACFA,YAAYN,oBAAoBK;IAClC,EAAE,OAAOE,IAAI;QACX,MAAMC,OAAOhB,sBAAsBa,iBAAiBD;QACpD,IAAII,SAASH,iBACX,MAAM,IAAIP,MACR,CAAC,gDAAgD,EAAEW,OACjDF,IACA,gBAAgB,EAAEH,KAAK;QAI7B,IAAI;YACFE,YAAYN,oBAAoBQ;QAClC,EAAE,OAAOE,IAAI;YACX,MAAM,IAAIZ,MACR,CAAC,sDAAsD,EAAEW,OACvDF,IACA,iBAAiB,EAAEE,OAAOC,IAAI,gBAAgB,EAAEN,KAAK;QAE3D;IACF;IAEA,OAAOxB,sBAAsB0B,WAAWxB;AAC1C"}

package/dist/es/ai-model/shared/model-locate-result/bbox.mjs

@@ -0,0 +1,60 @@
+function maxPixelIndex(size) {
+    return Math.max(size - 1, 0);
+}
+function normalizedCoordinateToPixelIndex(value, normalizedBy, size) {
+    return Math.round(value * maxPixelIndex(size) / normalizedBy);
+}
+function mapNormalizedCoordinatesToPixelBbox(coordinates, normalizedBy, width, height) {
+    const [left, top, right, bottom] = coordinates;
+    return [
+        normalizedCoordinateToPixelIndex(left, normalizedBy, width),
+        normalizedCoordinateToPixelIndex(top, normalizedBy, height),
+        normalizedCoordinateToPixelIndex(right, normalizedBy, width),
+        normalizedCoordinateToPixelIndex(bottom, normalizedBy, height)
+    ];
+}
+function expandPointToBbox(x, y, maxX, maxY, halfSize) {
+    return [
+        Math.max(0, x - halfSize),
+        Math.max(0, y - halfSize),
+        Math.min(maxX, x + halfSize),
+        Math.min(maxY, y + halfSize)
+    ];
+}
+function clamp(value, min, max) {
+    return Math.min(Math.max(value, min), max);
+}
+function assertFinitePixelBbox(pixelBbox, rawResult) {
+    if (4 !== pixelBbox.length || !pixelBbox.every((value)=>'number' == typeof value && Number.isFinite(value))) throw new Error(`invalid locate bbox data: ${JSON.stringify(rawResult)} `);
+}
+function assertPixelBboxOrder(pixelBbox, rawResult) {
+    const [left, top, right, bottom] = pixelBbox;
+    if (right >= left && bottom >= top) return;
+    throw new Error(`locate pixel bbox has invalid coordinate order: bbox=${JSON.stringify(rawResult)} pixelBbox=${JSON.stringify(pixelBbox)}`);
+}
+function assertPixelBboxInsideImage(pixelBbox, rawResult, width, height) {
+    const [left, top, right, bottom] = pixelBbox;
+    const maxRight = maxPixelIndex(width);
+    const maxBottom = maxPixelIndex(height);
+    const outOfImage = left < 0 || top < 0 || right > maxRight || bottom > maxBottom;
+    if (!outOfImage) return;
+    throw new Error(`locate pixel bbox is outside the image size: bbox=${JSON.stringify(rawResult)} imageSize=${width}x${height}`);
+}
+function finalizePixelBbox(pixelBbox, rawResult, { preparedSize, contentSize }) {
+    const { width, height } = preparedSize;
+    assertFinitePixelBbox(pixelBbox, rawResult);
+    assertPixelBboxOrder(pixelBbox, rawResult);
+    assertPixelBboxInsideImage(pixelBbox, rawResult, width, height);
+    const rightLimit = maxPixelIndex(contentSize?.width ?? width);
+    const bottomLimit = maxPixelIndex(contentSize?.height ?? height);
+    const [left, top, right, bottom] = pixelBbox;
+    return [
+        clamp(left, 0, rightLimit),
+        clamp(top, 0, bottomLimit),
+        clamp(right, 0, rightLimit),
+        clamp(bottom, 0, bottomLimit)
+    ];
+}
+export { expandPointToBbox, finalizePixelBbox, mapNormalizedCoordinatesToPixelBbox, maxPixelIndex, normalizedCoordinateToPixelIndex };
+
+//# sourceMappingURL=bbox.mjs.map

package/dist/es/ai-model/shared/model-locate-result/bbox.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/shared/model-locate-result/bbox.mjs","sources":["../../../../../src/ai-model/shared/model-locate-result/bbox.ts"],"sourcesContent":["import type { LocateResultBbox, LocateResultContext, PixelBbox } from './types';\n\nexport function maxPixelIndex(size: number) {\n  return Math.max(size - 1, 0);\n}\n\nexport function normalizedCoordinateToPixelIndex(\n  value: number,\n  normalizedBy: number,\n  size: number,\n) {\n  return Math.round((value * maxPixelIndex(size)) / normalizedBy);\n}\n\nexport function mapNormalizedCoordinatesToPixelBbox(\n  coordinates: LocateResultBbox,\n  normalizedBy: number,\n  width: number,\n  height: number,\n): PixelBbox {\n  // PixelBbox uses inclusive pixel indexes, so normalized coordinates map to\n  // size - 1.\n  const [left, top, right, bottom] = coordinates;\n  return [\n    normalizedCoordinateToPixelIndex(left, normalizedBy, width),\n    normalizedCoordinateToPixelIndex(top, normalizedBy, height),\n    normalizedCoordinateToPixelIndex(right, normalizedBy, width),\n    normalizedCoordinateToPixelIndex(bottom, normalizedBy, height),\n  ];\n}\n\nexport function expandPointToBbox(\n  x: number,\n  y: number,\n  maxX: number,\n  maxY: number,\n  halfSize: number,\n): LocateResultBbox {\n  return [\n    Math.max(0, x - halfSize),\n    Math.max(0, y - halfSize),\n    Math.min(maxX, x + halfSize),\n    Math.min(maxY, y + halfSize),\n  ];\n}\n\nfunction clamp(value: number, min: number, max: number) {\n  return Math.min(Math.max(value, min), max);\n}\n\nfunction assertFinitePixelBbox(\n  pixelBbox: readonly number[],\n  rawResult: unknown,\n): asserts pixelBbox is PixelBbox {\n  if (\n    pixelBbox.length !== 4 ||\n    !pixelBbox.every(\n      (value) => typeof value === 'number' && Number.isFinite(value),\n    )\n  ) {\n    throw new Error(`invalid locate bbox data: ${JSON.stringify(rawResult)} `);\n  }\n}\n\nfunction assertPixelBboxOrder(pixelBbox: PixelBbox, rawResult: unknown) {\n  const [left, top, right, bottom] = pixelBbox;\n  if (right >= left && bottom >= top) {\n    return;\n  }\n\n  throw new Error(\n    `locate pixel bbox has invalid coordinate order: bbox=${JSON.stringify(\n      rawResult,\n    )} pixelBbox=${JSON.stringify(pixelBbox)}`,\n  );\n}\n\nfunction assertPixelBboxInsideImage(\n  pixelBbox: PixelBbox,\n  rawResult: unknown,\n  width: number,\n  height: number,\n) {\n  const [left, top, right, bottom] = pixelBbox;\n  const maxRight = maxPixelIndex(width);\n  const maxBottom = maxPixelIndex(height);\n  const outOfImage =\n    left < 0 || top < 0 || right > maxRight || bottom > maxBottom;\n\n  if (!outOfImage) {\n    return;\n  }\n\n  throw new Error(\n    `locate pixel bbox is outside the image size: bbox=${JSON.stringify(\n      rawResult,\n    )} imageSize=${width}x${height}`,\n  );\n}\n\nexport function finalizePixelBbox(\n  pixelBbox: PixelBbox,\n  rawResult: unknown,\n  { preparedSize, contentSize }: LocateResultContext,\n): PixelBbox {\n  const { width, height } = preparedSize;\n  assertFinitePixelBbox(pixelBbox, rawResult);\n  assertPixelBboxOrder(pixelBbox, rawResult);\n  assertPixelBboxInsideImage(pixelBbox, rawResult, width, height);\n\n  const rightLimit = maxPixelIndex(contentSize?.width ?? width);\n  const bottomLimit = maxPixelIndex(contentSize?.height ?? height);\n  const [left, top, right, bottom] = pixelBbox;\n\n  return [\n    clamp(left, 0, rightLimit),\n    clamp(top, 0, bottomLimit),\n    clamp(right, 0, rightLimit),\n    clamp(bottom, 0, bottomLimit),\n  ];\n}\n"],"names":["maxPixelIndex","size","Math","normalizedCoordinateToPixelIndex","value","normalizedBy","mapNormalizedCoordinatesToPixelBbox","coordinates","width","height","left","top","right","bottom","expandPointToBbox","x","y","maxX","maxY","halfSize","clamp","min","max","assertFinitePixelBbox","pixelBbox","rawResult","Number","Error","JSON","assertPixelBboxOrder","assertPixelBboxInsideImage","maxRight","maxBottom","outOfImage","finalizePixelBbox","preparedSize","contentSize","rightLimit","bottomLimit"],"mappings":"AAEO,SAASA,cAAcC,IAAY;IACxC,OAAOC,KAAK,GAAG,CAACD,OAAO,GAAG;AAC5B;AAEO,SAASE,iCACdC,KAAa,EACbC,YAAoB,EACpBJ,IAAY;IAEZ,OAAOC,KAAK,KAAK,CAAEE,QAAQJ,cAAcC,QAASI;AACpD;AAEO,SAASC,oCACdC,WAA6B,EAC7BF,YAAoB,EACpBG,KAAa,EACbC,MAAc;IAId,MAAM,CAACC,MAAMC,KAAKC,OAAOC,OAAO,GAAGN;IACnC,OAAO;QACLJ,iCAAiCO,MAAML,cAAcG;QACrDL,iCAAiCQ,KAAKN,cAAcI;QACpDN,iCAAiCS,OAAOP,cAAcG;QACtDL,iCAAiCU,QAAQR,cAAcI;KACxD;AACH;AAEO,SAASK,kBACdC,CAAS,EACTC,CAAS,EACTC,IAAY,EACZC,IAAY,EACZC,QAAgB;IAEhB,OAAO;QACLjB,KAAK,GAAG,CAAC,GAAGa,IAAII;QAChBjB,KAAK,GAAG,CAAC,GAAGc,IAAIG;QAChBjB,KAAK,GAAG,CAACe,MAAMF,IAAII;QACnBjB,KAAK,GAAG,CAACgB,MAAMF,IAAIG;KACpB;AACH;AAEA,SAASC,MAAMhB,KAAa,EAAEiB,GAAW,EAAEC,GAAW;IACpD,OAAOpB,KAAK,GAAG,CAACA,KAAK,GAAG,CAACE,OAAOiB,MAAMC;AACxC;AAEA,SAASC,sBACPC,SAA4B,EAC5BC,SAAkB;IAElB,IACED,AAAqB,MAArBA,UAAU,MAAM,IAChB,CAACA,UAAU,KAAK,CACd,CAACpB,QAAU,AAAiB,YAAjB,OAAOA,SAAsBsB,OAAO,QAAQ,CAACtB,SAG1D,MAAM,IAAIuB,MAAM,CAAC,0BAA0B,EAAEC,KAAK,SAAS,CAACH,WAAW,CAAC,CAAC;AAE7E;AAEA,SAASI,qBAAqBL,SAAoB,EAAEC,SAAkB;IACpE,MAAM,CAACf,MAAMC,KAAKC,OAAOC,OAAO,GAAGW;IACnC,IAAIZ,SAASF,QAAQG,UAAUF,KAC7B;IAGF,MAAM,IAAIgB,MACR,CAAC,qDAAqD,EAAEC,KAAK,SAAS,CACpEH,WACA,WAAW,EAAEG,KAAK,SAAS,CAACJ,YAAY;AAE9C;AAEA,SAASM,2BACPN,SAAoB,EACpBC,SAAkB,EAClBjB,KAAa,EACbC,MAAc;IAEd,MAAM,CAACC,MAAMC,KAAKC,OAAOC,OAAO,GAAGW;IACnC,MAAMO,WAAW/B,cAAcQ;IAC/B,MAAMwB,YAAYhC,cAAcS;IAChC,MAAMwB,aACJvB,OAAO,KAAKC,MAAM,KAAKC,QAAQmB,YAAYlB,SAASmB;IAEtD,IAAI,CAACC,YACH;IAGF,MAAM,IAAIN,MACR,CAAC,kDAAkD,EAAEC,KAAK,SAAS,CACjEH,WACA,WAAW,EAAEjB,MAAM,CAAC,EAAEC,QAAQ;AAEpC;AAEO,SAASyB,kBACdV,SAAoB,EACpBC,SAAkB,EAClB,EAAEU,YAAY,EAAEC,WAAW,EAAuB;IAElD,MAAM,EAAE5B,KAAK,EAAEC,MAAM,EAAE,GAAG0B;IAC1BZ,sBAAsBC,WAAWC;IACjCI,qBAAqBL,WAAWC;IAChCK,2BAA2BN,WAAWC,WAAWjB,OAAOC;IAExD,MAAM4B,aAAarC,cAAcoC,aAAa,SAAS5B;IACvD,MAAM8B,cAActC,cAAcoC,aAAa,UAAU3B;IACzD,MAAM,CAACC,MAAMC,KAAKC,OAAOC,OAAO,GAAGW;IAEnC,OAAO;QACLJ,MAAMV,MAAM,GAAG2B;QACfjB,MAAMT,KAAK,GAAG2B;QACdlB,MAAMR,OAAO,GAAGyB;QAChBjB,MAAMP,QAAQ,GAAGyB;KAClB;AACH"}

package/dist/es/ai-model/shared/model-locate-result/coordinate-distance.mjs

@@ -0,0 +1,10 @@
+function createCoordinateDistanceToPixels(size, coordinateSystem) {
+    return (delta, axis)=>{
+        if (void 0 === coordinateSystem.normalizedBy) return Math.round(Math.abs(delta));
+        const length = 'x' === axis ? size.width : size.height;
+        return Math.round(Math.abs(delta) * length / coordinateSystem.normalizedBy);
+    };
+}
+export { createCoordinateDistanceToPixels };
+
+//# sourceMappingURL=coordinate-distance.mjs.map

package/dist/es/ai-model/shared/model-locate-result/coordinate-distance.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/shared/model-locate-result/coordinate-distance.mjs","sources":["../../../../../src/ai-model/shared/model-locate-result/coordinate-distance.ts"],"sourcesContent":["import type { Size } from '@/types';\nimport type { ResolvedLocateResultCoordinates } from './types';\n\nexport type CoordinateDistanceAxis = 'x' | 'y';\n\nexport function createCoordinateDistanceToPixels(\n  size: Size,\n  coordinateSystem: ResolvedLocateResultCoordinates,\n) {\n  return (delta: number, axis: CoordinateDistanceAxis): number => {\n    if (coordinateSystem.normalizedBy === undefined) {\n      return Math.round(Math.abs(delta));\n    }\n\n    const length = axis === 'x' ? size.width : size.height;\n    return Math.round(\n      (Math.abs(delta) * length) / coordinateSystem.normalizedBy,\n    );\n  };\n}\n"],"names":["createCoordinateDistanceToPixels","size","coordinateSystem","delta","axis","undefined","Math","length"],"mappings":"AAKO,SAASA,iCACdC,IAAU,EACVC,gBAAiD;IAEjD,OAAO,CAACC,OAAeC;QACrB,IAAIF,AAAkCG,WAAlCH,iBAAiB,YAAY,EAC/B,OAAOI,KAAK,KAAK,CAACA,KAAK,GAAG,CAACH;QAG7B,MAAMI,SAASH,AAAS,QAATA,OAAeH,KAAK,KAAK,GAAGA,KAAK,MAAM;QACtD,OAAOK,KAAK,KAAK,CACdA,KAAK,GAAG,CAACH,SAASI,SAAUL,iBAAiB,YAAY;IAE9D;AACF"}

package/dist/es/ai-model/shared/model-locate-result/errors.mjs

@@ -0,0 +1,9 @@
+const modelConfigDocUrl = 'https://midscenejs.com/model-config';
+const defaultModelFamilyRequiredForLocateMessage = `Default model family is required for locate. Configure MIDSCENE_MODEL_FAMILY so Midscene can parse locate coordinates correctly. ${modelConfigDocUrl}`;
+function planningModelFamilyRequiredForLocateMessage(slot) {
+    if ('planning' === slot) return `Planning model family is required because aiAct is asking the planning model to return locate coordinates. Configure MIDSCENE_PLANNING_MODEL_FAMILY for the planning model, or remove the separate planning model config and configure MIDSCENE_MODEL_FAMILY on the default model. ${modelConfigDocUrl}`;
+    return `Default model family is required because aiAct is asking the default model to return locate coordinates during planning. Configure MIDSCENE_MODEL_FAMILY so Midscene can parse planning locate coordinates correctly. ${modelConfigDocUrl}`;
+}
+export { defaultModelFamilyRequiredForLocateMessage, planningModelFamilyRequiredForLocateMessage };
+
+//# sourceMappingURL=errors.mjs.map

package/dist/es/ai-model/shared/model-locate-result/errors.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/shared/model-locate-result/errors.mjs","sources":["../../../../../src/ai-model/shared/model-locate-result/errors.ts"],"sourcesContent":["const modelConfigDocUrl = 'https://midscenejs.com/model-config';\n\nexport const defaultModelFamilyRequiredForLocateMessage = `Default model family is required for locate. Configure MIDSCENE_MODEL_FAMILY so Midscene can parse locate coordinates correctly. ${modelConfigDocUrl}`;\n\nexport function planningModelFamilyRequiredForLocateMessage(slot?: string) {\n  if (slot === 'planning') {\n    return `Planning model family is required because aiAct is asking the planning model to return locate coordinates. Configure MIDSCENE_PLANNING_MODEL_FAMILY for the planning model, or remove the separate planning model config and configure MIDSCENE_MODEL_FAMILY on the default model. ${modelConfigDocUrl}`;\n  }\n\n  return `Default model family is required because aiAct is asking the default model to return locate coordinates during planning. Configure MIDSCENE_MODEL_FAMILY so Midscene can parse planning locate coordinates correctly. ${modelConfigDocUrl}`;\n}\n"],"names":["modelConfigDocUrl","defaultModelFamilyRequiredForLocateMessage","planningModelFamilyRequiredForLocateMessage","slot"],"mappings":"AAAA,MAAMA,oBAAoB;AAEnB,MAAMC,6CAA6C,CAAC,iIAAiI,EAAED,mBAAmB;AAE1M,SAASE,4CAA4CC,IAAa;IACvE,IAAIA,AAAS,eAATA,MACF,OAAO,CAAC,mRAAmR,EAAEH,mBAAmB;IAGlT,OAAO,CAAC,sNAAsN,EAAEA,mBAAmB;AACrP"}

package/dist/es/ai-model/shared/model-locate-result/factory.mjs

@@ -0,0 +1,39 @@
+import { finalizePixelBbox } from "./bbox.mjs";
+import { parseNumericLocateResult } from "./parse.mjs";
+import { mapLocateResultToPixelBboxByCoordinates } from "./pixel-bbox-mapper.mjs";
+import { createLocateResultPromptSpec } from "./prompt-spec.mjs";
+function resolveLocateResultCoordinates(coordinates) {
+    const order = coordinates.order ?? 'xy';
+    if (void 0 !== coordinates.normalizedBy && coordinates.normalizedBy <= 0) throw new Error(`locate result coordinates normalizedBy must be positive: ${coordinates.normalizedBy}`);
+    return {
+        shape: coordinates.shape,
+        order,
+        normalizedBy: coordinates.normalizedBy
+    };
+}
+function assertValidParsedLocateResult(result) {
+    if (!result || 'object' != typeof result) throw new Error(`invalid parsed locate result: expected object, got ${JSON.stringify(result)}`);
+    const coordinatesMeta = result.coordinatesMeta;
+    const expectedLength = coordinatesMeta?.shape === 'bbox' ? 4 : coordinatesMeta?.shape === 'point' ? 2 : 0;
+    if (!expectedLength) throw new Error(`invalid parsed locate result: unsupported coordinatesMeta.shape ${JSON.stringify(coordinatesMeta?.shape)}`);
+    const coordinates = result.coordinates;
+    if (!Array.isArray(coordinates) || coordinates.length !== expectedLength || !coordinates.every((value)=>'number' == typeof value && Number.isFinite(value))) throw new Error(`invalid parsed locate result: ${coordinatesMeta.shape} coordinates must be ${expectedLength} finite numbers, got ${JSON.stringify(coordinates)}`);
+}
+function createLocateResultCodec(config) {
+    const resolvedCoordinates = resolveLocateResultCoordinates(config.coordinates);
+    const parseRawLocateValue = config.parseRawLocateValue ?? ((input)=>parseNumericLocateResult(resolvedCoordinates, input));
+    const mapLocateResultToPixelBbox = config.mapLocateResultToPixelBbox ?? ((result, ctx)=>mapLocateResultToPixelBboxByCoordinates(result, ctx));
+    const toPixelBbox = (rawResult, context)=>{
+        const parsedResult = parseRawLocateValue(rawResult);
+        assertValidParsedLocateResult(parsedResult);
+        const pixelBbox = mapLocateResultToPixelBbox(parsedResult, context);
+        return finalizePixelBbox(pixelBbox, rawResult, context);
+    };
+    return {
+        promptSpec: createLocateResultPromptSpec(resolvedCoordinates),
+        toPixelBbox
+    };
+}
+export { createLocateResultCodec, resolveLocateResultCoordinates };
+
+//# sourceMappingURL=factory.mjs.map

package/dist/es/ai-model/shared/model-locate-result/factory.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/shared/model-locate-result/factory.mjs","sources":["../../../../../src/ai-model/shared/model-locate-result/factory.ts"],"sourcesContent":["import { finalizePixelBbox } from './bbox';\nimport { parseNumericLocateResult } from './parse';\nimport { mapLocateResultToPixelBboxByCoordinates } from './pixel-bbox-mapper';\nimport { createLocateResultPromptSpec } from './prompt-spec';\nimport type {\n  LocateResultCodec,\n  LocateResultContext,\n  LocateResultCoordinates,\n  LocateResultFormatDefinition,\n  LocateResultValue,\n  ResolvedLocateResultCoordinates,\n} from './types';\n\nexport function resolveLocateResultCoordinates(\n  coordinates: LocateResultCoordinates,\n): ResolvedLocateResultCoordinates {\n  const order = coordinates.order ?? 'xy';\n  if (coordinates.normalizedBy !== undefined && coordinates.normalizedBy <= 0) {\n    throw new Error(\n      `locate result coordinates normalizedBy must be positive: ${coordinates.normalizedBy}`,\n    );\n  }\n  return {\n    shape: coordinates.shape,\n    order,\n    normalizedBy: coordinates.normalizedBy,\n  };\n}\n\nfunction assertValidParsedLocateResult(result: LocateResultValue): void {\n  if (!result || typeof result !== 'object') {\n    throw new Error(\n      `invalid parsed locate result: expected object, got ${JSON.stringify(\n        result,\n      )}`,\n    );\n  }\n\n  const coordinatesMeta = result.coordinatesMeta;\n  const expectedLength =\n    coordinatesMeta?.shape === 'bbox'\n      ? 4\n      : coordinatesMeta?.shape === 'point'\n        ? 2\n        : 0;\n  if (!expectedLength) {\n    throw new Error(\n      `invalid parsed locate result: unsupported coordinatesMeta.shape ${JSON.stringify(\n        coordinatesMeta?.shape,\n      )}`,\n    );\n  }\n\n  const coordinates = result.coordinates;\n  if (\n    !Array.isArray(coordinates) ||\n    coordinates.length !== expectedLength ||\n    !coordinates.every(\n      (value) => typeof value === 'number' && Number.isFinite(value),\n    )\n  ) {\n    throw new Error(\n      `invalid parsed locate result: ${coordinatesMeta.shape} coordinates must be ${expectedLength} finite numbers, got ${JSON.stringify(\n        coordinates,\n      )}`,\n    );\n  }\n}\n\nexport function createLocateResultCodec(\n  config: LocateResultFormatDefinition,\n): LocateResultCodec {\n  const resolvedCoordinates = resolveLocateResultCoordinates(\n    config.coordinates,\n  );\n  const parseRawLocateValue =\n    config.parseRawLocateValue ??\n    ((input) => parseNumericLocateResult(resolvedCoordinates, input));\n  const mapLocateResultToPixelBbox =\n    config.mapLocateResultToPixelBbox ??\n    ((result, ctx) => mapLocateResultToPixelBboxByCoordinates(result, ctx));\n\n  const toPixelBbox = (rawResult: unknown, context: LocateResultContext) => {\n    const parsedResult = parseRawLocateValue(rawResult);\n    assertValidParsedLocateResult(parsedResult);\n    const pixelBbox = mapLocateResultToPixelBbox(parsedResult, context);\n    return finalizePixelBbox(pixelBbox, rawResult, context);\n  };\n\n  return {\n    promptSpec: createLocateResultPromptSpec(resolvedCoordinates),\n    toPixelBbox,\n  };\n}\n"],"names":["resolveLocateResultCoordinates","coordinates","order","undefined","Error","assertValidParsedLocateResult","result","JSON","coordinatesMeta","expectedLength","Array","value","Number","createLocateResultCodec","config","resolvedCoordinates","parseRawLocateValue","input","parseNumericLocateResult","mapLocateResultToPixelBbox","ctx","mapLocateResultToPixelBboxByCoordinates","toPixelBbox","rawResult","context","parsedResult","pixelBbox","finalizePixelBbox","createLocateResultPromptSpec"],"mappings":";;;;AAaO,SAASA,+BACdC,WAAoC;IAEpC,MAAMC,QAAQD,YAAY,KAAK,IAAI;IACnC,IAAIA,AAA6BE,WAA7BF,YAAY,YAAY,IAAkBA,YAAY,YAAY,IAAI,GACxE,MAAM,IAAIG,MACR,CAAC,yDAAyD,EAAEH,YAAY,YAAY,EAAE;IAG1F,OAAO;QACL,OAAOA,YAAY,KAAK;QACxBC;QACA,cAAcD,YAAY,YAAY;IACxC;AACF;AAEA,SAASI,8BAA8BC,MAAyB;IAC9D,IAAI,CAACA,UAAU,AAAkB,YAAlB,OAAOA,QACpB,MAAM,IAAIF,MACR,CAAC,mDAAmD,EAAEG,KAAK,SAAS,CAClED,SACC;IAIP,MAAME,kBAAkBF,OAAO,eAAe;IAC9C,MAAMG,iBACJD,iBAAiB,UAAU,SACvB,IACAA,iBAAiB,UAAU,UACzB,IACA;IACR,IAAI,CAACC,gBACH,MAAM,IAAIL,MACR,CAAC,gEAAgE,EAAEG,KAAK,SAAS,CAC/EC,iBAAiB,QAChB;IAIP,MAAMP,cAAcK,OAAO,WAAW;IACtC,IACE,CAACI,MAAM,OAAO,CAACT,gBACfA,YAAY,MAAM,KAAKQ,kBACvB,CAACR,YAAY,KAAK,CAChB,CAACU,QAAU,AAAiB,YAAjB,OAAOA,SAAsBC,OAAO,QAAQ,CAACD,SAG1D,MAAM,IAAIP,MACR,CAAC,8BAA8B,EAAEI,gBAAgB,KAAK,CAAC,qBAAqB,EAAEC,eAAe,qBAAqB,EAAEF,KAAK,SAAS,CAChIN,cACC;AAGT;AAEO,SAASY,wBACdC,MAAoC;IAEpC,MAAMC,sBAAsBf,+BAC1Bc,OAAO,WAAW;IAEpB,MAAME,sBACJF,OAAO,mBAAmB,IACxB,EAAAG,QAAUC,yBAAyBH,qBAAqBE,MAAK;IACjE,MAAME,6BACJL,OAAO,0BAA0B,IAC/B,EAAAR,QAAQc,MAAQC,wCAAwCf,QAAQc,IAAG;IAEvE,MAAME,cAAc,CAACC,WAAoBC;QACvC,MAAMC,eAAeT,oBAAoBO;QACzClB,8BAA8BoB;QAC9B,MAAMC,YAAYP,2BAA2BM,cAAcD;QAC3D,OAAOG,kBAAkBD,WAAWH,WAAWC;IACjD;IAEA,OAAO;QACL,YAAYI,6BAA6Bb;QACzCO;IACF;AACF"}

package/dist/es/ai-model/shared/model-locate-result/index.mjs

@@ -0,0 +1,6 @@
+import { createLocateResultCodec, resolveLocateResultCoordinates } from "./factory.mjs";
+import { formatLocateExampleValue } from "./prompt-spec.mjs";
+import { createCoordinateDistanceToPixels } from "./coordinate-distance.mjs";
+import { isBboxLocateResultValue, isPointLocateResultValue } from "./types.mjs";
+import { createLocateResultValue, parseCoordinateList, unwrapCoordinateListLikeInput } from "./parse.mjs";
+export { createCoordinateDistanceToPixels, createLocateResultCodec, createLocateResultValue, formatLocateExampleValue, isBboxLocateResultValue, isPointLocateResultValue, parseCoordinateList, resolveLocateResultCoordinates, unwrapCoordinateListLikeInput };

package/dist/es/ai-model/shared/model-locate-result/parse.mjs

@@ -0,0 +1,45 @@
+function unwrapCoordinateListLikeInput(coordinateList) {
+    if (Array.isArray(coordinateList)) {
+        if (Array.isArray(coordinateList[0])) return coordinateList[0];
+    }
+    return coordinateList;
+}
+function parseCoordinateList(input, label) {
+    const unwrapped = unwrapCoordinateListLikeInput(input);
+    const values = 'string' == typeof unwrapped ? unwrapped.trim().split(/[\s,]+/).filter(Boolean) : unwrapped;
+    if (!Array.isArray(values)) throw new Error(`invalid ${label} data: ${JSON.stringify(input)} `);
+    const numericValues = values.map((value)=>'number' == typeof value ? value : Number(value));
+    if (!numericValues.every((value)=>Number.isFinite(value))) throw new Error(`invalid ${label} data: ${JSON.stringify(input)} `);
+    return numericValues;
+}
+function createLocateResultValue(coordinatesMeta, coordinates) {
+    if ('point' === coordinatesMeta.shape) return {
+        coordinates: [
+            coordinates[0],
+            coordinates[1]
+        ],
+        coordinatesMeta
+    };
+    return {
+        coordinates: [
+            coordinates[0],
+            coordinates[1],
+            coordinates[2],
+            coordinates[3]
+        ],
+        coordinatesMeta
+    };
+}
+function parseNumericLocateResult(resolvedCoordinates, input) {
+    if ('point' === resolvedCoordinates.shape) {
+        const point = parseCoordinateList(input, 'point');
+        if (point.length < 2) throw new Error(`invalid point data: ${JSON.stringify(input)} `);
+        return createLocateResultValue(resolvedCoordinates, point);
+    }
+    const bbox = parseCoordinateList(input, 'bbox');
+    if (4 !== bbox.length) throw new Error(`invalid bbox data: ${JSON.stringify(input)} `);
+    return createLocateResultValue(resolvedCoordinates, bbox);
+}
+export { createLocateResultValue, parseCoordinateList, parseNumericLocateResult, unwrapCoordinateListLikeInput };
+
+//# sourceMappingURL=parse.mjs.map

package/dist/es/ai-model/shared/model-locate-result/parse.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/shared/model-locate-result/parse.mjs","sources":["../../../../../src/ai-model/shared/model-locate-result/parse.ts"],"sourcesContent":["import type {\n  BboxLocateResultCoordinates,\n  BboxLocateResultValue,\n  LocateResultBbox,\n  LocateResultPoint,\n  LocateResultValue,\n  PointLocateResultCoordinates,\n  PointLocateResultValue,\n  ResolvedLocateResultCoordinates,\n} from './types';\n\ntype CoordinateListLikeInput =\n  | number[]\n  | string[]\n  | string\n  | (number[] | string[])[];\n\nexport function unwrapCoordinateListLikeInput(\n  coordinateList: CoordinateListLikeInput,\n): number[] | string[] | string {\n  if (Array.isArray(coordinateList)) {\n    if (Array.isArray(coordinateList[0])) {\n      return coordinateList[0] as number[] | string[];\n    }\n    return coordinateList as number[] | string[];\n  }\n  return coordinateList as string;\n}\n\nexport function parseCoordinateList(input: unknown, label: string): number[] {\n  const unwrapped = unwrapCoordinateListLikeInput(\n    input as CoordinateListLikeInput,\n  );\n  const values =\n    typeof unwrapped === 'string'\n      ? unwrapped\n          .trim()\n          .split(/[\\s,]+/)\n          .filter(Boolean)\n      : unwrapped;\n\n  if (!Array.isArray(values)) {\n    throw new Error(`invalid ${label} data: ${JSON.stringify(input)} `);\n  }\n\n  const numericValues = values.map((value) =>\n    typeof value === 'number' ? value : Number(value),\n  );\n\n  if (!numericValues.every((value) => Number.isFinite(value))) {\n    throw new Error(`invalid ${label} data: ${JSON.stringify(input)} `);\n  }\n\n  return numericValues;\n}\n\nexport function createLocateResultValue(\n  coordinatesMeta: PointLocateResultCoordinates,\n  coordinates: number[],\n): PointLocateResultValue;\nexport function createLocateResultValue(\n  coordinatesMeta: BboxLocateResultCoordinates,\n  coordinates: number[],\n): BboxLocateResultValue;\nexport function createLocateResultValue(\n  coordinatesMeta: ResolvedLocateResultCoordinates,\n  coordinates: number[],\n): LocateResultValue {\n  if (coordinatesMeta.shape === 'point') {\n    return {\n      coordinates: [coordinates[0], coordinates[1]] as LocateResultPoint,\n      coordinatesMeta,\n    };\n  }\n\n  return {\n    coordinates: [\n      coordinates[0],\n      coordinates[1],\n      coordinates[2],\n      coordinates[3],\n    ] as LocateResultBbox,\n    coordinatesMeta,\n  };\n}\n\nexport function parseNumericLocateResult(\n  resolvedCoordinates: ResolvedLocateResultCoordinates,\n  input: unknown,\n): LocateResultValue {\n  if (resolvedCoordinates.shape === 'point') {\n    const point = parseCoordinateList(input, 'point');\n    if (point.length < 2) {\n      throw new Error(`invalid point data: ${JSON.stringify(input)} `);\n    }\n    return createLocateResultValue(resolvedCoordinates, point);\n  }\n\n  const bbox = parseCoordinateList(input, 'bbox');\n  if (bbox.length !== 4) {\n    throw new Error(`invalid bbox data: ${JSON.stringify(input)} `);\n  }\n\n  return createLocateResultValue(resolvedCoordinates, bbox);\n}\n"],"names":["unwrapCoordinateListLikeInput","coordinateList","Array","parseCoordinateList","input","label","unwrapped","values","Boolean","Error","JSON","numericValues","value","Number","createLocateResultValue","coordinatesMeta","coordinates","parseNumericLocateResult","resolvedCoordinates","point","bbox"],"mappings":"AAiBO,SAASA,8BACdC,cAAuC;IAEvC,IAAIC,MAAM,OAAO,CAACD,iBAChB;QAAA,IAAIC,MAAM,OAAO,CAACD,cAAc,CAAC,EAAE,GACjC,OAAOA,cAAc,CAAC,EAAE;IAC1B;IAGF,OAAOA;AACT;AAEO,SAASE,oBAAoBC,KAAc,EAAEC,KAAa;IAC/D,MAAMC,YAAYN,8BAChBI;IAEF,MAAMG,SACJ,AAAqB,YAArB,OAAOD,YACHA,UACG,IAAI,GACJ,KAAK,CAAC,UACN,MAAM,CAACE,WACVF;IAEN,IAAI,CAACJ,MAAM,OAAO,CAACK,SACjB,MAAM,IAAIE,MAAM,CAAC,QAAQ,EAAEJ,MAAM,OAAO,EAAEK,KAAK,SAAS,CAACN,OAAO,CAAC,CAAC;IAGpE,MAAMO,gBAAgBJ,OAAO,GAAG,CAAC,CAACK,QAChC,AAAiB,YAAjB,OAAOA,QAAqBA,QAAQC,OAAOD;IAG7C,IAAI,CAACD,cAAc,KAAK,CAAC,CAACC,QAAUC,OAAO,QAAQ,CAACD,SAClD,MAAM,IAAIH,MAAM,CAAC,QAAQ,EAAEJ,MAAM,OAAO,EAAEK,KAAK,SAAS,CAACN,OAAO,CAAC,CAAC;IAGpE,OAAOO;AACT;AAUO,SAASG,wBACdC,eAAgD,EAChDC,WAAqB;IAErB,IAAID,AAA0B,YAA1BA,gBAAgB,KAAK,EACvB,OAAO;QACL,aAAa;YAACC,WAAW,CAAC,EAAE;YAAEA,WAAW,CAAC,EAAE;SAAC;QAC7CD;IACF;IAGF,OAAO;QACL,aAAa;YACXC,WAAW,CAAC,EAAE;YACdA,WAAW,CAAC,EAAE;YACdA,WAAW,CAAC,EAAE;YACdA,WAAW,CAAC,EAAE;SACf;QACDD;IACF;AACF;AAEO,SAASE,yBACdC,mBAAoD,EACpDd,KAAc;IAEd,IAAIc,AAA8B,YAA9BA,oBAAoB,KAAK,EAAc;QACzC,MAAMC,QAAQhB,oBAAoBC,OAAO;QACzC,IAAIe,MAAM,MAAM,GAAG,GACjB,MAAM,IAAIV,MAAM,CAAC,oBAAoB,EAAEC,KAAK,SAAS,CAACN,OAAO,CAAC,CAAC;QAEjE,OAAOU,wBAAwBI,qBAAqBC;IACtD;IAEA,MAAMC,OAAOjB,oBAAoBC,OAAO;IACxC,IAAIgB,AAAgB,MAAhBA,KAAK,MAAM,EACb,MAAM,IAAIX,MAAM,CAAC,mBAAmB,EAAEC,KAAK,SAAS,CAACN,OAAO,CAAC,CAAC;IAGhE,OAAOU,wBAAwBI,qBAAqBE;AACtD"}

package/dist/es/ai-model/shared/model-locate-result/pixel-bbox-mapper.mjs

@@ -0,0 +1,67 @@
+import { expandPointToBbox, mapNormalizedCoordinatesToPixelBbox, maxPixelIndex } from "./bbox.mjs";
+const defaultBboxSize = 20;
+function resolveCoordinateLimits(result, width, height) {
+    const resolvedCoordinates = result.coordinatesMeta;
+    const normalizedBy = resolvedCoordinates.normalizedBy;
+    if (void 0 !== normalizedBy) return result.coordinates.map(()=>normalizedBy);
+    if ('bbox' === resolvedCoordinates.shape) return 'yx' === resolvedCoordinates.order ? [
+        height,
+        width,
+        height,
+        width
+    ] : [
+        width,
+        height,
+        width,
+        height
+    ];
+    return 'yx' === resolvedCoordinates.order ? [
+        height,
+        width
+    ] : [
+        width,
+        height
+    ];
+}
+function assertLocateResultCoordinates(result, width, height) {
+    const resolvedCoordinates = result.coordinatesMeta;
+    const normalizedBy = resolvedCoordinates.normalizedBy;
+    const limits = resolveCoordinateLimits(result, width, height);
+    const outOfRange = result.coordinates.some((value, index)=>{
+        const limit = limits[index];
+        return 'number' != typeof value || !Number.isFinite(value) || value < 0 || value > limit;
+    });
+    if (!outOfRange) return;
+    const source = void 0 !== normalizedBy ? `normalized range [0, ${normalizedBy}]` : `image size [0, ${width}]x[0, ${height}]`;
+    const normalizedInfo = void 0 !== normalizedBy ? ` normalizedBy=${normalizedBy}` : '';
+    throw new Error(`locate result coordinates ${JSON.stringify(result.coordinates)} exceed ${source}. shape=${resolvedCoordinates.shape} order=${resolvedCoordinates.order}${normalizedInfo} limits=${JSON.stringify(limits)}`);
+}
+function reorderCoordinatesToXy(coordinates, order) {
+    if ('yx' !== order) return coordinates;
+    if (4 === coordinates.length) {
+        const [top, left, bottom, right] = coordinates;
+        return [
+            left,
+            top,
+            right,
+            bottom
+        ];
+    }
+    const [y, x] = coordinates;
+    return [
+        x,
+        y
+    ];
+}
+function mapLocateResultToPixelBboxByCoordinates(result, { preparedSize }) {
+    const { width, height } = preparedSize;
+    const resolvedCoordinates = result.coordinatesMeta;
+    const normalizedBy = resolvedCoordinates.normalizedBy;
+    assertLocateResultCoordinates(result, width, height);
+    const xyCoordinates = reorderCoordinatesToXy(result.coordinates, resolvedCoordinates.order);
+    const xyBbox = 4 === xyCoordinates.length ? xyCoordinates : expandPointToBbox(xyCoordinates[0], xyCoordinates[1], normalizedBy ?? maxPixelIndex(width), normalizedBy ?? maxPixelIndex(height), void 0 === normalizedBy ? defaultBboxSize / 2 : normalizedBy / 100);
+    return void 0 === normalizedBy ? xyBbox : mapNormalizedCoordinatesToPixelBbox(xyBbox, normalizedBy, width, height);
+}
+export { mapLocateResultToPixelBboxByCoordinates };
+
+//# sourceMappingURL=pixel-bbox-mapper.mjs.map

package/dist/es/ai-model/shared/model-locate-result/pixel-bbox-mapper.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/shared/model-locate-result/pixel-bbox-mapper.mjs","sources":["../../../../../src/ai-model/shared/model-locate-result/pixel-bbox-mapper.ts"],"sourcesContent":["import {\n  expandPointToBbox,\n  mapNormalizedCoordinatesToPixelBbox,\n  maxPixelIndex,\n} from './bbox';\nimport type {\n  LocateResultBbox,\n  LocateResultValue,\n  PixelBbox,\n  ResolvedLocateResultCoordinates,\n} from './types';\n\ntype Point = [number, number];\n\nconst defaultBboxSize = 20; // must be even number\n\nfunction resolveCoordinateLimits(\n  result: LocateResultValue,\n  width: number,\n  height: number,\n): number[] {\n  const resolvedCoordinates = result.coordinatesMeta;\n  const normalizedBy = resolvedCoordinates.normalizedBy;\n  if (normalizedBy !== undefined) {\n    return result.coordinates.map(() => normalizedBy);\n  }\n\n  if (resolvedCoordinates.shape === 'bbox') {\n    return resolvedCoordinates.order === 'yx'\n      ? [height, width, height, width]\n      : [width, height, width, height];\n  }\n\n  return resolvedCoordinates.order === 'yx' ? [height, width] : [width, height];\n}\n\nfunction assertLocateResultCoordinates(\n  result: LocateResultValue,\n  width: number,\n  height: number,\n) {\n  const resolvedCoordinates = result.coordinatesMeta;\n  const normalizedBy = resolvedCoordinates.normalizedBy;\n  const limits = resolveCoordinateLimits(result, width, height);\n  const outOfRange = result.coordinates.some((value, index) => {\n    const limit = limits[index];\n    return (\n      typeof value !== 'number' ||\n      !Number.isFinite(value) ||\n      value < 0 ||\n      value > limit\n    );\n  });\n\n  if (!outOfRange) {\n    return;\n  }\n\n  const source =\n    normalizedBy !== undefined\n      ? `normalized range [0, ${normalizedBy}]`\n      : `image size [0, ${width}]x[0, ${height}]`;\n  const normalizedInfo =\n    normalizedBy !== undefined ? ` normalizedBy=${normalizedBy}` : '';\n  throw new Error(\n    `locate result coordinates ${JSON.stringify(\n      result.coordinates,\n    )} exceed ${source}. shape=${\n      resolvedCoordinates.shape\n    } order=${resolvedCoordinates.order}${normalizedInfo} limits=${JSON.stringify(\n      limits,\n    )}`,\n  );\n}\n\nfunction reorderCoordinatesToXy(\n  coordinates: LocateResultBbox,\n  order: ResolvedLocateResultCoordinates['order'],\n): LocateResultBbox;\nfunction reorderCoordinatesToXy(\n  coordinates: Point,\n  order: ResolvedLocateResultCoordinates['order'],\n): Point;\nfunction reorderCoordinatesToXy(\n  coordinates: LocateResultBbox | Point,\n  order: ResolvedLocateResultCoordinates['order'],\n): LocateResultBbox | Point;\nfunction reorderCoordinatesToXy(\n  coordinates: LocateResultBbox | Point,\n  order: ResolvedLocateResultCoordinates['order'],\n): LocateResultBbox | Point {\n  if (order !== 'yx') {\n    return coordinates;\n  }\n\n  if (coordinates.length === 4) {\n    const [top, left, bottom, right] = coordinates;\n    return [left, top, right, bottom];\n  }\n\n  const [y, x] = coordinates;\n  return [x, y];\n}\n\nexport function mapLocateResultToPixelBboxByCoordinates(\n  result: LocateResultValue,\n  { preparedSize }: { preparedSize: { width: number; height: number } },\n): PixelBbox {\n  // The parsed result metadata decides whether this maps a bbox or expands a\n  // point, and how to interpret coordinate order and normalization.\n  const { width, height } = preparedSize;\n  const resolvedCoordinates = result.coordinatesMeta;\n  const normalizedBy = resolvedCoordinates.normalizedBy;\n  assertLocateResultCoordinates(result, width, height);\n\n  const xyCoordinates = reorderCoordinatesToXy(\n    result.coordinates,\n    resolvedCoordinates.order,\n  );\n\n  const xyBbox =\n    xyCoordinates.length === 4\n      ? xyCoordinates\n      : expandPointToBbox(\n          xyCoordinates[0],\n          xyCoordinates[1],\n          normalizedBy ?? maxPixelIndex(width),\n          normalizedBy ?? maxPixelIndex(height),\n          normalizedBy === undefined ? defaultBboxSize / 2 : normalizedBy / 100,\n        );\n\n  return normalizedBy === undefined\n    ? xyBbox\n    : mapNormalizedCoordinatesToPixelBbox(xyBbox, normalizedBy, width, height);\n}\n"],"names":["defaultBboxSize","resolveCoordinateLimits","result","width","height","resolvedCoordinates","normalizedBy","undefined","assertLocateResultCoordinates","limits","outOfRange","value","index","limit","Number","source","normalizedInfo","Error","JSON","reorderCoordinatesToXy","coordinates","order","top","left","bottom","right","y","x","mapLocateResultToPixelBboxByCoordinates","preparedSize","xyCoordinates","xyBbox","expandPointToBbox","maxPixelIndex","mapNormalizedCoordinatesToPixelBbox"],"mappings":";AAcA,MAAMA,kBAAkB;AAExB,SAASC,wBACPC,MAAyB,EACzBC,KAAa,EACbC,MAAc;IAEd,MAAMC,sBAAsBH,OAAO,eAAe;IAClD,MAAMI,eAAeD,oBAAoB,YAAY;IACrD,IAAIC,AAAiBC,WAAjBD,cACF,OAAOJ,OAAO,WAAW,CAAC,GAAG,CAAC,IAAMI;IAGtC,IAAID,AAA8B,WAA9BA,oBAAoB,KAAK,EAC3B,OAAOA,AAA8B,SAA9BA,oBAAoB,KAAK,GAC5B;QAACD;QAAQD;QAAOC;QAAQD;KAAM,GAC9B;QAACA;QAAOC;QAAQD;QAAOC;KAAO;IAGpC,OAAOC,AAA8B,SAA9BA,oBAAoB,KAAK,GAAY;QAACD;QAAQD;KAAM,GAAG;QAACA;QAAOC;KAAO;AAC/E;AAEA,SAASI,8BACPN,MAAyB,EACzBC,KAAa,EACbC,MAAc;IAEd,MAAMC,sBAAsBH,OAAO,eAAe;IAClD,MAAMI,eAAeD,oBAAoB,YAAY;IACrD,MAAMI,SAASR,wBAAwBC,QAAQC,OAAOC;IACtD,MAAMM,aAAaR,OAAO,WAAW,CAAC,IAAI,CAAC,CAACS,OAAOC;QACjD,MAAMC,QAAQJ,MAAM,CAACG,MAAM;QAC3B,OACE,AAAiB,YAAjB,OAAOD,SACP,CAACG,OAAO,QAAQ,CAACH,UACjBA,QAAQ,KACRA,QAAQE;IAEZ;IAEA,IAAI,CAACH,YACH;IAGF,MAAMK,SACJT,AAAiBC,WAAjBD,eACI,CAAC,qBAAqB,EAAEA,aAAa,CAAC,CAAC,GACvC,CAAC,eAAe,EAAEH,MAAM,MAAM,EAAEC,OAAO,CAAC,CAAC;IAC/C,MAAMY,iBACJV,AAAiBC,WAAjBD,eAA6B,CAAC,cAAc,EAAEA,cAAc,GAAG;IACjE,MAAM,IAAIW,MACR,CAAC,0BAA0B,EAAEC,KAAK,SAAS,CACzChB,OAAO,WAAW,EAClB,QAAQ,EAAEa,OAAO,QAAQ,EACzBV,oBAAoB,KAAK,CAC1B,OAAO,EAAEA,oBAAoB,KAAK,GAAGW,eAAe,QAAQ,EAAEE,KAAK,SAAS,CAC3ET,SACC;AAEP;AAcA,SAASU,uBACPC,WAAqC,EACrCC,KAA+C;IAE/C,IAAIA,AAAU,SAAVA,OACF,OAAOD;IAGT,IAAIA,AAAuB,MAAvBA,YAAY,MAAM,EAAQ;QAC5B,MAAM,CAACE,KAAKC,MAAMC,QAAQC,MAAM,GAAGL;QACnC,OAAO;YAACG;YAAMD;YAAKG;YAAOD;SAAO;IACnC;IAEA,MAAM,CAACE,GAAGC,EAAE,GAAGP;IACf,OAAO;QAACO;QAAGD;KAAE;AACf;AAEO,SAASE,wCACd1B,MAAyB,EACzB,EAAE2B,YAAY,EAAuD;IAIrE,MAAM,EAAE1B,KAAK,EAAEC,MAAM,EAAE,GAAGyB;IAC1B,MAAMxB,sBAAsBH,OAAO,eAAe;IAClD,MAAMI,eAAeD,oBAAoB,YAAY;IACrDG,8BAA8BN,QAAQC,OAAOC;IAE7C,MAAM0B,gBAAgBX,uBACpBjB,OAAO,WAAW,EAClBG,oBAAoB,KAAK;IAG3B,MAAM0B,SACJD,AAAyB,MAAzBA,cAAc,MAAM,GAChBA,gBACAE,kBACEF,aAAa,CAAC,EAAE,EAChBA,aAAa,CAAC,EAAE,EAChBxB,gBAAgB2B,cAAc9B,QAC9BG,gBAAgB2B,cAAc7B,SAC9BE,AAAiBC,WAAjBD,eAA6BN,kBAAkB,IAAIM,eAAe;IAG1E,OAAOA,AAAiBC,WAAjBD,eACHyB,SACAG,oCAAoCH,QAAQzB,cAAcH,OAAOC;AACvE"}

package/dist/es/ai-model/shared/model-locate-result/prompt-spec.mjs

@@ -0,0 +1,110 @@
+function formatLocateExampleValue(value) {
+    return Array.isArray(value) ? `[${value.join(', ')}]` : JSON.stringify(value);
+}
+function describeLocateResultCoordinates({ shape, order, normalizedBy }) {
+    const descriptor = 'point' === shape ? 'point' : '2d bounding box';
+    const coordinateDescription = void 0 !== normalizedBy ? `normalized to 0-${normalizedBy} relative to the screenshot. Do NOT use pixel coordinates or screenshot width/height` : 'in actual pixel coordinates relative to the screenshot';
+    if ('point' === shape) {
+        const orderDescription = 'yx' === order ? '[y, x]' : '[x, y]';
+        return `${descriptor}, should be ${orderDescription} ${coordinateDescription}.`;
+    }
+    const orderDescription = 'yx' === order ? '[ymin, xmin, ymax, xmax]' : '[xmin, ymin, xmax, ymax]';
+    return `${descriptor}, should be ${orderDescription} ${coordinateDescription}.`;
+}
+function describeLocateResultValueSchema({ shape }) {
+    return 'point' === shape ? '[number, number]' : '[number, number, number, number]';
+}
+function locateResultExampleValue(resolvedCoordinates, region) {
+    const [left, top, right, bottom] = region;
+    if ('point' === resolvedCoordinates.shape) {
+        const x = Math.round((left + right) / 2);
+        const y = Math.round((top + bottom) / 2);
+        return 'yx' === resolvedCoordinates.order ? [
+            y,
+            x
+        ] : [
+            x,
+            y
+        ];
+    }
+    return 'yx' === resolvedCoordinates.order ? [
+        top,
+        left,
+        bottom,
+        right
+    ] : region;
+}
+const locateResultExampleRegions = [
+    [
+        100,
+        100,
+        200,
+        200
+    ],
+    [
+        345,
+        442,
+        458,
+        483
+    ],
+    [
+        120,
+        180,
+        380,
+        210
+    ],
+    [
+        120,
+        240,
+        380,
+        270
+    ],
+    [
+        50,
+        100,
+        200,
+        200
+    ],
+    [
+        300,
+        400,
+        500,
+        500
+    ],
+    [
+        600,
+        100,
+        800,
+        250
+    ],
+    [
+        50,
+        600,
+        250,
+        750
+    ]
+];
+function createExampleValues(resolvedCoordinates) {
+    return locateResultExampleRegions.map((region)=>locateResultExampleValue(resolvedCoordinates, region));
+}
+function locateResultKey({ shape }) {
+    return 'point' === shape ? 'point' : 'bbox';
+}
+function locateResultName({ shape }, { plural = false } = {}) {
+    return 'bbox' === shape ? plural ? 'bounding boxes' : 'bounding box' : plural ? 'points' : 'point';
+}
+function createLocateResultPromptSpec(resolvedCoordinates) {
+    return {
+        resultKey: locateResultKey(resolvedCoordinates),
+        resultValueSchema: describeLocateResultValueSchema(resolvedCoordinates),
+        resultValueDescription: describeLocateResultCoordinates(resolvedCoordinates),
+        resultNoun: locateResultName(resolvedCoordinates),
+        resultNounPlural: locateResultName(resolvedCoordinates, {
+            plural: true
+        }),
+        exampleValues: createExampleValues(resolvedCoordinates)
+    };
+}
+export { createLocateResultPromptSpec, describeLocateResultValueSchema, formatLocateExampleValue, locateResultExampleRegions, locateResultExampleValue };
+
+//# sourceMappingURL=prompt-spec.mjs.map

package/dist/es/ai-model/shared/model-locate-result/prompt-spec.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/shared/model-locate-result/prompt-spec.mjs","sources":["../../../../../src/ai-model/shared/model-locate-result/prompt-spec.ts"],"sourcesContent":["import type {\n  LocateResultBbox,\n  LocateResultPromptSpec,\n  NonEmptyArray,\n  ResolvedLocateResultCoordinates,\n} from './types';\n\nexport function formatLocateExampleValue(value: unknown): string {\n  return Array.isArray(value) ? `[${value.join(', ')}]` : JSON.stringify(value);\n}\n\nfunction describeLocateResultCoordinates({\n  shape,\n  order,\n  normalizedBy,\n}: ResolvedLocateResultCoordinates): string {\n  const descriptor = shape === 'point' ? 'point' : '2d bounding box';\n  const coordinateDescription =\n    normalizedBy !== undefined\n      ? `normalized to 0-${normalizedBy} relative to the screenshot. Do NOT use pixel coordinates or screenshot width/height`\n      : 'in actual pixel coordinates relative to the screenshot';\n\n  if (shape === 'point') {\n    const orderDescription = order === 'yx' ? '[y, x]' : '[x, y]';\n    return `${descriptor}, should be ${orderDescription} ${coordinateDescription}.`;\n  }\n\n  const orderDescription =\n    order === 'yx' ? '[ymin, xmin, ymax, xmax]' : '[xmin, ymin, xmax, ymax]';\n  return `${descriptor}, should be ${orderDescription} ${coordinateDescription}.`;\n}\n\nexport function describeLocateResultValueSchema({\n  shape,\n}: ResolvedLocateResultCoordinates): string {\n  return shape === 'point'\n    ? '[number, number]'\n    : '[number, number, number, number]';\n}\n\nexport function locateResultExampleValue(\n  resolvedCoordinates: ResolvedLocateResultCoordinates,\n  region: LocateResultBbox,\n): number[] {\n  const [left, top, right, bottom] = region;\n  if (resolvedCoordinates.shape === 'point') {\n    const x = Math.round((left + right) / 2);\n    const y = Math.round((top + bottom) / 2);\n    return resolvedCoordinates.order === 'yx' ? [y, x] : [x, y];\n  }\n  return resolvedCoordinates.order === 'yx'\n    ? [top, left, bottom, right]\n    : region;\n}\n\n// Internal xy regions used only for prompt examples.\n// Each item is [xmin, ymin, xmax, ymax], before model-specific order mapping.\nexport const locateResultExampleRegions: LocateResultBbox[] = [\n  [100, 100, 200, 200],\n  [345, 442, 458, 483],\n  [120, 180, 380, 210],\n  [120, 240, 380, 270],\n  [50, 100, 200, 200],\n  [300, 400, 500, 500],\n  [600, 100, 800, 250],\n  [50, 600, 250, 750],\n];\n\nfunction createExampleValues(\n  resolvedCoordinates: ResolvedLocateResultCoordinates,\n): NonEmptyArray<unknown> {\n  return locateResultExampleRegions.map((region) =>\n    locateResultExampleValue(resolvedCoordinates, region),\n  ) as NonEmptyArray<unknown>;\n}\n\nfunction locateResultKey({ shape }: ResolvedLocateResultCoordinates): string {\n  return shape === 'point' ? 'point' : 'bbox';\n}\n\nfunction locateResultName(\n  { shape }: ResolvedLocateResultCoordinates,\n  { plural = false }: { plural?: boolean } = {},\n): string {\n  return shape === 'bbox'\n    ? plural\n      ? 'bounding boxes'\n      : 'bounding box'\n    : plural\n      ? 'points'\n      : 'point';\n}\n\nexport function createLocateResultPromptSpec(\n  resolvedCoordinates: ResolvedLocateResultCoordinates,\n): LocateResultPromptSpec {\n  return {\n    resultKey: locateResultKey(resolvedCoordinates),\n    resultValueSchema: describeLocateResultValueSchema(resolvedCoordinates),\n    resultValueDescription:\n      describeLocateResultCoordinates(resolvedCoordinates),\n    resultNoun: locateResultName(resolvedCoordinates),\n    resultNounPlural: locateResultName(resolvedCoordinates, { plural: true }),\n    exampleValues: createExampleValues(resolvedCoordinates),\n  };\n}\n"],"names":["formatLocateExampleValue","value","Array","JSON","describeLocateResultCoordinates","shape","order","normalizedBy","descriptor","coordinateDescription","undefined","orderDescription","describeLocateResultValueSchema","locateResultExampleValue","resolvedCoordinates","region","left","top","right","bottom","x","Math","y","locateResultExampleRegions","createExampleValues","locateResultKey","locateResultName","plural","createLocateResultPromptSpec"],"mappings":"AAOO,SAASA,yBAAyBC,KAAc;IACrD,OAAOC,MAAM,OAAO,CAACD,SAAS,CAAC,CAAC,EAAEA,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC,GAAGE,KAAK,SAAS,CAACF;AACzE;AAEA,SAASG,gCAAgC,EACvCC,KAAK,EACLC,KAAK,EACLC,YAAY,EACoB;IAChC,MAAMC,aAAaH,AAAU,YAAVA,QAAoB,UAAU;IACjD,MAAMI,wBACJF,AAAiBG,WAAjBH,eACI,CAAC,gBAAgB,EAAEA,aAAa,oFAAoF,CAAC,GACrH;IAEN,IAAIF,AAAU,YAAVA,OAAmB;QACrB,MAAMM,mBAAmBL,AAAU,SAAVA,QAAiB,WAAW;QACrD,OAAO,GAAGE,WAAW,YAAY,EAAEG,iBAAiB,CAAC,EAAEF,sBAAsB,CAAC,CAAC;IACjF;IAEA,MAAME,mBACJL,AAAU,SAAVA,QAAiB,6BAA6B;IAChD,OAAO,GAAGE,WAAW,YAAY,EAAEG,iBAAiB,CAAC,EAAEF,sBAAsB,CAAC,CAAC;AACjF;AAEO,SAASG,gCAAgC,EAC9CP,KAAK,EAC2B;IAChC,OAAOA,AAAU,YAAVA,QACH,qBACA;AACN;AAEO,SAASQ,yBACdC,mBAAoD,EACpDC,MAAwB;IAExB,MAAM,CAACC,MAAMC,KAAKC,OAAOC,OAAO,GAAGJ;IACnC,IAAID,AAA8B,YAA9BA,oBAAoB,KAAK,EAAc;QACzC,MAAMM,IAAIC,KAAK,KAAK,CAAEL,AAAAA,CAAAA,OAAOE,KAAI,IAAK;QACtC,MAAMI,IAAID,KAAK,KAAK,CAAEJ,AAAAA,CAAAA,MAAME,MAAK,IAAK;QACtC,OAAOL,AAA8B,SAA9BA,oBAAoB,KAAK,GAAY;YAACQ;YAAGF;SAAE,GAAG;YAACA;YAAGE;SAAE;IAC7D;IACA,OAAOR,AAA8B,SAA9BA,oBAAoB,KAAK,GAC5B;QAACG;QAAKD;QAAMG;QAAQD;KAAM,GAC1BH;AACN;AAIO,MAAMQ,6BAAiD;IAC5D;QAAC;QAAK;QAAK;QAAK;KAAI;IACpB;QAAC;QAAK;QAAK;QAAK;KAAI;IACpB;QAAC;QAAK;QAAK;QAAK;KAAI;IACpB;QAAC;QAAK;QAAK;QAAK;KAAI;IACpB;QAAC;QAAI;QAAK;QAAK;KAAI;IACnB;QAAC;QAAK;QAAK;QAAK;KAAI;IACpB;QAAC;QAAK;QAAK;QAAK;KAAI;IACpB;QAAC;QAAI;QAAK;QAAK;KAAI;CACpB;AAED,SAASC,oBACPV,mBAAoD;IAEpD,OAAOS,2BAA2B,GAAG,CAAC,CAACR,SACrCF,yBAAyBC,qBAAqBC;AAElD;AAEA,SAASU,gBAAgB,EAAEpB,KAAK,EAAmC;IACjE,OAAOA,AAAU,YAAVA,QAAoB,UAAU;AACvC;AAEA,SAASqB,iBACP,EAAErB,KAAK,EAAmC,EAC1C,EAAEsB,SAAS,KAAK,EAAwB,GAAG,CAAC,CAAC;IAE7C,OAAOtB,AAAU,WAAVA,QACHsB,SACE,mBACA,iBACFA,SACE,WACA;AACR;AAEO,SAASC,6BACdd,mBAAoD;IAEpD,OAAO;QACL,WAAWW,gBAAgBX;QAC3B,mBAAmBF,gCAAgCE;QACnD,wBACEV,gCAAgCU;QAClC,YAAYY,iBAAiBZ;QAC7B,kBAAkBY,iBAAiBZ,qBAAqB;YAAE,QAAQ;QAAK;QACvE,eAAeU,oBAAoBV;IACrC;AACF"}

package/dist/es/ai-model/shared/model-locate-result/types.mjs

@@ -0,0 +1,9 @@
+function isBboxLocateResultValue(result) {
+    return 'bbox' === result.coordinatesMeta.shape;
+}
+function isPointLocateResultValue(result) {
+    return 'point' === result.coordinatesMeta.shape;
+}
+export { isBboxLocateResultValue, isPointLocateResultValue };
+
+//# sourceMappingURL=types.mjs.map

package/dist/es/ai-model/shared/model-locate-result/types.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/shared/model-locate-result/types.mjs","sources":["../../../../../src/ai-model/shared/model-locate-result/types.ts"],"sourcesContent":["import type { Bbox } from '../../../types';\n\nexport type { Bbox };\nexport type LocateResultBbox = Bbox;\nexport type PixelBbox = Bbox;\nexport type NonEmptyArray<T> = [T, ...T[]];\nexport type RawLocateValue = unknown;\n\nexport type LocateResultPoint = [number, number];\nexport type PointLocateResultCoordinates = ResolvedLocateResultCoordinates & {\n  shape: 'point';\n};\nexport type BboxLocateResultCoordinates = ResolvedLocateResultCoordinates & {\n  shape: 'bbox';\n};\n\nexport type LocateResultValue =\n  | {\n      coordinates: LocateResultPoint;\n      coordinatesMeta: PointLocateResultCoordinates;\n    }\n  | {\n      coordinates: LocateResultBbox;\n      coordinatesMeta: BboxLocateResultCoordinates;\n    };\n\nexport type PointLocateResultValue = Extract<\n  LocateResultValue,\n  { coordinatesMeta: { shape: 'point' } }\n>;\nexport type BboxLocateResultValue = Extract<\n  LocateResultValue,\n  { coordinatesMeta: { shape: 'bbox' } }\n>;\n\nexport function isBboxLocateResultValue(\n  result: LocateResultValue,\n): result is BboxLocateResultValue {\n  return result.coordinatesMeta.shape === 'bbox';\n}\n\nexport function isPointLocateResultValue(\n  result: LocateResultValue,\n): result is PointLocateResultValue {\n  return result.coordinatesMeta.shape === 'point';\n}\n\nexport type LocateResultShape = 'bbox' | 'point';\n\nexport interface LocateResultContext {\n  preparedSize: {\n    width: number;\n    height: number;\n  };\n  contentSize?: {\n    width: number;\n    height: number;\n  };\n}\n\nexport interface LocateResultPromptSpec {\n  resultKey: string;\n  resultValueSchema: string;\n  resultValueDescription: string;\n  resultNoun: string;\n  resultNounPlural: string;\n  exampleValues: NonEmptyArray<unknown>;\n}\n\nexport interface LocateResultCodec {\n  promptSpec: LocateResultPromptSpec;\n  toPixelBbox(input: RawLocateValue, ctx: LocateResultContext): PixelBbox;\n}\n\nexport interface LocateResultCoordinates {\n  shape: LocateResultShape;\n  order?: 'xy' | 'yx';\n  normalizedBy?: number;\n}\n\nexport type ResolvedLocateResultCoordinates =\n  | {\n      shape: 'point';\n      order: 'xy' | 'yx';\n      normalizedBy?: number;\n    }\n  | {\n      shape: 'bbox';\n      order: 'xy' | 'yx';\n      normalizedBy?: number;\n    };\n\nexport type RawLocateValueParser = (input: RawLocateValue) => LocateResultValue;\nexport type LocateResultPixelBboxMapper = (\n  result: LocateResultValue,\n  ctx: LocateResultContext,\n) => PixelBbox;\n\n/**\n * Declarative config for the standard locate workflow.\n *\n * The standard workflow has three steps:\n * 1. `coordinates` is expanded into prompt wording, a default\n *    raw result parser, and a default pixel bbox mapper.\n * 2. `parseRawLocateValue` converts that raw result value into Midscene's\n *    internal `LocateResultValue` shape:\n *    `{ coordinates, coordinatesMeta }`. Omit it when the model returns a\n *    plain numeric bbox/point matching `coordinates`; provide it when the\n *    model needs repair, fallback handling, or per-result coordinate metadata.\n * 3. `mapLocateResultToPixelBbox` converts the parsed result into a pixel bbox\n *    `[left, top, right, bottom]`. Omit it when `coordinates` is enough to describe\n *    the coordinate system and order; provide it only for model-specific\n *    conversion rules.\n *\n * This format only describes one raw coordinate value. The operation protocol\n * owns the response envelope and identifies target/reference values before the\n * runtime codec converts each value independently.\n *\n * Example 1: a GLM-like model that directly matches the standard coordinates.\n *\n * ```ts\n * resultFormat: {\n *   coordinates: { shape: 'bbox', order: 'xy', normalizedBy: 1000 },\n * }\n * ```\n *\n * Example 2: Qwen 2.5 returns pixel coordinates, but may return a point-like\n * value that needs custom parsing/fallback. The default pixel bbox mapper is\n * bypassed only if custom fallback sizing is required.\n *\n * ```ts\n * resultFormat: {\n *   coordinates: { shape: 'bbox', order: 'xy' },\n *   parseRawLocateValue: parseQwen25RawLocateValue,\n *   mapLocateResultToPixelBbox: normalizeQwen25ResultToPixelBbox,\n * }\n * ```\n *\n * Example 3: a model with a custom raw value shape can keep the standard\n * workflow while replacing parsing and mapping.\n *\n * ```ts\n * resultFormat: {\n *   coordinates: { shape: 'bbox', order: 'xy' },\n *   parseRawLocateValue: (raw) => ({\n *     coordinates: [\n *       Number((raw as any).left),\n *       Number((raw as any).top),\n *       Number((raw as any).right),\n *       Number((raw as any).bottom),\n *     ],\n *     coordinatesMeta: { shape: 'bbox', order: 'xy' },\n *   }),\n *   mapLocateResultToPixelBbox: (result) => result.coordinates,\n * }\n * ```\n */\nexport type LocateResultFormatDefinition = {\n  /**\n   * Common locate result coordinates shorthand. This is the preferred config surface\n   * for normal models because it keeps result type, coordinate system, and\n   * coordinate order in one orthogonal field.\n   */\n  coordinates: LocateResultCoordinates;\n  /**\n   * Parses one raw coordinate value into a `LocateResultValue`. This function\n   * should handle response repair, bbox-vs-point fallback, and the coordinate\n   * metadata that describes the parsed coordinates.\n   */\n  parseRawLocateValue?: RawLocateValueParser;\n  /**\n   * Maps the parsed result into a pixel bbox. Most models should omit this\n   * and let `coordinates` drive the default conversion. Provide it only when point\n   * fallback size, clipping, or coordinate semantics are model-specific.\n   */\n  mapLocateResultToPixelBbox?: LocateResultPixelBboxMapper;\n};\n"],"names":["isBboxLocateResultValue","result","isPointLocateResultValue"],"mappings":"AAmCO,SAASA,wBACdC,MAAyB;IAEzB,OAAOA,AAAiC,WAAjCA,OAAO,eAAe,CAAC,KAAK;AACrC;AAEO,SAASC,yBACdD,MAAyB;IAEzB,OAAOA,AAAiC,YAAjCA,OAAO,eAAe,CAAC,KAAK;AACrC"}

package/dist/es/ai-model/shared/multimodal-prompt.mjs

@@ -0,0 +1,55 @@
+import { userPromptToMultimodalPrompt, userPromptToString } from "../../common.mjs";
+import { preProcessImageUrl } from "@aiscene/shared/img";
+const prepareReferenceImages = async (multimodalPrompt)=>{
+    const referenceImages = [];
+    for (const image of multimodalPrompt?.images ?? [])referenceImages.push({
+        name: image.name,
+        url: await preProcessImageUrl(image.url, !!multimodalPrompt?.convertHttpImage2Base64)
+    });
+    return referenceImages;
+};
+const prepareUserPrompt = async (userPrompt)=>({
+        text: userPromptToString(userPrompt),
+        referenceImages: await prepareReferenceImages(userPromptToMultimodalPrompt(userPrompt))
+    });
+const preparedReferenceImagesToChatMessages = (referenceImages)=>{
+    if (0 === referenceImages.length) return [];
+    return [
+        {
+            role: 'user',
+            content: [
+                {
+                    type: 'text',
+                    text: 'Next, I will provide all the reference images. These reference images are supporting context only, not the current screenshot being evaluated, unless the task explicitly asks for comparison or matching.'
+                }
+            ]
+        },
+        ...referenceImages.flatMap((image)=>[
+                {
+                    role: 'user',
+                    content: [
+                        {
+                            type: 'text',
+                            text: `this is the reference image named '${image.name}'. It is a reference image, not the current screenshot:`
+                        }
+                    ]
+                },
+                {
+                    role: 'user',
+                    content: [
+                        {
+                            type: 'image_url',
+                            image_url: {
+                                url: image.url,
+                                detail: 'high'
+                            }
+                        }
+                    ]
+                }
+            ])
+    ];
+};
+const multimodalPromptToChatMessages = async (multimodalPrompt)=>preparedReferenceImagesToChatMessages(await prepareReferenceImages(multimodalPrompt));
+export { multimodalPromptToChatMessages, prepareUserPrompt, preparedReferenceImagesToChatMessages };
+
+//# sourceMappingURL=multimodal-prompt.mjs.map

package/dist/es/ai-model/shared/multimodal-prompt.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/shared/multimodal-prompt.mjs","sources":["../../../../src/ai-model/shared/multimodal-prompt.ts"],"sourcesContent":["import {\n  type TMultimodalPrompt,\n  type TUserPrompt,\n  userPromptToMultimodalPrompt,\n  userPromptToString,\n} from '@/common';\nimport { preProcessImageUrl } from '@aiscene/shared/img';\nimport type { ChatCompletionUserMessageParam } from 'openai/resources/index';\n\nexport interface PreparedReferenceImage {\n  name: string;\n  url: string;\n}\n\nexport interface PreparedUserPrompt {\n  text: string;\n  referenceImages: PreparedReferenceImage[];\n}\n\nconst prepareReferenceImages = async (\n  multimodalPrompt?: TMultimodalPrompt,\n): Promise<PreparedReferenceImage[]> => {\n  const referenceImages: PreparedReferenceImage[] = [];\n  for (const image of multimodalPrompt?.images ?? []) {\n    referenceImages.push({\n      name: image.name,\n      url: await preProcessImageUrl(\n        image.url,\n        !!multimodalPrompt?.convertHttpImage2Base64,\n      ),\n    });\n  }\n  return referenceImages;\n};\n\nexport const prepareUserPrompt = async (\n  userPrompt: TUserPrompt,\n): Promise<PreparedUserPrompt> => ({\n  text: userPromptToString(userPrompt),\n  referenceImages: await prepareReferenceImages(\n    userPromptToMultimodalPrompt(userPrompt),\n  ),\n});\n\nexport const preparedReferenceImagesToChatMessages = (\n  referenceImages: PreparedReferenceImage[],\n): ChatCompletionUserMessageParam[] => {\n  if (referenceImages.length === 0) {\n    return [];\n  }\n\n  return [\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: 'Next, I will provide all the reference images. These reference images are supporting context only, not the current screenshot being evaluated, unless the task explicitly asks for comparison or matching.',\n        },\n      ],\n    },\n    ...referenceImages.flatMap((image): ChatCompletionUserMessageParam[] => [\n      {\n        role: 'user',\n        content: [\n          {\n            type: 'text',\n            text: `this is the reference image named '${image.name}'. It is a reference image, not the current screenshot:`,\n          },\n        ],\n      },\n      {\n        role: 'user',\n        content: [\n          {\n            type: 'image_url',\n            image_url: {\n              url: image.url,\n              detail: 'high',\n            },\n          },\n        ],\n      },\n    ]),\n  ];\n};\n\nexport const multimodalPromptToChatMessages = async (\n  multimodalPrompt?: TMultimodalPrompt,\n): Promise<ChatCompletionUserMessageParam[]> =>\n  preparedReferenceImagesToChatMessages(\n    await prepareReferenceImages(multimodalPrompt),\n  );\n"],"names":["prepareReferenceImages","multimodalPrompt","referenceImages","image","preProcessImageUrl","prepareUserPrompt","userPrompt","userPromptToString","userPromptToMultimodalPrompt","preparedReferenceImagesToChatMessages","multimodalPromptToChatMessages"],"mappings":";;AAmBA,MAAMA,yBAAyB,OAC7BC;IAEA,MAAMC,kBAA4C,EAAE;IACpD,KAAK,MAAMC,SAASF,kBAAkB,UAAU,EAAE,CAChDC,gBAAgB,IAAI,CAAC;QACnB,MAAMC,MAAM,IAAI;QAChB,KAAK,MAAMC,mBACTD,MAAM,GAAG,EACT,CAAC,CAACF,kBAAkB;IAExB;IAEF,OAAOC;AACT;AAEO,MAAMG,oBAAoB,OAC/BC,aACiC;QACjC,MAAMC,mBAAmBD;QACzB,iBAAiB,MAAMN,uBACrBQ,6BAA6BF;IAEjC;AAEO,MAAMG,wCAAwC,CACnDP;IAEA,IAAIA,AAA2B,MAA3BA,gBAAgB,MAAM,EACxB,OAAO,EAAE;IAGX,OAAO;QACL;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM;gBACR;aACD;QACH;WACGA,gBAAgB,OAAO,CAAC,CAACC,QAA4C;gBACtE;oBACE,MAAM;oBACN,SAAS;wBACP;4BACE,MAAM;4BACN,MAAM,CAAC,mCAAmC,EAAEA,MAAM,IAAI,CAAC,uDAAuD,CAAC;wBACjH;qBACD;gBACH;gBACA;oBACE,MAAM;oBACN,SAAS;wBACP;4BACE,MAAM;4BACN,WAAW;gCACT,KAAKA,MAAM,GAAG;gCACd,QAAQ;4BACV;wBACF;qBACD;gBACH;aACD;KACF;AACH;AAEO,MAAMO,iCAAiC,OAC5CT,mBAEAQ,sCACE,MAAMT,uBAAuBC"}