@midscene/core 1.16.2 → 1.16.6-beta-20261023073257.0

package/dist/es/ai-model/service-caller/openai/siwc/refresh.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/service-caller/openai/siwc/refresh.mjs","sources":["../../../../../../src/ai-model/service-caller/openai/siwc/refresh.ts"],"sourcesContent":["import { type JWTVerifyGetKey, createRemoteJWKSet, jwtVerify } from 'jose';\nimport { tokenResponseError, withSiwcStage } from './errors';\ninterface RefreshableCredentials {\n  client_id?: string;\n  refresh_token?: string;\n  subject?: string;\n  id_token?: string;\n}\n\nexport async function refreshSiwcCredentials<T extends RefreshableCredentials>(\n  saved: T,\n  signal: AbortSignal,\n  dependencies?: { fetch: typeof fetch; keys: JWTVerifyGetKey },\n) {\n  const { client_id: clientId, refresh_token: refreshToken } = saved;\n  if (!clientId || !refreshToken) {\n    throw new Error('Missing client ID or refresh token.');\n  }\n  const issuer = 'https://auth.openai.com';\n  signal.throwIfAborted();\n  const response = await withSiwcStage('Token refresh', async () => {\n    const response = await (dependencies?.fetch ?? fetch)(\n      `${issuer}/api/accounts/oauth/token`,\n      {\n        method: 'POST',\n        headers: { 'Content-Type': 'application/x-www-form-urlencoded' },\n        body: new URLSearchParams({\n          grant_type: 'refresh_token',\n          client_id: clientId,\n          refresh_token: refreshToken,\n          resource: 'https://api.openai.com/v1',\n        }),\n        signal,\n        redirect: 'error',\n      },\n    );\n    if (!response.ok) {\n      throw await tokenResponseError(response);\n    }\n    return response;\n  });\n  const tokens = await withSiwcStage('Refresh response parsing', () =>\n    response.json(),\n  );\n  if (\n    !tokens ||\n    typeof tokens.access_token !== 'string' ||\n    !tokens.access_token.trim() ||\n    typeof tokens.refresh_token !== 'string' ||\n    !tokens.refresh_token.trim() ||\n    typeof tokens.expires_in !== 'number' ||\n    !Number.isFinite(tokens.expires_in) ||\n    tokens.expires_in <= 0 ||\n    typeof tokens.token_type !== 'string' ||\n    tokens.token_type.toLowerCase() !== 'bearer' ||\n    typeof tokens.scope !== 'string'\n  ) {\n    throw new Error('OpenAI returned an invalid refresh response.');\n  }\n  const scopes = tokens.scope.split(/\\s+/);\n  if (!scopes.includes('chatgpt.tokens.use.direct')) {\n    throw new Error('ChatGPT plan usage is no longer authorized.');\n  }\n  if (tokens.id_token !== undefined) {\n    if (typeof tokens.id_token !== 'string' || !tokens.id_token) {\n      throw new Error('Invalid refreshed ID token.');\n    }\n    const keys =\n      dependencies?.keys ??\n      createRemoteJWKSet(new URL(`${issuer}/.well-known/jwks.json`));\n    const { payload } = await withSiwcStage('ID token verification', () =>\n      jwtVerify(tokens.id_token, keys, {\n        issuer,\n        audience: saved.client_id,\n        algorithms: ['RS256'],\n        // Some CI machines have incorrect system clocks. Do not compare ID token\n        // exp/nbf claims against local time when accepting newly returned tokens.\n        // Keep signature, issuer, audience and required-claim validation.\n        clockTolerance: Number.POSITIVE_INFINITY,\n        requiredClaims: ['sub', 'exp', 'iat'],\n      }),\n    );\n    if (payload.sub !== saved.subject) {\n      throw new Error('Refreshed identity does not match the saved account.');\n    }\n  }\n  signal.throwIfAborted();\n  return {\n    ...saved,\n    access_token: tokens.access_token,\n    refresh_token: tokens.refresh_token,\n    id_token: tokens.id_token ?? saved.id_token,\n    scopes,\n    expires_at: Date.now() + tokens.expires_in * 1000,\n  };\n}\n"],"names":["refreshSiwcCredentials","saved","signal","dependencies","clientId","refreshToken","Error","issuer","response","withSiwcStage","fetch","URLSearchParams","tokenResponseError","tokens","Number","scopes","undefined","keys","createRemoteJWKSet","URL","payload","jwtVerify","Date"],"mappings":";;AASO,eAAeA,uBACpBC,KAAQ,EACRC,MAAmB,EACnBC,YAA6D;IAE7D,MAAM,EAAE,WAAWC,QAAQ,EAAE,eAAeC,YAAY,EAAE,GAAGJ;IAC7D,IAAI,CAACG,YAAY,CAACC,cAChB,MAAM,IAAIC,MAAM;IAElB,MAAMC,SAAS;IACfL,OAAO,cAAc;IACrB,MAAMM,WAAW,MAAMC,cAAc,iBAAiB;QACpD,MAAMD,WAAW,MAAOL,AAAAA,CAAAA,cAAc,SAASO,KAAI,EACjD,GAAGH,OAAO,yBAAyB,CAAC,EACpC;YACE,QAAQ;YACR,SAAS;gBAAE,gBAAgB;YAAoC;YAC/D,MAAM,IAAII,gBAAgB;gBACxB,YAAY;gBACZ,WAAWP;gBACX,eAAeC;gBACf,UAAU;YACZ;YACAH;YACA,UAAU;QACZ;QAEF,IAAI,CAACM,SAAS,EAAE,EACd,MAAM,MAAMI,mBAAmBJ;QAEjC,OAAOA;IACT;IACA,MAAMK,SAAS,MAAMJ,cAAc,4BAA4B,IAC7DD,SAAS,IAAI;IAEf,IACE,CAACK,UACD,AAA+B,YAA/B,OAAOA,OAAO,YAAY,IAC1B,CAACA,OAAO,YAAY,CAAC,IAAI,MACzB,AAAgC,YAAhC,OAAOA,OAAO,aAAa,IAC3B,CAACA,OAAO,aAAa,CAAC,IAAI,MAC1B,AAA6B,YAA7B,OAAOA,OAAO,UAAU,IACxB,CAACC,OAAO,QAAQ,CAACD,OAAO,UAAU,KAClCA,OAAO,UAAU,IAAI,KACrB,AAA6B,YAA7B,OAAOA,OAAO,UAAU,IACxBA,AAAoC,aAApCA,OAAO,UAAU,CAAC,WAAW,MAC7B,AAAwB,YAAxB,OAAOA,OAAO,KAAK,EAEnB,MAAM,IAAIP,MAAM;IAElB,MAAMS,SAASF,OAAO,KAAK,CAAC,KAAK,CAAC;IAClC,IAAI,CAACE,OAAO,QAAQ,CAAC,8BACnB,MAAM,IAAIT,MAAM;IAElB,IAAIO,AAAoBG,WAApBH,OAAO,QAAQ,EAAgB;QACjC,IAAI,AAA2B,YAA3B,OAAOA,OAAO,QAAQ,IAAiB,CAACA,OAAO,QAAQ,EACzD,MAAM,IAAIP,MAAM;QAElB,MAAMW,OACJd,cAAc,QACde,mBAAmB,IAAIC,IAAI,GAAGZ,OAAO,sBAAsB,CAAC;QAC9D,MAAM,EAAEa,OAAO,EAAE,GAAG,MAAMX,cAAc,yBAAyB,IAC/DY,UAAUR,OAAO,QAAQ,EAAEI,MAAM;gBAC/BV;gBACA,UAAUN,MAAM,SAAS;gBACzB,YAAY;oBAAC;iBAAQ;gBAIrB,gBAAgBa;gBAChB,gBAAgB;oBAAC;oBAAO;oBAAO;iBAAM;YACvC;QAEF,IAAIM,QAAQ,GAAG,KAAKnB,MAAM,OAAO,EAC/B,MAAM,IAAIK,MAAM;IAEpB;IACAJ,OAAO,cAAc;IACrB,OAAO;QACL,GAAGD,KAAK;QACR,cAAcY,OAAO,YAAY;QACjC,eAAeA,OAAO,aAAa;QACnC,UAAUA,OAAO,QAAQ,IAAIZ,MAAM,QAAQ;QAC3Cc;QACA,YAAYO,KAAK,GAAG,KAAKT,AAAoB,OAApBA,OAAO,UAAU;IAC5C;AACF"}

package/dist/es/ai-model/service-caller/openai/siwc/token-manager.mjs

@@ -0,0 +1,51 @@
+import { getCredentialStorage } from "./credential-storage.mjs";
+import { refreshSiwcCredentials } from "./refresh.mjs";
+function _define_property(obj, key, value) {
+    if (key in obj) Object.defineProperty(obj, key, {
+        value: value,
+        enumerable: true,
+        configurable: true,
+        writable: true
+    });
+    else obj[key] = value;
+    return obj;
+}
+class SiwcTokenManager {
+    async getAccessToken() {
+        const storage = await getCredentialStorage();
+        const record = await storage.read();
+        return record.credentials.access_token;
+    }
+    refreshAccessToken(previousAccessToken) {
+        this.pendingRefresh ??= this.refreshAndSave(previousAccessToken).finally(()=>{
+            this.pendingRefresh = void 0;
+        });
+        return this.pendingRefresh;
+    }
+    async refreshAndSave(previousAccessToken) {
+        const storage = await getCredentialStorage();
+        const record = await storage.read();
+        const saved = record.credentials;
+        if (saved.access_token !== previousAccessToken) return saved.access_token;
+        try {
+            const credentials = await refreshSiwcCredentials(saved, AbortSignal.timeout(30000));
+            await storage.save({
+                ...record,
+                updated_at: new Date().toISOString(),
+                credentials
+            });
+            return credentials.access_token;
+        } catch (cause) {
+            throw new Error(`Unable to refresh SIWC credentials: ${cause instanceof Error ? cause.message : String(cause)} ${storage.reauthorizationHint}`, {
+                cause
+            });
+        }
+    }
+    constructor(){
+        _define_property(this, "pendingRefresh", void 0);
+    }
+}
+const siwcTokenManager = new SiwcTokenManager();
+export { siwcTokenManager };
+
+//# sourceMappingURL=token-manager.mjs.map

package/dist/es/ai-model/service-caller/openai/siwc/token-manager.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/service-caller/openai/siwc/token-manager.mjs","sources":["../../../../../../src/ai-model/service-caller/openai/siwc/token-manager.ts"],"sourcesContent":["import { getCredentialStorage } from './credential-storage';\nimport { refreshSiwcCredentials } from './refresh';\n\nclass SiwcTokenManager {\n  private pendingRefresh?: Promise<string>;\n\n  async getAccessToken(): Promise<string> {\n    const storage = await getCredentialStorage();\n    const record = await storage.read();\n    // Some CI machines have incorrect system clocks. Let the server reject\n    // expired tokens instead of comparing expires_at against local time.\n    return record.credentials.access_token;\n  }\n\n  /** Only refresh operations are shared; ordinary reads remain independent. */\n  refreshAccessToken(previousAccessToken: string): Promise<string> {\n    this.pendingRefresh ??= this.refreshAndSave(previousAccessToken).finally(\n      () => {\n        this.pendingRefresh = undefined;\n      },\n    );\n    return this.pendingRefresh;\n  }\n\n  private async refreshAndSave(previousAccessToken: string): Promise<string> {\n    // Re-read after acquiring the refresh slot: another caller may have saved\n    // a new token since this request first read its credentials.\n    const storage = await getCredentialStorage();\n    const record = await storage.read();\n    const saved = record.credentials;\n    if (saved.access_token !== previousAccessToken) {\n      return saved.access_token;\n    }\n    try {\n      const credentials = await refreshSiwcCredentials(\n        saved,\n        AbortSignal.timeout(30_000),\n      );\n      await storage.save({\n        ...record,\n        updated_at: new Date().toISOString(),\n        credentials,\n      });\n      return credentials.access_token;\n    } catch (cause) {\n      throw new Error(\n        `Unable to refresh SIWC credentials: ${cause instanceof Error ? cause.message : String(cause)} ${storage.reauthorizationHint}`,\n        { cause },\n      );\n    }\n  }\n}\n\nexport const siwcTokenManager = new SiwcTokenManager();\n"],"names":["SiwcTokenManager","storage","getCredentialStorage","record","previousAccessToken","undefined","saved","credentials","refreshSiwcCredentials","AbortSignal","Date","cause","Error","String","siwcTokenManager"],"mappings":";;;;;;;;;;;;AAGA,MAAMA;IAGJ,MAAM,iBAAkC;QACtC,MAAMC,UAAU,MAAMC;QACtB,MAAMC,SAAS,MAAMF,QAAQ,IAAI;QAGjC,OAAOE,OAAO,WAAW,CAAC,YAAY;IACxC;IAGA,mBAAmBC,mBAA2B,EAAmB;QAC/D,IAAI,CAAC,cAAc,KAAK,IAAI,CAAC,cAAc,CAACA,qBAAqB,OAAO,CACtE;YACE,IAAI,CAAC,cAAc,GAAGC;QACxB;QAEF,OAAO,IAAI,CAAC,cAAc;IAC5B;IAEA,MAAc,eAAeD,mBAA2B,EAAmB;QAGzE,MAAMH,UAAU,MAAMC;QACtB,MAAMC,SAAS,MAAMF,QAAQ,IAAI;QACjC,MAAMK,QAAQH,OAAO,WAAW;QAChC,IAAIG,MAAM,YAAY,KAAKF,qBACzB,OAAOE,MAAM,YAAY;QAE3B,IAAI;YACF,MAAMC,cAAc,MAAMC,uBACxBF,OACAG,YAAY,OAAO,CAAC;YAEtB,MAAMR,QAAQ,IAAI,CAAC;gBACjB,GAAGE,MAAM;gBACT,YAAY,IAAIO,OAAO,WAAW;gBAClCH;YACF;YACA,OAAOA,YAAY,YAAY;QACjC,EAAE,OAAOI,OAAO;YACd,MAAM,IAAIC,MACR,CAAC,oCAAoC,EAAED,iBAAiBC,QAAQD,MAAM,OAAO,GAAGE,OAAOF,OAAO,CAAC,EAAEV,QAAQ,mBAAmB,EAAE,EAC9H;gBAAEU;YAAM;QAEZ;IACF;;QA9CA,uBAAQ,kBAAR;;AA+CF;AAEO,MAAMG,mBAAmB,IAAId"}

package/dist/es/ai-model/service-caller/system-one/call-system-one.mjs

@@ -0,0 +1,69 @@
+import { assert } from "@midscene/shared/utils";
+import { systemOneResponseSchema } from "./schema.mjs";
+async function callSystemOne(context, prepared) {
+    const start = Date.now();
+    context.recordEvent?.({
+        type: 'request',
+        request: {
+            url: prepared.url,
+            method: 'POST',
+            body: prepared.body
+        }
+    });
+    try {
+        const response = await fetch(prepared.url, {
+            method: 'POST',
+            headers: prepared.headers,
+            body: prepared.body,
+            signal: context.requestSignal,
+            ...prepared.dispatcher ? {
+                dispatcher: prepared.dispatcher
+            } : {}
+        });
+        const body = await response.text();
+        const http = {
+            status: response.status,
+            ok: response.ok,
+            body
+        };
+        context.recordEvent?.({
+            type: 'http-response',
+            http
+        });
+        if (!response.ok) throw new Error(`Jev request failed (${response.status}): ${body.slice(0, 4000)}`);
+        const result = systemOneResponseSchema.parse(JSON.parse(body));
+        for (const [name, question] of Object.entries(prepared.questions)){
+            assert(result.answers[name]?.type === question.type, `Missing or invalid Jev answer: ${name}`);
+            const answer = result.answers[name];
+            if ('choice' === question.type && 'choice' === answer.type) assert(Object.hasOwn(question.criteria, answer.choice), `Invalid Jev choice: ${name}`);
+        }
+        const input = result.usage?.input_tokens;
+        const output = result.usage?.output_tokens;
+        const final = {
+            content: body,
+            isStreamed: false,
+            rawUsage: null != input && null != output ? {
+                prompt_tokens: input,
+                completion_tokens: output,
+                total_tokens: input + output
+            } : void 0,
+            timeCost: Date.now() - start,
+            requestId: void 0,
+            responseModelName: result.model
+        };
+        context.recordEvent?.({
+            type: 'response',
+            final
+        });
+        return final;
+    } catch (error) {
+        context.recordEvent?.({
+            type: 'error',
+            error: error instanceof Error ? error.message : String(error)
+        });
+        throw error;
+    }
+}
+export { callSystemOne };
+
+//# sourceMappingURL=call-system-one.mjs.map

package/dist/es/ai-model/service-caller/system-one/call-system-one.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/service-caller/system-one/call-system-one.mjs","sources":["../../../../../src/ai-model/service-caller/system-one/call-system-one.ts"],"sourcesContent":["import { assert } from '@midscene/shared/utils';\nimport type { ModelCallContext, ModelCallResult } from '../types';\nimport type { prepareSystemOne } from './prepare-system-one';\nimport { systemOneResponseSchema } from './schema';\n\nexport async function callSystemOne(\n  context: ModelCallContext,\n  prepared: Awaited<ReturnType<typeof prepareSystemOne>>,\n): Promise<ModelCallResult> {\n  const start = Date.now();\n  context.recordEvent?.({\n    type: 'request',\n    request: { url: prepared.url, method: 'POST', body: prepared.body },\n  });\n  try {\n    const response = await fetch(prepared.url, {\n      method: 'POST',\n      headers: prepared.headers,\n      body: prepared.body,\n      signal: context.requestSignal,\n      ...(prepared.dispatcher ? { dispatcher: prepared.dispatcher } : {}),\n    });\n    const body = await response.text();\n    const http = { status: response.status, ok: response.ok, body };\n    context.recordEvent?.({ type: 'http-response', http });\n    if (!response.ok) {\n      throw new Error(\n        `Jev request failed (${response.status}): ${body.slice(0, 4000)}`,\n      );\n    }\n    const result = systemOneResponseSchema.parse(JSON.parse(body));\n    for (const [name, question] of Object.entries(prepared.questions)) {\n      assert(\n        result.answers[name]?.type === question.type,\n        `Missing or invalid Jev answer: ${name}`,\n      );\n      const answer = result.answers[name];\n      if (question.type === 'choice' && answer.type === 'choice') {\n        assert(\n          Object.hasOwn(question.criteria, answer.choice),\n          `Invalid Jev choice: ${name}`,\n        );\n      }\n    }\n    const input = result.usage?.input_tokens;\n    const output = result.usage?.output_tokens;\n    const final: ModelCallResult = {\n      content: body,\n      isStreamed: false,\n      rawUsage:\n        input != null && output != null\n          ? {\n              prompt_tokens: input,\n              completion_tokens: output,\n              total_tokens: input + output,\n            }\n          : undefined,\n      timeCost: Date.now() - start,\n      // TypeSafe and OpenRouter use different ID field names; defer compatibility handling.\n      requestId: undefined,\n      responseModelName: result.model,\n    };\n    context.recordEvent?.({ type: 'response', final });\n    return final;\n  } catch (error) {\n    context.recordEvent?.({\n      type: 'error',\n      error: error instanceof Error ? error.message : String(error),\n    });\n    throw error;\n  }\n}\n"],"names":["callSystemOne","context","prepared","start","Date","response","fetch","body","http","Error","result","systemOneResponseSchema","JSON","name","question","Object","assert","answer","input","output","final","undefined","error","String"],"mappings":";;AAKO,eAAeA,cACpBC,OAAyB,EACzBC,QAAsD;IAEtD,MAAMC,QAAQC,KAAK,GAAG;IACtBH,QAAQ,WAAW,GAAG;QACpB,MAAM;QACN,SAAS;YAAE,KAAKC,SAAS,GAAG;YAAE,QAAQ;YAAQ,MAAMA,SAAS,IAAI;QAAC;IACpE;IACA,IAAI;QACF,MAAMG,WAAW,MAAMC,MAAMJ,SAAS,GAAG,EAAE;YACzC,QAAQ;YACR,SAASA,SAAS,OAAO;YACzB,MAAMA,SAAS,IAAI;YACnB,QAAQD,QAAQ,aAAa;YAC7B,GAAIC,SAAS,UAAU,GAAG;gBAAE,YAAYA,SAAS,UAAU;YAAC,IAAI,CAAC,CAAC;QACpE;QACA,MAAMK,OAAO,MAAMF,SAAS,IAAI;QAChC,MAAMG,OAAO;YAAE,QAAQH,SAAS,MAAM;YAAE,IAAIA,SAAS,EAAE;YAAEE;QAAK;QAC9DN,QAAQ,WAAW,GAAG;YAAE,MAAM;YAAiBO;QAAK;QACpD,IAAI,CAACH,SAAS,EAAE,EACd,MAAM,IAAII,MACR,CAAC,oBAAoB,EAAEJ,SAAS,MAAM,CAAC,GAAG,EAAEE,KAAK,KAAK,CAAC,GAAG,OAAO;QAGrE,MAAMG,SAASC,wBAAwB,KAAK,CAACC,KAAK,KAAK,CAACL;QACxD,KAAK,MAAM,CAACM,MAAMC,SAAS,IAAIC,OAAO,OAAO,CAACb,SAAS,SAAS,EAAG;YACjEc,OACEN,OAAO,OAAO,CAACG,KAAK,EAAE,SAASC,SAAS,IAAI,EAC5C,CAAC,+BAA+B,EAAED,MAAM;YAE1C,MAAMI,SAASP,OAAO,OAAO,CAACG,KAAK;YACnC,IAAIC,AAAkB,aAAlBA,SAAS,IAAI,IAAiBG,AAAgB,aAAhBA,OAAO,IAAI,EAC3CD,OACED,OAAO,MAAM,CAACD,SAAS,QAAQ,EAAEG,OAAO,MAAM,GAC9C,CAAC,oBAAoB,EAAEJ,MAAM;QAGnC;QACA,MAAMK,QAAQR,OAAO,KAAK,EAAE;QAC5B,MAAMS,SAAST,OAAO,KAAK,EAAE;QAC7B,MAAMU,QAAyB;YAC7B,SAASb;YACT,YAAY;YACZ,UACEW,AAAS,QAATA,SAAiBC,AAAU,QAAVA,SACb;gBACE,eAAeD;gBACf,mBAAmBC;gBACnB,cAAcD,QAAQC;YACxB,IACAE;YACN,UAAUjB,KAAK,GAAG,KAAKD;YAEvB,WAAWkB;YACX,mBAAmBX,OAAO,KAAK;QACjC;QACAT,QAAQ,WAAW,GAAG;YAAE,MAAM;YAAYmB;QAAM;QAChD,OAAOA;IACT,EAAE,OAAOE,OAAO;QACdrB,QAAQ,WAAW,GAAG;YACpB,MAAM;YACN,OAAOqB,iBAAiBb,QAAQa,MAAM,OAAO,GAAGC,OAAOD;QACzD;QACA,MAAMA;IACR;AACF"}

package/dist/es/ai-model/service-caller/system-one/index.mjs

@@ -0,0 +1,3 @@
+import { callSystemOne } from "./call-system-one.mjs";
+import { prepareSystemOne } from "./prepare-system-one.mjs";
+export { callSystemOne, prepareSystemOne };

package/dist/es/ai-model/service-caller/system-one/prepare-system-one.mjs

@@ -0,0 +1,44 @@
+import { assert } from "@midscene/shared/utils";
+import { createProxyAgentIfNeeded } from "../proxy/index.mjs";
+import { encodeSystemOneImages } from "./utils.mjs";
+async function prepareSystemOne(config, request) {
+    assert(config.openaiBaseURL, 'Jev requires MIDSCENE_*_MODEL_BASE_URL');
+    assert(config.openaiApiKey, 'Jev requires MIDSCENE_*_MODEL_API_KEY');
+    assert(!config.protocol || 'openai-chat' === config.protocol, 'Jev uses System One, not the Responses API');
+    const base = config.openaiBaseURL.replace(/\/+$/, '');
+    const url = `${base}/systemone`;
+    const isOpenRouter = 'openrouter.ai' === new URL(base).hostname;
+    const { imageState, assets } = encodeSystemOneImages(request.images, isOpenRouter);
+    const body = {
+        ...config.extraBody,
+        model: config.modelName,
+        state: {
+            ...request.state,
+            ...imageState
+        },
+        questions: request.questions,
+        ...!isOpenRouter && Object.keys(assets).length ? {
+            assets
+        } : {}
+    };
+    const serializedBody = JSON.stringify(body);
+    assert(new TextEncoder().encode(serializedBody).length <= 33554432, 'Jev request exceeds the 32 MiB body limit');
+    return {
+        url,
+        body: serializedBody,
+        questions: request.questions,
+        headers: {
+            ...config.openaiExtraConfig?.defaultHeaders,
+            Authorization: `Bearer ${config.openaiApiKey}`,
+            'Content-Type': 'application/json',
+            'TypeSafe-Danger-Zone-Images': 'true'
+        },
+        dispatcher: await createProxyAgentIfNeeded({
+            socksProxy: config.socksProxy,
+            httpProxy: config.httpProxy
+        })
+    };
+}
+export { prepareSystemOne };
+
+//# sourceMappingURL=prepare-system-one.mjs.map

package/dist/es/ai-model/service-caller/system-one/prepare-system-one.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/service-caller/system-one/prepare-system-one.mjs","sources":["../../../../../src/ai-model/service-caller/system-one/prepare-system-one.ts"],"sourcesContent":["import type { IModelConfig } from '@midscene/shared/env';\nimport { assert } from '@midscene/shared/utils';\nimport { createProxyAgentIfNeeded } from '../proxy';\nimport type { SystemOneRequest } from './schema';\nimport { encodeSystemOneImages } from './utils';\n\nexport async function prepareSystemOne(\n  config: IModelConfig,\n  request: SystemOneRequest,\n) {\n  assert(config.openaiBaseURL, 'Jev requires MIDSCENE_*_MODEL_BASE_URL');\n  assert(config.openaiApiKey, 'Jev requires MIDSCENE_*_MODEL_API_KEY');\n  assert(\n    !config.protocol || config.protocol === 'openai-chat',\n    'Jev uses System One, not the Responses API',\n  );\n  const base = config.openaiBaseURL.replace(/\\/+$/, '');\n  const url = `${base}/systemone`;\n  const isOpenRouter = new URL(base).hostname === 'openrouter.ai';\n  const { imageState, assets } = encodeSystemOneImages(\n    request.images,\n    isOpenRouter,\n  );\n  const body = {\n    ...config.extraBody,\n    model: config.modelName,\n    state: { ...request.state, ...imageState },\n    questions: request.questions,\n    ...(!isOpenRouter && Object.keys(assets).length ? { assets } : {}),\n  };\n  const serializedBody = JSON.stringify(body);\n  assert(\n    new TextEncoder().encode(serializedBody).length <= 32 * 1024 * 1024,\n    'Jev request exceeds the 32 MiB body limit',\n  );\n  return {\n    url,\n    body: serializedBody,\n    questions: request.questions,\n    headers: {\n      ...(config.openaiExtraConfig?.defaultHeaders as\n        | Record<string, string>\n        | undefined),\n      Authorization: `Bearer ${config.openaiApiKey}`,\n      'Content-Type': 'application/json',\n      'TypeSafe-Danger-Zone-Images': 'true',\n    },\n    dispatcher: await createProxyAgentIfNeeded({\n      socksProxy: config.socksProxy,\n      httpProxy: config.httpProxy,\n    }),\n  };\n}\n"],"names":["prepareSystemOne","config","request","assert","base","url","isOpenRouter","URL","imageState","assets","encodeSystemOneImages","body","Object","serializedBody","JSON","TextEncoder","createProxyAgentIfNeeded"],"mappings":";;;AAMO,eAAeA,iBACpBC,MAAoB,EACpBC,OAAyB;IAEzBC,OAAOF,OAAO,aAAa,EAAE;IAC7BE,OAAOF,OAAO,YAAY,EAAE;IAC5BE,OACE,CAACF,OAAO,QAAQ,IAAIA,AAAoB,kBAApBA,OAAO,QAAQ,EACnC;IAEF,MAAMG,OAAOH,OAAO,aAAa,CAAC,OAAO,CAAC,QAAQ;IAClD,MAAMI,MAAM,GAAGD,KAAK,UAAU,CAAC;IAC/B,MAAME,eAAe,AAA2B,oBAA3B,IAAIC,IAAIH,MAAM,QAAQ;IAC3C,MAAM,EAAEI,UAAU,EAAEC,MAAM,EAAE,GAAGC,sBAC7BR,QAAQ,MAAM,EACdI;IAEF,MAAMK,OAAO;QACX,GAAGV,OAAO,SAAS;QACnB,OAAOA,OAAO,SAAS;QACvB,OAAO;YAAE,GAAGC,QAAQ,KAAK;YAAE,GAAGM,UAAU;QAAC;QACzC,WAAWN,QAAQ,SAAS;QAC5B,GAAI,CAACI,gBAAgBM,OAAO,IAAI,CAACH,QAAQ,MAAM,GAAG;YAAEA;QAAO,IAAI,CAAC,CAAC;IACnE;IACA,MAAMI,iBAAiBC,KAAK,SAAS,CAACH;IACtCR,OACE,IAAIY,cAAc,MAAM,CAACF,gBAAgB,MAAM,IAAI,UACnD;IAEF,OAAO;QACLR;QACA,MAAMQ;QACN,WAAWX,QAAQ,SAAS;QAC5B,SAAS;YACP,GAAID,OAAO,iBAAiB,EAAE,cAAc;YAG5C,eAAe,CAAC,OAAO,EAAEA,OAAO,YAAY,EAAE;YAC9C,gBAAgB;YAChB,+BAA+B;QACjC;QACA,YAAY,MAAMe,yBAAyB;YACzC,YAAYf,OAAO,UAAU;YAC7B,WAAWA,OAAO,SAAS;QAC7B;IACF;AACF"}

package/dist/es/ai-model/service-caller/system-one/schema.mjs

@@ -0,0 +1,32 @@
+import { z } from "zod";
+const probability = z.number().min(0).max(1);
+const systemOneAnswerSchema = z.discriminatedUnion('type', [
+    z.object({
+        type: z.literal('noul'),
+        noul: probability
+    }),
+    z.object({
+        type: z.literal('choice'),
+        choice: z.string()
+    }),
+    z.object({
+        type: z.literal('box'),
+        box: z.object({
+            left: probability,
+            right: probability,
+            top: probability,
+            bottom: probability
+        }).refine((box)=>box.left < box.right && box.top < box.bottom, 'Invalid box edges')
+    })
+]);
+const systemOneResponseSchema = z.object({
+    model: z.string().optional(),
+    answers: z.record(systemOneAnswerSchema),
+    usage: z.object({
+        input_tokens: z.number().nonnegative().nullable().optional(),
+        output_tokens: z.number().nonnegative().nullable().optional()
+    }).optional()
+});
+export { systemOneAnswerSchema, systemOneResponseSchema };
+
+//# sourceMappingURL=schema.mjs.map

package/dist/es/ai-model/service-caller/system-one/schema.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/service-caller/system-one/schema.mjs","sources":["../../../../../src/ai-model/service-caller/system-one/schema.ts"],"sourcesContent":["import { z } from 'zod';\n\nexport interface SystemOneRequest {\n  state: Record<string, unknown>;\n  images?: Record<string, string>;\n  questions: Record<\n    string,\n    | { type: 'noul' | 'box'; instructions: unknown }\n    | {\n        type: 'choice';\n        instructions: unknown;\n        criteria: Record<string, string>;\n      }\n  >;\n}\n\nconst probability = z.number().min(0).max(1);\nexport const systemOneAnswerSchema = z.discriminatedUnion('type', [\n  z.object({ type: z.literal('noul'), noul: probability }),\n  z.object({ type: z.literal('choice'), choice: z.string() }),\n  z.object({\n    type: z.literal('box'),\n    box: z\n      .object({\n        left: probability,\n        right: probability,\n        top: probability,\n        bottom: probability,\n      })\n      .refine(\n        (box) => box.left < box.right && box.top < box.bottom,\n        'Invalid box edges',\n      ),\n  }),\n]);\nexport const systemOneResponseSchema = z.object({\n  model: z.string().optional(),\n  answers: z.record(systemOneAnswerSchema),\n  usage: z\n    .object({\n      input_tokens: z.number().nonnegative().nullable().optional(),\n      output_tokens: z.number().nonnegative().nullable().optional(),\n    })\n    .optional(),\n});\n"],"names":["probability","z","systemOneAnswerSchema","box","systemOneResponseSchema"],"mappings":";AAgBA,MAAMA,cAAcC,EAAE,MAAM,GAAG,GAAG,CAAC,GAAG,GAAG,CAAC;AACnC,MAAMC,wBAAwBD,EAAE,kBAAkB,CAAC,QAAQ;IAChEA,EAAE,MAAM,CAAC;QAAE,MAAMA,EAAE,OAAO,CAAC;QAAS,MAAMD;IAAY;IACtDC,EAAE,MAAM,CAAC;QAAE,MAAMA,EAAE,OAAO,CAAC;QAAW,QAAQA,EAAE,MAAM;IAAG;IACzDA,EAAE,MAAM,CAAC;QACP,MAAMA,EAAE,OAAO,CAAC;QAChB,KAAKA,EAAAA,MACI,CAAC;YACN,MAAMD;YACN,OAAOA;YACP,KAAKA;YACL,QAAQA;QACV,GACC,MAAM,CACL,CAACG,MAAQA,IAAI,IAAI,GAAGA,IAAI,KAAK,IAAIA,IAAI,GAAG,GAAGA,IAAI,MAAM,EACrD;IAEN;CACD;AACM,MAAMC,0BAA0BH,EAAE,MAAM,CAAC;IAC9C,OAAOA,EAAE,MAAM,GAAG,QAAQ;IAC1B,SAASA,EAAE,MAAM,CAACC;IAClB,OAAOD,EAAAA,MACE,CAAC;QACN,cAAcA,EAAE,MAAM,GAAG,WAAW,GAAG,QAAQ,GAAG,QAAQ;QAC1D,eAAeA,EAAE,MAAM,GAAG,WAAW,GAAG,QAAQ,GAAG,QAAQ;IAC7D,GACC,QAAQ;AACb"}

package/dist/es/ai-model/service-caller/system-one/utils.mjs

@@ -0,0 +1,34 @@
+import { assert, uuid } from "@midscene/shared/utils";
+function encodeSystemOneImages(images, isOpenRouter) {
+    const assets = {};
+    const imageState = Object.fromEntries(Object.entries(images ?? {}).map(([name, dataURL])=>{
+        if (isOpenRouter) return [
+            name,
+            {
+                type: 'image_url',
+                image_url: {
+                    url: dataURL
+                }
+            }
+        ];
+        const reference = uuid();
+        const match = /^data:image\/(png|jpe?g);base64,(.+)$/.exec(dataURL);
+        assert(match, 'System One images must be prepared PNG or JPEG data URLs');
+        assets[reference] = {
+            type: 'image',
+            format: 'png' === match[1] ? 'png' : 'jpg',
+            data_base64: match[2]
+        };
+        return [
+            name,
+            reference
+        ];
+    }));
+    return {
+        imageState,
+        assets
+    };
+}
+export { encodeSystemOneImages };
+
+//# sourceMappingURL=utils.mjs.map

package/dist/es/ai-model/service-caller/system-one/utils.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/service-caller/system-one/utils.mjs","sources":["../../../../../src/ai-model/service-caller/system-one/utils.ts"],"sourcesContent":["import { assert, uuid } from '@midscene/shared/utils';\nimport type { SystemOneRequest } from './schema';\n\nexport function encodeSystemOneImages(\n  images: SystemOneRequest['images'],\n  isOpenRouter: boolean,\n) {\n  const assets: Record<string, unknown> = {};\n  const imageState = Object.fromEntries(\n    Object.entries(images ?? {}).map(([name, dataURL]) => {\n      if (isOpenRouter) {\n        return [name, { type: 'image_url', image_url: { url: dataURL } }];\n      }\n      const reference = uuid();\n      const match = /^data:image\\/(png|jpe?g);base64,(.+)$/.exec(dataURL);\n      assert(match, 'System One images must be prepared PNG or JPEG data URLs');\n      assets[reference] = {\n        type: 'image',\n        format: match[1] === 'png' ? 'png' : 'jpg',\n        data_base64: match[2],\n      };\n      return [name, reference];\n    }),\n  );\n  return { imageState, assets };\n}\n"],"names":["encodeSystemOneImages","images","isOpenRouter","assets","imageState","Object","name","dataURL","reference","uuid","match","assert"],"mappings":";AAGO,SAASA,sBACdC,MAAkC,EAClCC,YAAqB;IAErB,MAAMC,SAAkC,CAAC;IACzC,MAAMC,aAAaC,OAAO,WAAW,CACnCA,OAAO,OAAO,CAACJ,UAAU,CAAC,GAAG,GAAG,CAAC,CAAC,CAACK,MAAMC,QAAQ;QAC/C,IAAIL,cACF,OAAO;YAACI;YAAM;gBAAE,MAAM;gBAAa,WAAW;oBAAE,KAAKC;gBAAQ;YAAE;SAAE;QAEnE,MAAMC,YAAYC;QAClB,MAAMC,QAAQ,wCAAwC,IAAI,CAACH;QAC3DI,OAAOD,OAAO;QACdP,MAAM,CAACK,UAAU,GAAG;YAClB,MAAM;YACN,QAAQE,AAAa,UAAbA,KAAK,CAAC,EAAE,GAAa,QAAQ;YACrC,aAAaA,KAAK,CAAC,EAAE;QACvB;QACA,OAAO;YAACJ;YAAME;SAAU;IAC1B;IAEF,OAAO;QAAEJ;QAAYD;IAAO;AAC9B"}

package/dist/es/ai-model/service-caller/utils.mjs

@@ -53,12 +53,12 @@ function appendAIRequestFailureSummary(error, attemptErrors, maxAttempts) {
     error.message = `${error.message}\nPrevious AI call attempt errors:\n${details}`;
     return error;
 }
-const buildUsageInfo = ({ apiProtocol, stream, usageData, requestId, timeCost, totalTimeCost, retryCount, modelName, modelDescription, responseModelName, slot, internalCallId })=>{
+const buildUsageInfo = ({ requestKind, stream, usageData, requestId, timeCost, totalTimeCost, retryCount, modelName, modelDescription, responseModelName, slot, internalCallId })=>{
     if (!usageData) return;
     const cachedInputTokens = usageData?.prompt_tokens_details?.cached_tokens;
     return {
         ...usageData,
-        api_protocol: apiProtocol,
+        request_kind: requestKind,
         stream,
         prompt_tokens: usageData.prompt_tokens ?? 0,
         completion_tokens: usageData.completion_tokens ?? 0,

package/dist/es/ai-model/service-caller/utils.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/service-caller/utils.mjs","sources":["../../../../src/ai-model/service-caller/utils.ts"],"sourcesContent":["import type { RawAssistantOutput } from '@/types';\nimport type { AIUsageInfo } from '@/types';\nimport type OpenAI from 'openai';\nimport type { ModelRuntime } from '../models';\n\n// Error class that preserves usage and rawResponse when AI call parsing fails\nexport class AIResponseParseError extends Error {\n  usage?: AIUsageInfo;\n  /**\n   * Adapter-extracted content used by Midscene for parsing. This is not the\n   * full provider response or choices[0].message.\n   */\n  rawResponse: string;\n  rawAssistantOutput?: RawAssistantOutput;\n  reasoningContent?: string;\n\n  constructor(\n    message: string,\n    rawResponse: string,\n    usage?: AIUsageInfo,\n    rawAssistantOutput?: RawAssistantOutput,\n    reasoningContent?: string,\n  ) {\n    super(message);\n    this.name = 'AIResponseParseError';\n    this.rawResponse = rawResponse;\n    this.usage = usage;\n    this.rawAssistantOutput = rawAssistantOutput;\n    this.reasoningContent = reasoningContent;\n  }\n}\n\n/**\n * Internal field name stamped onto every AIUsageInfo shaped by callAI().\n * Used for cross-path dedup when the provider does not return a request_id.\n */\nexport const INTERNAL_CALL_ID_FIELD = '_midscene_call_id';\n\nlet internalCallIdCounter = 0;\nexport function nextInternalCallId(): string {\n  internalCallIdCounter += 1;\n  return `call_${internalCallIdCounter}`;\n}\n\nexport function stringifyForDebug(value: unknown): string {\n  try {\n    return JSON.stringify(value);\n  } catch (_error) {\n    return String(value);\n  }\n}\n\nfunction getErrorMessage(error: unknown): string {\n  return error instanceof Error ? error.message : String(error);\n}\n\nexport function toError(error: unknown): Error {\n  return error instanceof Error ? error : new Error(String(error));\n}\n\nexport function normalizeRetryCount(retryCount: unknown): number {\n  if (typeof retryCount !== 'number' || !Number.isFinite(retryCount)) {\n    return 1;\n  }\n\n  return Math.max(0, Math.floor(retryCount));\n}\n\nexport function appendAIRequestFailureSummary<T extends Error>(\n  error: T,\n  attemptErrors: Array<{ attempt: number; error: unknown }>,\n  maxAttempts: number,\n): T {\n  const failedAttempts = attemptErrors.length;\n  const retries = Math.max(0, failedAttempts - 1);\n  const retryLabel = retries === 1 ? 'retry' : 'retries';\n  const originalMessage = error.message;\n  const previousAttemptErrors = attemptErrors.slice(0, -1);\n\n  error.message = `AI model request failed after ${retries} ${retryLabel} (${failedAttempts}/${maxAttempts} attempts). Last error: ${originalMessage}`;\n\n  if (previousAttemptErrors.length === 0) {\n    return error;\n  }\n\n  const details = previousAttemptErrors\n    .map(\n      ({ attempt, error }) => `Attempt ${attempt}: ${getErrorMessage(error)}`,\n    )\n    .join('\\n');\n\n  error.message = `${error.message}\\nPrevious AI call attempt errors:\\n${details}`;\n  return error;\n}\n\nexport const buildUsageInfo = ({\n  apiProtocol,\n  stream,\n  usageData,\n  requestId,\n  timeCost,\n  totalTimeCost,\n  retryCount,\n  modelName,\n  modelDescription,\n  responseModelName,\n  slot,\n  internalCallId,\n}: {\n  apiProtocol: NonNullable<AIUsageInfo['api_protocol']>;\n  stream: boolean;\n  usageData?: OpenAI.CompletionUsage;\n  requestId?: string | null;\n  timeCost?: number;\n  totalTimeCost: number;\n  retryCount: number;\n  modelName: string;\n  modelDescription: string;\n  responseModelName?: string;\n  slot: ModelRuntime['config']['slot'];\n  internalCallId: string;\n}): AIUsageInfo | undefined => {\n  if (!usageData) return undefined;\n\n  const cachedInputTokens = (\n    usageData as { prompt_tokens_details?: { cached_tokens?: number } }\n  )?.prompt_tokens_details?.cached_tokens;\n\n  return {\n    ...usageData,\n    api_protocol: apiProtocol,\n    stream,\n    prompt_tokens: usageData.prompt_tokens ?? 0,\n    completion_tokens: usageData.completion_tokens ?? 0,\n    total_tokens: usageData.total_tokens ?? 0,\n    cached_input: cachedInputTokens ?? 0,\n    time_cost: timeCost ?? 0,\n    total_time_cost: totalTimeCost,\n    retry_count: retryCount,\n    model_name: modelName,\n    model_description: modelDescription,\n    response_model_name: responseModelName,\n    slot,\n    // Left undefined at the raw call layer. The agent's onUsage callback\n    // fills it from modelConfig.slot for metrics collection, and task\n    // layers use withUsageIntent() to stamp a more specific semantic\n    // intent (e.g. 'planning', 'insight') when attaching usage to tasks.\n    intent: undefined,\n    request_id: requestId ?? undefined,\n    // Internal stable ID for cross-path dedup when request_id is absent.\n    [INTERNAL_CALL_ID_FIELD]: internalCallId,\n  } satisfies AIUsageInfo;\n};\n\nexport const hasUsableText = (\n  value: string | null | undefined,\n): value is string => typeof value === 'string' && value.trim().length > 0;\n"],"names":["AIResponseParseError","Error","message","rawResponse","usage","rawAssistantOutput","reasoningContent","INTERNAL_CALL_ID_FIELD","internalCallIdCounter","nextInternalCallId","stringifyForDebug","value","JSON","_error","String","getErrorMessage","error","toError","normalizeRetryCount","retryCount","Number","Math","appendAIRequestFailureSummary","attemptErrors","maxAttempts","failedAttempts","retries","retryLabel","originalMessage","previousAttemptErrors","details","attempt","buildUsageInfo","apiProtocol","stream","usageData","requestId","timeCost","totalTimeCost","modelName","modelDescription","responseModelName","slot","internalCallId","cachedInputTokens","undefined","hasUsableText"],"mappings":";;;;;;;;;;AAMO,MAAMA,6BAA6BC;IAUxC,YACEC,OAAe,EACfC,WAAmB,EACnBC,KAAmB,EACnBC,kBAAuC,EACvCC,gBAAyB,CACzB;QACA,KAAK,CAACJ,UAhBR,yCAKA,+CACA,sDACA;QAUE,IAAI,CAAC,IAAI,GAAG;QACZ,IAAI,CAAC,WAAW,GAAGC;QACnB,IAAI,CAAC,KAAK,GAAGC;QACb,IAAI,CAAC,kBAAkB,GAAGC;QAC1B,IAAI,CAAC,gBAAgB,GAAGC;IAC1B;AACF;AAMO,MAAMC,yBAAyB;AAEtC,IAAIC,wBAAwB;AACrB,SAASC;IACdD,yBAAyB;IACzB,OAAO,CAAC,KAAK,EAAEA,uBAAuB;AACxC;AAEO,SAASE,kBAAkBC,KAAc;IAC9C,IAAI;QACF,OAAOC,KAAK,SAAS,CAACD;IACxB,EAAE,OAAOE,QAAQ;QACf,OAAOC,OAAOH;IAChB;AACF;AAEA,SAASI,gBAAgBC,KAAc;IACrC,OAAOA,iBAAiBf,QAAQe,MAAM,OAAO,GAAGF,OAAOE;AACzD;AAEO,SAASC,QAAQD,KAAc;IACpC,OAAOA,iBAAiBf,QAAQe,QAAQ,IAAIf,MAAMa,OAAOE;AAC3D;AAEO,SAASE,oBAAoBC,UAAmB;IACrD,IAAI,AAAsB,YAAtB,OAAOA,cAA2B,CAACC,OAAO,QAAQ,CAACD,aACrD,OAAO;IAGT,OAAOE,KAAK,GAAG,CAAC,GAAGA,KAAK,KAAK,CAACF;AAChC;AAEO,SAASG,8BACdN,KAAQ,EACRO,aAAyD,EACzDC,WAAmB;IAEnB,MAAMC,iBAAiBF,cAAc,MAAM;IAC3C,MAAMG,UAAUL,KAAK,GAAG,CAAC,GAAGI,iBAAiB;IAC7C,MAAME,aAAaD,AAAY,MAAZA,UAAgB,UAAU;IAC7C,MAAME,kBAAkBZ,MAAM,OAAO;IACrC,MAAMa,wBAAwBN,cAAc,KAAK,CAAC,GAAG;IAErDP,MAAM,OAAO,GAAG,CAAC,8BAA8B,EAAEU,QAAQ,CAAC,EAAEC,WAAW,EAAE,EAAEF,eAAe,CAAC,EAAED,YAAY,wBAAwB,EAAEI,iBAAiB;IAEpJ,IAAIC,AAAiC,MAAjCA,sBAAsB,MAAM,EAC9B,OAAOb;IAGT,MAAMc,UAAUD,sBACb,GAAG,CACF,CAAC,EAAEE,OAAO,EAAEf,KAAK,EAAE,GAAK,CAAC,QAAQ,EAAEe,QAAQ,EAAE,EAAEhB,gBAAgBC,QAAQ,EAExE,IAAI,CAAC;IAERA,MAAM,OAAO,GAAG,GAAGA,MAAM,OAAO,CAAC,oCAAoC,EAAEc,SAAS;IAChF,OAAOd;AACT;AAEO,MAAMgB,iBAAiB,CAAC,EAC7BC,WAAW,EACXC,MAAM,EACNC,SAAS,EACTC,SAAS,EACTC,QAAQ,EACRC,aAAa,EACbnB,UAAU,EACVoB,SAAS,EACTC,gBAAgB,EAChBC,iBAAiB,EACjBC,IAAI,EACJC,cAAc,EAcf;IACC,IAAI,CAACR,WAAW;IAEhB,MAAMS,oBACJT,WACC,uBAAuB;IAE1B,OAAO;QACL,GAAGA,SAAS;QACZ,cAAcF;QACdC;QACA,eAAeC,UAAU,aAAa,IAAI;QAC1C,mBAAmBA,UAAU,iBAAiB,IAAI;QAClD,cAAcA,UAAU,YAAY,IAAI;QACxC,cAAcS,qBAAqB;QACnC,WAAWP,YAAY;QACvB,iBAAiBC;QACjB,aAAanB;QACb,YAAYoB;QACZ,mBAAmBC;QACnB,qBAAqBC;QACrBC;QAKA,QAAQG;QACR,YAAYT,aAAaS;QAEzB,CAACtC,uBAAuB,EAAEoC;IAC5B;AACF;AAEO,MAAMG,gBAAgB,CAC3BnC,QACoB,AAAiB,YAAjB,OAAOA,SAAsBA,MAAM,IAAI,GAAG,MAAM,GAAG"}
+{"version":3,"file":"ai-model/service-caller/utils.mjs","sources":["../../../../src/ai-model/service-caller/utils.ts"],"sourcesContent":["import type { RawAssistantOutput } from '@/types';\nimport type { AIUsageInfo } from '@/types';\nimport type OpenAI from 'openai';\nimport type { ModelRuntime } from '../models';\n\n// Error class that preserves usage and rawResponse when AI call parsing fails\nexport class AIResponseParseError extends Error {\n  usage?: AIUsageInfo;\n  /**\n   * Adapter-extracted content used by Midscene for parsing. This is not the\n   * full provider response or choices[0].message.\n   */\n  rawResponse: string;\n  rawAssistantOutput?: RawAssistantOutput;\n  reasoningContent?: string;\n\n  constructor(\n    message: string,\n    rawResponse: string,\n    usage?: AIUsageInfo,\n    rawAssistantOutput?: RawAssistantOutput,\n    reasoningContent?: string,\n  ) {\n    super(message);\n    this.name = 'AIResponseParseError';\n    this.rawResponse = rawResponse;\n    this.usage = usage;\n    this.rawAssistantOutput = rawAssistantOutput;\n    this.reasoningContent = reasoningContent;\n  }\n}\n\n/**\n * Internal field name stamped onto AIUsageInfo by the shared model call pipeline.\n * Used for cross-path dedup when the provider does not return a request_id.\n */\nexport const INTERNAL_CALL_ID_FIELD = '_midscene_call_id';\n\nlet internalCallIdCounter = 0;\nexport function nextInternalCallId(): string {\n  internalCallIdCounter += 1;\n  return `call_${internalCallIdCounter}`;\n}\n\nexport function stringifyForDebug(value: unknown): string {\n  try {\n    return JSON.stringify(value);\n  } catch (_error) {\n    return String(value);\n  }\n}\n\nfunction getErrorMessage(error: unknown): string {\n  return error instanceof Error ? error.message : String(error);\n}\n\nexport function toError(error: unknown): Error {\n  return error instanceof Error ? error : new Error(String(error));\n}\n\nexport function normalizeRetryCount(retryCount: unknown): number {\n  if (typeof retryCount !== 'number' || !Number.isFinite(retryCount)) {\n    return 1;\n  }\n\n  return Math.max(0, Math.floor(retryCount));\n}\n\nexport function appendAIRequestFailureSummary<T extends Error>(\n  error: T,\n  attemptErrors: Array<{ attempt: number; error: unknown }>,\n  maxAttempts: number,\n): T {\n  const failedAttempts = attemptErrors.length;\n  const retries = Math.max(0, failedAttempts - 1);\n  const retryLabel = retries === 1 ? 'retry' : 'retries';\n  const originalMessage = error.message;\n  const previousAttemptErrors = attemptErrors.slice(0, -1);\n\n  error.message = `AI model request failed after ${retries} ${retryLabel} (${failedAttempts}/${maxAttempts} attempts). Last error: ${originalMessage}`;\n\n  if (previousAttemptErrors.length === 0) {\n    return error;\n  }\n\n  const details = previousAttemptErrors\n    .map(\n      ({ attempt, error }) => `Attempt ${attempt}: ${getErrorMessage(error)}`,\n    )\n    .join('\\n');\n\n  error.message = `${error.message}\\nPrevious AI call attempt errors:\\n${details}`;\n  return error;\n}\n\nexport const buildUsageInfo = ({\n  requestKind,\n  stream,\n  usageData,\n  requestId,\n  timeCost,\n  totalTimeCost,\n  retryCount,\n  modelName,\n  modelDescription,\n  responseModelName,\n  slot,\n  internalCallId,\n}: {\n  requestKind: NonNullable<AIUsageInfo['request_kind']>;\n  stream: boolean;\n  usageData?: OpenAI.CompletionUsage;\n  requestId?: string | null;\n  timeCost?: number;\n  totalTimeCost: number;\n  retryCount: number;\n  modelName: string;\n  modelDescription: string;\n  responseModelName?: string;\n  slot: ModelRuntime['config']['slot'];\n  internalCallId: string;\n}): AIUsageInfo | undefined => {\n  if (!usageData) return undefined;\n\n  const cachedInputTokens = (\n    usageData as { prompt_tokens_details?: { cached_tokens?: number } }\n  )?.prompt_tokens_details?.cached_tokens;\n\n  return {\n    ...usageData,\n    request_kind: requestKind,\n    stream,\n    prompt_tokens: usageData.prompt_tokens ?? 0,\n    completion_tokens: usageData.completion_tokens ?? 0,\n    total_tokens: usageData.total_tokens ?? 0,\n    cached_input: cachedInputTokens ?? 0,\n    time_cost: timeCost ?? 0,\n    total_time_cost: totalTimeCost,\n    retry_count: retryCount,\n    model_name: modelName,\n    model_description: modelDescription,\n    response_model_name: responseModelName,\n    slot,\n    // Left undefined at the raw call layer. The agent's onUsage callback\n    // fills it from modelConfig.slot for metrics collection, and task\n    // layers use withUsageIntent() to stamp a more specific semantic\n    // intent (e.g. 'planning', 'insight') when attaching usage to tasks.\n    intent: undefined,\n    request_id: requestId ?? undefined,\n    // Internal stable ID for cross-path dedup when request_id is absent.\n    [INTERNAL_CALL_ID_FIELD]: internalCallId,\n  } satisfies AIUsageInfo;\n};\n\nexport const hasUsableText = (\n  value: string | null | undefined,\n): value is string => typeof value === 'string' && value.trim().length > 0;\n"],"names":["AIResponseParseError","Error","message","rawResponse","usage","rawAssistantOutput","reasoningContent","INTERNAL_CALL_ID_FIELD","internalCallIdCounter","nextInternalCallId","stringifyForDebug","value","JSON","_error","String","getErrorMessage","error","toError","normalizeRetryCount","retryCount","Number","Math","appendAIRequestFailureSummary","attemptErrors","maxAttempts","failedAttempts","retries","retryLabel","originalMessage","previousAttemptErrors","details","attempt","buildUsageInfo","requestKind","stream","usageData","requestId","timeCost","totalTimeCost","modelName","modelDescription","responseModelName","slot","internalCallId","cachedInputTokens","undefined","hasUsableText"],"mappings":";;;;;;;;;;AAMO,MAAMA,6BAA6BC;IAUxC,YACEC,OAAe,EACfC,WAAmB,EACnBC,KAAmB,EACnBC,kBAAuC,EACvCC,gBAAyB,CACzB;QACA,KAAK,CAACJ,UAhBR,yCAKA,+CACA,sDACA;QAUE,IAAI,CAAC,IAAI,GAAG;QACZ,IAAI,CAAC,WAAW,GAAGC;QACnB,IAAI,CAAC,KAAK,GAAGC;QACb,IAAI,CAAC,kBAAkB,GAAGC;QAC1B,IAAI,CAAC,gBAAgB,GAAGC;IAC1B;AACF;AAMO,MAAMC,yBAAyB;AAEtC,IAAIC,wBAAwB;AACrB,SAASC;IACdD,yBAAyB;IACzB,OAAO,CAAC,KAAK,EAAEA,uBAAuB;AACxC;AAEO,SAASE,kBAAkBC,KAAc;IAC9C,IAAI;QACF,OAAOC,KAAK,SAAS,CAACD;IACxB,EAAE,OAAOE,QAAQ;QACf,OAAOC,OAAOH;IAChB;AACF;AAEA,SAASI,gBAAgBC,KAAc;IACrC,OAAOA,iBAAiBf,QAAQe,MAAM,OAAO,GAAGF,OAAOE;AACzD;AAEO,SAASC,QAAQD,KAAc;IACpC,OAAOA,iBAAiBf,QAAQe,QAAQ,IAAIf,MAAMa,OAAOE;AAC3D;AAEO,SAASE,oBAAoBC,UAAmB;IACrD,IAAI,AAAsB,YAAtB,OAAOA,cAA2B,CAACC,OAAO,QAAQ,CAACD,aACrD,OAAO;IAGT,OAAOE,KAAK,GAAG,CAAC,GAAGA,KAAK,KAAK,CAACF;AAChC;AAEO,SAASG,8BACdN,KAAQ,EACRO,aAAyD,EACzDC,WAAmB;IAEnB,MAAMC,iBAAiBF,cAAc,MAAM;IAC3C,MAAMG,UAAUL,KAAK,GAAG,CAAC,GAAGI,iBAAiB;IAC7C,MAAME,aAAaD,AAAY,MAAZA,UAAgB,UAAU;IAC7C,MAAME,kBAAkBZ,MAAM,OAAO;IACrC,MAAMa,wBAAwBN,cAAc,KAAK,CAAC,GAAG;IAErDP,MAAM,OAAO,GAAG,CAAC,8BAA8B,EAAEU,QAAQ,CAAC,EAAEC,WAAW,EAAE,EAAEF,eAAe,CAAC,EAAED,YAAY,wBAAwB,EAAEI,iBAAiB;IAEpJ,IAAIC,AAAiC,MAAjCA,sBAAsB,MAAM,EAC9B,OAAOb;IAGT,MAAMc,UAAUD,sBACb,GAAG,CACF,CAAC,EAAEE,OAAO,EAAEf,KAAK,EAAE,GAAK,CAAC,QAAQ,EAAEe,QAAQ,EAAE,EAAEhB,gBAAgBC,QAAQ,EAExE,IAAI,CAAC;IAERA,MAAM,OAAO,GAAG,GAAGA,MAAM,OAAO,CAAC,oCAAoC,EAAEc,SAAS;IAChF,OAAOd;AACT;AAEO,MAAMgB,iBAAiB,CAAC,EAC7BC,WAAW,EACXC,MAAM,EACNC,SAAS,EACTC,SAAS,EACTC,QAAQ,EACRC,aAAa,EACbnB,UAAU,EACVoB,SAAS,EACTC,gBAAgB,EAChBC,iBAAiB,EACjBC,IAAI,EACJC,cAAc,EAcf;IACC,IAAI,CAACR,WAAW;IAEhB,MAAMS,oBACJT,WACC,uBAAuB;IAE1B,OAAO;QACL,GAAGA,SAAS;QACZ,cAAcF;QACdC;QACA,eAAeC,UAAU,aAAa,IAAI;QAC1C,mBAAmBA,UAAU,iBAAiB,IAAI;QAClD,cAAcA,UAAU,YAAY,IAAI;QACxC,cAAcS,qBAAqB;QACnC,WAAWP,YAAY;QACvB,iBAAiBC;QACjB,aAAanB;QACb,YAAYoB;QACZ,mBAAmBC;QACnB,qBAAqBC;QACrBC;QAKA,QAAQG;QACR,YAAYT,aAAaS;QAEzB,CAACtC,uBAAuB,EAAEoC;IAC5B;AACF;AAEO,MAAMG,gBAAgB,CAC3BnC,QACoB,AAAiB,YAAjB,OAAOA,SAAsBA,MAAM,IAAI,GAAG,MAAM,GAAG"}

package/dist/es/ai-model/shared/model-locate-result/factory.mjs

@@ -6,6 +6,9 @@ import { assertLocateResultCoordinateRangeAndOrder, assertLocateResultStructure
 function resolveLocateResultCoordinates(coordinates) {
     if (void 0 !== coordinates.normalizedBy && coordinates.normalizedBy <= 0) throw new Error(`locate result coordinates normalizedBy must be positive: ${coordinates.normalizedBy}`);
     return {
+        ...'bbox' === coordinates.shape ? {
+            boundsType: coordinates.boundsType ?? 'inclusive'
+        } : {},
         shape: coordinates.shape,
         order: coordinates.order ?? 'xy',
         normalizedBy: coordinates.normalizedBy,
@@ -24,11 +27,12 @@ function createLocateResultCodec(config) {
             const result = parseRawLocateValue(rawResult);
             const { preparedSize, contentSize = preparedSize } = context;
             assertLocateResultStructure(result);
-            assertLocateResultCoordinateRangeAndOrder(result, preparedSize.width - 1, preparedSize.height - 1);
+            const edgeBounds = isBboxLocateResultValue(result) && 'edges' === result.coordinatesMeta.boundsType;
+            assertLocateResultCoordinateRangeAndOrder(result, preparedSize.width - (edgeBounds ? 0 : 1), preparedSize.height - (edgeBounds ? 0 : 1));
             const center = clampPixelPointToSize(mapLocateResultToPixelPoint(result, preparedSize), contentSize);
             const rect = isBboxLocateResultValue(result) ? (()=>{
-                const pixelBbox = clampPixelBboxToSize(mapLocateResultToPixelBbox(result, preparedSize), contentSize);
-                return pixelBboxToRect(pixelBbox);
+                const pixelBbox = clampPixelBboxToSize(mapLocateResultToPixelBbox(result, preparedSize), contentSize, result.coordinatesMeta.boundsType);
+                return pixelBboxToRect(pixelBbox, result.coordinatesMeta.boundsType);
             })() : void 0;
             return {
                 center,

package/dist/es/ai-model/shared/model-locate-result/factory.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/shared/model-locate-result/factory.mjs","sources":["../../../../../src/ai-model/shared/model-locate-result/factory.ts"],"sourcesContent":["import { parseNumericLocateResult } from './parse';\nimport {\n  clampPixelBboxToSize,\n  clampPixelPointToSize,\n  mapLocateResultToPixelBbox,\n  mapLocateResultToPixelPoint,\n  pixelBboxToRect,\n} from './pixel-mapper';\nimport { createLocateResultPromptSpec } from './prompt-spec';\nimport { isBboxLocateResultValue } from './types';\nimport type {\n  LocateResultCodec,\n  LocateResultCoordinates,\n  LocateResultFormatDefinition,\n  ResolvedLocateResultCoordinates,\n} from './types';\nimport {\n  assertLocateResultCoordinateRangeAndOrder,\n  assertLocateResultStructure,\n} from './validation';\n\nexport function resolveLocateResultCoordinates(\n  coordinates: LocateResultCoordinates,\n): ResolvedLocateResultCoordinates {\n  if (coordinates.normalizedBy !== undefined && coordinates.normalizedBy <= 0) {\n    throw new Error(\n      `locate result coordinates normalizedBy must be positive: ${coordinates.normalizedBy}`,\n    );\n  }\n  return {\n    shape: coordinates.shape,\n    order: coordinates.order ?? 'xy',\n    normalizedBy: coordinates.normalizedBy,\n    rounding: coordinates.rounding ?? 'round',\n  };\n}\n\nexport function createLocateResultCodec(\n  config: LocateResultFormatDefinition,\n): LocateResultCodec {\n  const resolvedCoordinates = resolveLocateResultCoordinates(\n    config.coordinates,\n  );\n  const parseRawLocateValue =\n    config.parseRawLocateValue ??\n    ((input) => parseNumericLocateResult(resolvedCoordinates, input));\n  return {\n    promptSpec: createLocateResultPromptSpec(resolvedCoordinates, {\n      resultKey: config.resultKey,\n      resultKeyAliases: config.resultKeyAliases,\n    }),\n    toPixelResult: (rawResult, context) => {\n      const result = parseRawLocateValue(rawResult);\n      const { preparedSize, contentSize = preparedSize } = context;\n      assertLocateResultStructure(result);\n      assertLocateResultCoordinateRangeAndOrder(\n        result,\n        preparedSize.width - 1,\n        preparedSize.height - 1,\n      );\n      // Clamp the mapped center independently from the optional bbox so padding\n      // removal cannot change the target by recomputing a clipped box's center.\n      const center = clampPixelPointToSize(\n        mapLocateResultToPixelPoint(result, preparedSize),\n        contentSize,\n      );\n      // Preserve an original bbox as region metadata for DeepLocate search areas\n      // and aiLocate's return value. Point-only results leave rect undefined;\n      // click coordinates always use the independently calculated center.\n      const rect = isBboxLocateResultValue(result)\n        ? (() => {\n            const pixelBbox = clampPixelBboxToSize(\n              mapLocateResultToPixelBbox(result, preparedSize),\n              contentSize,\n            );\n            return pixelBboxToRect(pixelBbox);\n          })()\n        : undefined;\n      return { center, rect };\n    },\n  };\n}\n"],"names":["resolveLocateResultCoordinates","coordinates","undefined","Error","createLocateResultCodec","config","resolvedCoordinates","parseRawLocateValue","input","parseNumericLocateResult","createLocateResultPromptSpec","rawResult","context","result","preparedSize","contentSize","assertLocateResultStructure","assertLocateResultCoordinateRangeAndOrder","center","clampPixelPointToSize","mapLocateResultToPixelPoint","rect","isBboxLocateResultValue","pixelBbox","clampPixelBboxToSize","mapLocateResultToPixelBbox","pixelBboxToRect"],"mappings":";;;;;AAqBO,SAASA,+BACdC,WAAoC;IAEpC,IAAIA,AAA6BC,WAA7BD,YAAY,YAAY,IAAkBA,YAAY,YAAY,IAAI,GACxE,MAAM,IAAIE,MACR,CAAC,yDAAyD,EAAEF,YAAY,YAAY,EAAE;IAG1F,OAAO;QACL,OAAOA,YAAY,KAAK;QACxB,OAAOA,YAAY,KAAK,IAAI;QAC5B,cAAcA,YAAY,YAAY;QACtC,UAAUA,YAAY,QAAQ,IAAI;IACpC;AACF;AAEO,SAASG,wBACdC,MAAoC;IAEpC,MAAMC,sBAAsBN,+BAC1BK,OAAO,WAAW;IAEpB,MAAME,sBACJF,OAAO,mBAAmB,IACxB,EAAAG,QAAUC,yBAAyBH,qBAAqBE,MAAK;IACjE,OAAO;QACL,YAAYE,6BAA6BJ,qBAAqB;YAC5D,WAAWD,OAAO,SAAS;YAC3B,kBAAkBA,OAAO,gBAAgB;QAC3C;QACA,eAAe,CAACM,WAAWC;YACzB,MAAMC,SAASN,oBAAoBI;YACnC,MAAM,EAAEG,YAAY,EAAEC,cAAcD,YAAY,EAAE,GAAGF;YACrDI,4BAA4BH;YAC5BI,0CACEJ,QACAC,aAAa,KAAK,GAAG,GACrBA,aAAa,MAAM,GAAG;YAIxB,MAAMI,SAASC,sBACbC,4BAA4BP,QAAQC,eACpCC;YAKF,MAAMM,OAAOC,wBAAwBT,UAChC,AAAC;gBACA,MAAMU,YAAYC,qBAChBC,2BAA2BZ,QAAQC,eACnCC;gBAEF,OAAOW,gBAAgBH;YACzB,OACArB;YACJ,OAAO;gBAAEgB;gBAAQG;YAAK;QACxB;IACF;AACF"}
+{"version":3,"file":"ai-model/shared/model-locate-result/factory.mjs","sources":["../../../../../src/ai-model/shared/model-locate-result/factory.ts"],"sourcesContent":["import { parseNumericLocateResult } from './parse';\nimport {\n  clampPixelBboxToSize,\n  clampPixelPointToSize,\n  mapLocateResultToPixelBbox,\n  mapLocateResultToPixelPoint,\n  pixelBboxToRect,\n} from './pixel-mapper';\nimport { createLocateResultPromptSpec } from './prompt-spec';\nimport { isBboxLocateResultValue } from './types';\nimport type {\n  LocateResultCodec,\n  LocateResultCoordinates,\n  LocateResultFormatDefinition,\n  ResolvedLocateResultCoordinates,\n} from './types';\nimport {\n  assertLocateResultCoordinateRangeAndOrder,\n  assertLocateResultStructure,\n} from './validation';\n\nexport function resolveLocateResultCoordinates(\n  coordinates: LocateResultCoordinates,\n): ResolvedLocateResultCoordinates {\n  if (coordinates.normalizedBy !== undefined && coordinates.normalizedBy <= 0) {\n    throw new Error(\n      `locate result coordinates normalizedBy must be positive: ${coordinates.normalizedBy}`,\n    );\n  }\n  return {\n    ...(coordinates.shape === 'bbox'\n      ? { boundsType: coordinates.boundsType ?? 'inclusive' }\n      : {}),\n    shape: coordinates.shape,\n    order: coordinates.order ?? 'xy',\n    normalizedBy: coordinates.normalizedBy,\n    rounding: coordinates.rounding ?? 'round',\n  };\n}\n\nexport function createLocateResultCodec(\n  config: LocateResultFormatDefinition,\n): LocateResultCodec {\n  const resolvedCoordinates = resolveLocateResultCoordinates(\n    config.coordinates,\n  );\n  const parseRawLocateValue =\n    config.parseRawLocateValue ??\n    ((input) => parseNumericLocateResult(resolvedCoordinates, input));\n  return {\n    promptSpec: createLocateResultPromptSpec(resolvedCoordinates, {\n      resultKey: config.resultKey,\n      resultKeyAliases: config.resultKeyAliases,\n    }),\n    toPixelResult: (rawResult, context) => {\n      const result = parseRawLocateValue(rawResult);\n      const { preparedSize, contentSize = preparedSize } = context;\n      assertLocateResultStructure(result);\n      const edgeBounds =\n        isBboxLocateResultValue(result) &&\n        result.coordinatesMeta.boundsType === 'edges';\n      assertLocateResultCoordinateRangeAndOrder(\n        result,\n        preparedSize.width - (edgeBounds ? 0 : 1),\n        preparedSize.height - (edgeBounds ? 0 : 1),\n      );\n      // Clamp the mapped center independently from the optional bbox so padding\n      // removal cannot change the target by recomputing a clipped box's center.\n      const center = clampPixelPointToSize(\n        mapLocateResultToPixelPoint(result, preparedSize),\n        contentSize,\n      );\n      // Preserve an original bbox as region metadata for DeepLocate search areas\n      // and aiLocate's return value. Point-only results leave rect undefined;\n      // click coordinates always use the independently calculated center.\n      const rect = isBboxLocateResultValue(result)\n        ? (() => {\n            const pixelBbox = clampPixelBboxToSize(\n              mapLocateResultToPixelBbox(result, preparedSize),\n              contentSize,\n              result.coordinatesMeta.boundsType,\n            );\n            return pixelBboxToRect(\n              pixelBbox,\n              result.coordinatesMeta.boundsType,\n            );\n          })()\n        : undefined;\n      return { center, rect };\n    },\n  };\n}\n"],"names":["resolveLocateResultCoordinates","coordinates","undefined","Error","createLocateResultCodec","config","resolvedCoordinates","parseRawLocateValue","input","parseNumericLocateResult","createLocateResultPromptSpec","rawResult","context","result","preparedSize","contentSize","assertLocateResultStructure","edgeBounds","isBboxLocateResultValue","assertLocateResultCoordinateRangeAndOrder","center","clampPixelPointToSize","mapLocateResultToPixelPoint","rect","pixelBbox","clampPixelBboxToSize","mapLocateResultToPixelBbox","pixelBboxToRect"],"mappings":";;;;;AAqBO,SAASA,+BACdC,WAAoC;IAEpC,IAAIA,AAA6BC,WAA7BD,YAAY,YAAY,IAAkBA,YAAY,YAAY,IAAI,GACxE,MAAM,IAAIE,MACR,CAAC,yDAAyD,EAAEF,YAAY,YAAY,EAAE;IAG1F,OAAO;QACL,GAAIA,AAAsB,WAAtBA,YAAY,KAAK,GACjB;YAAE,YAAYA,YAAY,UAAU,IAAI;QAAY,IACpD,CAAC,CAAC;QACN,OAAOA,YAAY,KAAK;QACxB,OAAOA,YAAY,KAAK,IAAI;QAC5B,cAAcA,YAAY,YAAY;QACtC,UAAUA,YAAY,QAAQ,IAAI;IACpC;AACF;AAEO,SAASG,wBACdC,MAAoC;IAEpC,MAAMC,sBAAsBN,+BAC1BK,OAAO,WAAW;IAEpB,MAAME,sBACJF,OAAO,mBAAmB,IACxB,EAAAG,QAAUC,yBAAyBH,qBAAqBE,MAAK;IACjE,OAAO;QACL,YAAYE,6BAA6BJ,qBAAqB;YAC5D,WAAWD,OAAO,SAAS;YAC3B,kBAAkBA,OAAO,gBAAgB;QAC3C;QACA,eAAe,CAACM,WAAWC;YACzB,MAAMC,SAASN,oBAAoBI;YACnC,MAAM,EAAEG,YAAY,EAAEC,cAAcD,YAAY,EAAE,GAAGF;YACrDI,4BAA4BH;YAC5B,MAAMI,aACJC,wBAAwBL,WACxBA,AAAsC,YAAtCA,OAAO,eAAe,CAAC,UAAU;YACnCM,0CACEN,QACAC,aAAa,KAAK,GAAIG,CAAAA,aAAa,IAAI,IACvCH,aAAa,MAAM,GAAIG,CAAAA,aAAa,IAAI;YAI1C,MAAMG,SAASC,sBACbC,4BAA4BT,QAAQC,eACpCC;YAKF,MAAMQ,OAAOL,wBAAwBL,UAChC,AAAC;gBACA,MAAMW,YAAYC,qBAChBC,2BAA2Bb,QAAQC,eACnCC,aACAF,OAAO,eAAe,CAAC,UAAU;gBAEnC,OAAOc,gBACLH,WACAX,OAAO,eAAe,CAAC,UAAU;YAErC,OACAX;YACJ,OAAO;gBAAEkB;gBAAQG;YAAK;QACxB;IACF;AACF"}

package/dist/es/ai-model/shared/model-locate-result/pixel-mapper.mjs

@@ -24,8 +24,14 @@ function clampPixelPointToSize([x, y], { width, height }) {
         clamp(y, 0, maxPixelCoordinate(height))
     ];
 }
-function clampPixelBboxToSize(pixelBbox, size) {
+function clampPixelBboxToSize(pixelBbox, size, boundsType = 'inclusive') {
     const [left, top, right, bottom] = pixelBbox;
+    if ('edges' === boundsType) return [
+        clamp(left, 0, size.width),
+        clamp(top, 0, size.height),
+        clamp(right, 0, size.width),
+        clamp(bottom, 0, size.height)
+    ];
     return [
         ...clampPixelPointToSize([
             left,
@@ -37,12 +43,12 @@ function clampPixelBboxToSize(pixelBbox, size) {
         ], size)
     ];
 }
-function pixelBboxToRect([left, top, right, bottom]) {
+function pixelBboxToRect([left, top, right, bottom], boundsType = 'inclusive') {
     return {
         left,
         top,
-        width: right - left + 1,
-        height: bottom - top + 1
+        width: right - left + ('inclusive' === boundsType ? 1 : 0),
+        height: bottom - top + ('inclusive' === boundsType ? 1 : 0)
     };
 }
 function mapLocateResultToPixelBbox(result, { width, height }) {

package/dist/es/ai-model/shared/model-locate-result/pixel-mapper.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/shared/model-locate-result/pixel-mapper.mjs","sources":["../../../../../src/ai-model/shared/model-locate-result/pixel-mapper.ts"],"sourcesContent":["import type { Rect } from '@/types';\nimport { isBboxLocateResultValue } from './types';\nimport type {\n  BboxLocateResultValue,\n  CoordinateRounding,\n  LocateResultPoint,\n  LocateResultValue,\n  PixelBbox,\n} from './types';\n\nfunction clamp(value: number, min: number, max: number) {\n  return Math.min(Math.max(value, min), max);\n}\n\nexport function maxPixelCoordinate(size: number) {\n  return Math.max(size - 1, 0);\n}\n\nexport function normalizedCoordinateToPixel(\n  value: number,\n  normalizedBy: number,\n  size: number,\n  rounding: CoordinateRounding,\n) {\n  const pixel = (value * size) / normalizedBy;\n  switch (rounding) {\n    case 'round':\n      return Math.round(pixel);\n    case 'trunc':\n      return Math.trunc(pixel);\n    case 'none':\n      return pixel;\n    default:\n      throw new Error(`Invalid coordinate rounding: ${rounding}`);\n  }\n}\n\n/** Restrict an already mapped point to the effective content area. */\nexport function clampPixelPointToSize(\n  [x, y]: LocateResultPoint,\n  { width, height }: { width: number; height: number },\n): LocateResultPoint {\n  return [\n    clamp(x, 0, maxPixelCoordinate(width)),\n    clamp(y, 0, maxPixelCoordinate(height)),\n  ];\n}\n\n/** Restrict an already mapped bbox to the effective content area. */\nexport function clampPixelBboxToSize(\n  pixelBbox: PixelBbox,\n  size: { width: number; height: number },\n): PixelBbox {\n  const [left, top, right, bottom] = pixelBbox;\n  return [\n    ...clampPixelPointToSize([left, top], size),\n    ...clampPixelPointToSize([right, bottom], size),\n  ];\n}\n\nexport function pixelBboxToRect([left, top, right, bottom]: PixelBbox): Rect {\n  return {\n    left,\n    top,\n    width: right - left + 1,\n    height: bottom - top + 1,\n  };\n}\n\nexport function mapLocateResultToPixelBbox(\n  result: BboxLocateResultValue,\n  { width, height }: { width: number; height: number },\n): PixelBbox {\n  const { normalizedBy, order, rounding } = result.coordinatesMeta;\n  const [first, second, third, fourth] = result.coordinates;\n  const xyBbox: PixelBbox =\n    order === 'yx' ? [second, first, fourth, third] : result.coordinates;\n  return normalizedBy === undefined\n    ? xyBbox\n    : [\n        normalizedCoordinateToPixel(xyBbox[0], normalizedBy, width, rounding),\n        normalizedCoordinateToPixel(xyBbox[1], normalizedBy, height, rounding),\n        normalizedCoordinateToPixel(xyBbox[2], normalizedBy, width, rounding),\n        normalizedCoordinateToPixel(xyBbox[3], normalizedBy, height, rounding),\n      ];\n}\n\n/** Resolve the target before any rounding or clipping can shift its center. */\nexport function mapLocateResultToPixelPoint(\n  result: LocateResultValue,\n  { width, height }: { width: number; height: number },\n): LocateResultPoint {\n  const { order, normalizedBy, rounding } = result.coordinatesMeta;\n  const [first, second] = isBboxLocateResultValue(result)\n    ? [\n        (result.coordinates[0] + result.coordinates[2]) / 2,\n        (result.coordinates[1] + result.coordinates[3]) / 2,\n      ]\n    : result.coordinates;\n  const [x, y] = order === 'yx' ? [second, first] : [first, second];\n  return normalizedBy === undefined\n    ? [x, y]\n    : [\n        normalizedCoordinateToPixel(x, normalizedBy, width, rounding),\n        normalizedCoordinateToPixel(y, normalizedBy, height, rounding),\n      ];\n}\n"],"names":["clamp","value","min","max","Math","maxPixelCoordinate","size","normalizedCoordinateToPixel","normalizedBy","rounding","pixel","Error","clampPixelPointToSize","x","y","width","height","clampPixelBboxToSize","pixelBbox","left","top","right","bottom","pixelBboxToRect","mapLocateResultToPixelBbox","result","order","first","second","third","fourth","xyBbox","undefined","mapLocateResultToPixelPoint","isBboxLocateResultValue"],"mappings":";AAUA,SAASA,MAAMC,KAAa,EAAEC,GAAW,EAAEC,GAAW;IACpD,OAAOC,KAAK,GAAG,CAACA,KAAK,GAAG,CAACH,OAAOC,MAAMC;AACxC;AAEO,SAASE,mBAAmBC,IAAY;IAC7C,OAAOF,KAAK,GAAG,CAACE,OAAO,GAAG;AAC5B;AAEO,SAASC,4BACdN,KAAa,EACbO,YAAoB,EACpBF,IAAY,EACZG,QAA4B;IAE5B,MAAMC,QAAST,QAAQK,OAAQE;IAC/B,OAAQC;QACN,KAAK;YACH,OAAOL,KAAK,KAAK,CAACM;QACpB,KAAK;YACH,OAAON,KAAK,KAAK,CAACM;QACpB,KAAK;YACH,OAAOA;QACT;YACE,MAAM,IAAIC,MAAM,CAAC,6BAA6B,EAAEF,UAAU;IAC9D;AACF;AAGO,SAASG,sBACd,CAACC,GAAGC,EAAqB,EACzB,EAAEC,KAAK,EAAEC,MAAM,EAAqC;IAEpD,OAAO;QACLhB,MAAMa,GAAG,GAAGR,mBAAmBU;QAC/Bf,MAAMc,GAAG,GAAGT,mBAAmBW;KAChC;AACH;AAGO,SAASC,qBACdC,SAAoB,EACpBZ,IAAuC;IAEvC,MAAM,CAACa,MAAMC,KAAKC,OAAOC,OAAO,GAAGJ;IACnC,OAAO;WACFN,sBAAsB;YAACO;YAAMC;SAAI,EAAEd;WACnCM,sBAAsB;YAACS;YAAOC;SAAO,EAAEhB;KAC3C;AACH;AAEO,SAASiB,gBAAgB,CAACJ,MAAMC,KAAKC,OAAOC,OAAkB;IACnE,OAAO;QACLH;QACAC;QACA,OAAOC,QAAQF,OAAO;QACtB,QAAQG,SAASF,MAAM;IACzB;AACF;AAEO,SAASI,2BACdC,MAA6B,EAC7B,EAAEV,KAAK,EAAEC,MAAM,EAAqC;IAEpD,MAAM,EAAER,YAAY,EAAEkB,KAAK,EAAEjB,QAAQ,EAAE,GAAGgB,OAAO,eAAe;IAChE,MAAM,CAACE,OAAOC,QAAQC,OAAOC,OAAO,GAAGL,OAAO,WAAW;IACzD,MAAMM,SACJL,AAAU,SAAVA,QAAiB;QAACE;QAAQD;QAAOG;QAAQD;KAAM,GAAGJ,OAAO,WAAW;IACtE,OAAOjB,AAAiBwB,WAAjBxB,eACHuB,SACA;QACExB,4BAA4BwB,MAAM,CAAC,EAAE,EAAEvB,cAAcO,OAAON;QAC5DF,4BAA4BwB,MAAM,CAAC,EAAE,EAAEvB,cAAcQ,QAAQP;QAC7DF,4BAA4BwB,MAAM,CAAC,EAAE,EAAEvB,cAAcO,OAAON;QAC5DF,4BAA4BwB,MAAM,CAAC,EAAE,EAAEvB,cAAcQ,QAAQP;KAC9D;AACP;AAGO,SAASwB,4BACdR,MAAyB,EACzB,EAAEV,KAAK,EAAEC,MAAM,EAAqC;IAEpD,MAAM,EAAEU,KAAK,EAAElB,YAAY,EAAEC,QAAQ,EAAE,GAAGgB,OAAO,eAAe;IAChE,MAAM,CAACE,OAAOC,OAAO,GAAGM,wBAAwBT,UAC5C;QACGA,CAAAA,OAAO,WAAW,CAAC,EAAE,GAAGA,OAAO,WAAW,CAAC,EAAC,IAAK;QACjDA,CAAAA,OAAO,WAAW,CAAC,EAAE,GAAGA,OAAO,WAAW,CAAC,EAAC,IAAK;KACnD,GACDA,OAAO,WAAW;IACtB,MAAM,CAACZ,GAAGC,EAAE,GAAGY,AAAU,SAAVA,QAAiB;QAACE;QAAQD;KAAM,GAAG;QAACA;QAAOC;KAAO;IACjE,OAAOpB,AAAiBwB,WAAjBxB,eACH;QAACK;QAAGC;KAAE,GACN;QACEP,4BAA4BM,GAAGL,cAAcO,OAAON;QACpDF,4BAA4BO,GAAGN,cAAcQ,QAAQP;KACtD;AACP"}
+{"version":3,"file":"ai-model/shared/model-locate-result/pixel-mapper.mjs","sources":["../../../../../src/ai-model/shared/model-locate-result/pixel-mapper.ts"],"sourcesContent":["import type { Rect } from '@/types';\nimport { isBboxLocateResultValue } from './types';\nimport type {\n  BboxBoundsType,\n  BboxLocateResultValue,\n  CoordinateRounding,\n  LocateResultPoint,\n  LocateResultValue,\n  PixelBbox,\n} from './types';\n\nfunction clamp(value: number, min: number, max: number) {\n  return Math.min(Math.max(value, min), max);\n}\n\nexport function maxPixelCoordinate(size: number) {\n  return Math.max(size - 1, 0);\n}\n\nexport function normalizedCoordinateToPixel(\n  value: number,\n  normalizedBy: number,\n  size: number,\n  rounding: CoordinateRounding,\n) {\n  const pixel = (value * size) / normalizedBy;\n  switch (rounding) {\n    case 'round':\n      return Math.round(pixel);\n    case 'trunc':\n      return Math.trunc(pixel);\n    case 'none':\n      return pixel;\n    default:\n      throw new Error(`Invalid coordinate rounding: ${rounding}`);\n  }\n}\n\n/** Restrict an already mapped point to the effective content area. */\nexport function clampPixelPointToSize(\n  [x, y]: LocateResultPoint,\n  { width, height }: { width: number; height: number },\n): LocateResultPoint {\n  return [\n    clamp(x, 0, maxPixelCoordinate(width)),\n    clamp(y, 0, maxPixelCoordinate(height)),\n  ];\n}\n\n/** Restrict an already mapped bbox to the effective content area. */\nexport function clampPixelBboxToSize(\n  pixelBbox: PixelBbox,\n  size: { width: number; height: number },\n  boundsType: BboxBoundsType = 'inclusive',\n): PixelBbox {\n  const [left, top, right, bottom] = pixelBbox;\n  if (boundsType === 'edges') {\n    return [\n      clamp(left, 0, size.width),\n      clamp(top, 0, size.height),\n      clamp(right, 0, size.width),\n      clamp(bottom, 0, size.height),\n    ];\n  }\n  return [\n    ...clampPixelPointToSize([left, top], size),\n    ...clampPixelPointToSize([right, bottom], size),\n  ];\n}\n\nexport function pixelBboxToRect(\n  [left, top, right, bottom]: PixelBbox,\n  boundsType: BboxBoundsType = 'inclusive',\n): Rect {\n  return {\n    left,\n    top,\n    width: right - left + (boundsType === 'inclusive' ? 1 : 0),\n    height: bottom - top + (boundsType === 'inclusive' ? 1 : 0),\n  };\n}\n\nexport function mapLocateResultToPixelBbox(\n  result: BboxLocateResultValue,\n  { width, height }: { width: number; height: number },\n): PixelBbox {\n  const { normalizedBy, order, rounding } = result.coordinatesMeta;\n  const [first, second, third, fourth] = result.coordinates;\n  const xyBbox: PixelBbox =\n    order === 'yx' ? [second, first, fourth, third] : result.coordinates;\n  return normalizedBy === undefined\n    ? xyBbox\n    : [\n        normalizedCoordinateToPixel(xyBbox[0], normalizedBy, width, rounding),\n        normalizedCoordinateToPixel(xyBbox[1], normalizedBy, height, rounding),\n        normalizedCoordinateToPixel(xyBbox[2], normalizedBy, width, rounding),\n        normalizedCoordinateToPixel(xyBbox[3], normalizedBy, height, rounding),\n      ];\n}\n\n/** Resolve the target before any rounding or clipping can shift its center. */\nexport function mapLocateResultToPixelPoint(\n  result: LocateResultValue,\n  { width, height }: { width: number; height: number },\n): LocateResultPoint {\n  const { order, normalizedBy, rounding } = result.coordinatesMeta;\n  const [first, second] = isBboxLocateResultValue(result)\n    ? [\n        (result.coordinates[0] + result.coordinates[2]) / 2,\n        (result.coordinates[1] + result.coordinates[3]) / 2,\n      ]\n    : result.coordinates;\n  const [x, y] = order === 'yx' ? [second, first] : [first, second];\n  return normalizedBy === undefined\n    ? [x, y]\n    : [\n        normalizedCoordinateToPixel(x, normalizedBy, width, rounding),\n        normalizedCoordinateToPixel(y, normalizedBy, height, rounding),\n      ];\n}\n"],"names":["clamp","value","min","max","Math","maxPixelCoordinate","size","normalizedCoordinateToPixel","normalizedBy","rounding","pixel","Error","clampPixelPointToSize","x","y","width","height","clampPixelBboxToSize","pixelBbox","boundsType","left","top","right","bottom","pixelBboxToRect","mapLocateResultToPixelBbox","result","order","first","second","third","fourth","xyBbox","undefined","mapLocateResultToPixelPoint","isBboxLocateResultValue"],"mappings":";AAWA,SAASA,MAAMC,KAAa,EAAEC,GAAW,EAAEC,GAAW;IACpD,OAAOC,KAAK,GAAG,CAACA,KAAK,GAAG,CAACH,OAAOC,MAAMC;AACxC;AAEO,SAASE,mBAAmBC,IAAY;IAC7C,OAAOF,KAAK,GAAG,CAACE,OAAO,GAAG;AAC5B;AAEO,SAASC,4BACdN,KAAa,EACbO,YAAoB,EACpBF,IAAY,EACZG,QAA4B;IAE5B,MAAMC,QAAST,QAAQK,OAAQE;IAC/B,OAAQC;QACN,KAAK;YACH,OAAOL,KAAK,KAAK,CAACM;QACpB,KAAK;YACH,OAAON,KAAK,KAAK,CAACM;QACpB,KAAK;YACH,OAAOA;QACT;YACE,MAAM,IAAIC,MAAM,CAAC,6BAA6B,EAAEF,UAAU;IAC9D;AACF;AAGO,SAASG,sBACd,CAACC,GAAGC,EAAqB,EACzB,EAAEC,KAAK,EAAEC,MAAM,EAAqC;IAEpD,OAAO;QACLhB,MAAMa,GAAG,GAAGR,mBAAmBU;QAC/Bf,MAAMc,GAAG,GAAGT,mBAAmBW;KAChC;AACH;AAGO,SAASC,qBACdC,SAAoB,EACpBZ,IAAuC,EACvCa,aAA6B,WAAW;IAExC,MAAM,CAACC,MAAMC,KAAKC,OAAOC,OAAO,GAAGL;IACnC,IAAIC,AAAe,YAAfA,YACF,OAAO;QACLnB,MAAMoB,MAAM,GAAGd,KAAK,KAAK;QACzBN,MAAMqB,KAAK,GAAGf,KAAK,MAAM;QACzBN,MAAMsB,OAAO,GAAGhB,KAAK,KAAK;QAC1BN,MAAMuB,QAAQ,GAAGjB,KAAK,MAAM;KAC7B;IAEH,OAAO;WACFM,sBAAsB;YAACQ;YAAMC;SAAI,EAAEf;WACnCM,sBAAsB;YAACU;YAAOC;SAAO,EAAEjB;KAC3C;AACH;AAEO,SAASkB,gBACd,CAACJ,MAAMC,KAAKC,OAAOC,OAAkB,EACrCJ,aAA6B,WAAW;IAExC,OAAO;QACLC;QACAC;QACA,OAAOC,QAAQF,OAAQD,CAAAA,AAAe,gBAAfA,aAA6B,IAAI;QACxD,QAAQI,SAASF,MAAOF,CAAAA,AAAe,gBAAfA,aAA6B,IAAI;IAC3D;AACF;AAEO,SAASM,2BACdC,MAA6B,EAC7B,EAAEX,KAAK,EAAEC,MAAM,EAAqC;IAEpD,MAAM,EAAER,YAAY,EAAEmB,KAAK,EAAElB,QAAQ,EAAE,GAAGiB,OAAO,eAAe;IAChE,MAAM,CAACE,OAAOC,QAAQC,OAAOC,OAAO,GAAGL,OAAO,WAAW;IACzD,MAAMM,SACJL,AAAU,SAAVA,QAAiB;QAACE;QAAQD;QAAOG;QAAQD;KAAM,GAAGJ,OAAO,WAAW;IACtE,OAAOlB,AAAiByB,WAAjBzB,eACHwB,SACA;QACEzB,4BAA4ByB,MAAM,CAAC,EAAE,EAAExB,cAAcO,OAAON;QAC5DF,4BAA4ByB,MAAM,CAAC,EAAE,EAAExB,cAAcQ,QAAQP;QAC7DF,4BAA4ByB,MAAM,CAAC,EAAE,EAAExB,cAAcO,OAAON;QAC5DF,4BAA4ByB,MAAM,CAAC,EAAE,EAAExB,cAAcQ,QAAQP;KAC9D;AACP;AAGO,SAASyB,4BACdR,MAAyB,EACzB,EAAEX,KAAK,EAAEC,MAAM,EAAqC;IAEpD,MAAM,EAAEW,KAAK,EAAEnB,YAAY,EAAEC,QAAQ,EAAE,GAAGiB,OAAO,eAAe;IAChE,MAAM,CAACE,OAAOC,OAAO,GAAGM,wBAAwBT,UAC5C;QACGA,CAAAA,OAAO,WAAW,CAAC,EAAE,GAAGA,OAAO,WAAW,CAAC,EAAC,IAAK;QACjDA,CAAAA,OAAO,WAAW,CAAC,EAAE,GAAGA,OAAO,WAAW,CAAC,EAAC,IAAK;KACnD,GACDA,OAAO,WAAW;IACtB,MAAM,CAACb,GAAGC,EAAE,GAAGa,AAAU,SAAVA,QAAiB;QAACE;QAAQD;KAAM,GAAG;QAACA;QAAOC;KAAO;IACjE,OAAOrB,AAAiByB,WAAjBzB,eACH;QAACK;QAAGC;KAAE,GACN;QACEP,4BAA4BM,GAAGL,cAAcO,OAAON;QACpDF,4BAA4BO,GAAGN,cAAcQ,QAAQP;KACtD;AACP"}

package/dist/es/ai-model/shared/model-locate-result/types.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/shared/model-locate-result/types.mjs","sources":["../../../../../src/ai-model/shared/model-locate-result/types.ts"],"sourcesContent":["import type { Rect } from '@midscene/shared/types';\nimport type { Bbox } from '../../../types';\n\nexport type { Bbox };\nexport type LocateResultBbox = Bbox;\nexport type PixelBbox = Bbox;\nexport type NonEmptyArray<T> = [T, ...T[]];\nexport type RawLocateValue = unknown;\n\nexport type LocateResultPoint = [number, number];\nexport type PointLocateResultCoordinates = ResolvedLocateResultCoordinates & {\n  shape: 'point';\n};\nexport type BboxLocateResultCoordinates = ResolvedLocateResultCoordinates & {\n  shape: 'bbox';\n};\n\nexport type LocateResultValue =\n  | {\n      coordinates: LocateResultPoint;\n      coordinatesMeta: PointLocateResultCoordinates;\n    }\n  | {\n      coordinates: LocateResultBbox;\n      coordinatesMeta: BboxLocateResultCoordinates;\n    };\n\nexport type PointLocateResultValue = Extract<\n  LocateResultValue,\n  { coordinatesMeta: { shape: 'point' } }\n>;\nexport type BboxLocateResultValue = Extract<\n  LocateResultValue,\n  { coordinatesMeta: { shape: 'bbox' } }\n>;\n\nexport function isBboxLocateResultValue(\n  result: LocateResultValue,\n): result is BboxLocateResultValue {\n  return result.coordinatesMeta.shape === 'bbox';\n}\n\nexport function isPointLocateResultValue(\n  result: LocateResultValue,\n): result is PointLocateResultValue {\n  return result.coordinatesMeta.shape === 'point';\n}\n\nexport type LocateResultShape = 'bbox' | 'point';\n\nexport interface LocateResultContext {\n  preparedSize: {\n    width: number;\n    height: number;\n  };\n  contentSize?: {\n    width: number;\n    height: number;\n  };\n}\n\nexport interface LocateResultPromptSpec {\n  resultKey: string;\n  /** Parse-only alternatives, tried in order after resultKey. */\n  resultKeyAliases?: string[];\n  resultValueSchema: string;\n  resultValueDescription: string;\n  resultNoun: string;\n  resultNounPlural: string;\n  exampleValues: NonEmptyArray<unknown>;\n}\n\nexport interface PixelLocateResult {\n  /** Raw bbox midpoint (or raw point), mapped to the locate image using the configured rounding. */\n  center: LocateResultPoint;\n  /**\n   * Original bbox mapped to locate-image pixels and clipped to content bounds.\n   * Undefined for point results; never derive center or click coordinates from it.\n   */\n  rect?: Rect;\n}\n\nexport interface LocateResultCodec {\n  promptSpec: LocateResultPromptSpec;\n  /** Parse once and independently map the center and optional original bbox. */\n  toPixelResult(\n    input: RawLocateValue,\n    ctx: LocateResultContext,\n  ): PixelLocateResult;\n}\n\nexport type CoordinateRounding = 'round' | 'trunc' | 'none';\n\nexport interface LocateResultCoordinates {\n  shape: LocateResultShape;\n  /** Axis order in the raw coordinates; defaults to xy. */\n  order?: 'xy' | 'yx';\n  /** Normalization range, e.g. 1 or 1000. Omit for model-image pixel coordinates. */\n  normalizedBy?: number;\n  /** Rounding after normalized-to-pixel mapping; defaults to round. Pixel inputs retain their precision. */\n  rounding?: CoordinateRounding;\n}\n\n/** Coordinate metadata with defaults resolved and an explicit rounding policy. */\nexport type ResolvedLocateResultCoordinates =\n  | {\n      shape: 'point';\n      order: 'xy' | 'yx';\n      normalizedBy?: number;\n      rounding: CoordinateRounding;\n    }\n  | {\n      shape: 'bbox';\n      order: 'xy' | 'yx';\n      normalizedBy?: number;\n      rounding: CoordinateRounding;\n    };\n\nexport type RawLocateValueParser = (input: RawLocateValue) => LocateResultValue;\n\n/**\n * Format of one model coordinate value, shared by element and search-area locate.\n * The operation protocol extracts target/reference values from the response;\n * the codec parses each value once and maps it to `{ center, rect }` using the\n * parsed result's coordinatesMeta. Rect is undefined for actual point results.\n * Raw-value parsing and normalized-coordinate rounding are configurable;\n * coordinate mapping is shared.\n */\nexport type LocateResultFormatDefinition = {\n  /** Response field used in prompts and parsing; defaults to bbox or point. */\n  resultKey?: string;\n  /** Parse-only alternatives, tried in order when the primary field is undefined. */\n  resultKeyAliases?: string[];\n  /**\n   * Expected shape, axis order, and normalization used by the prompt and default\n   * parser. A custom parser may return different metadata for an actual response.\n   */\n  coordinates: LocateResultCoordinates;\n  /**\n   * Optional model-specific parser for nonstandard formats or point/bbox fallback.\n   * Return `{ coordinates, coordinatesMeta }` describing the actual shape, axis\n   * order, and normalization. Preserve coordinate precision; do not round or map\n   * to pixels here. Include an explicit rounding policy in `coordinatesMeta`.\n   * Omit the parser to parse numeric values according to `coordinates`.\n   */\n  parseRawLocateValue?: RawLocateValueParser;\n};\n"],"names":["isBboxLocateResultValue","result","isPointLocateResultValue"],"mappings":"AAoCO,SAASA,wBACdC,MAAyB;IAEzB,OAAOA,AAAiC,WAAjCA,OAAO,eAAe,CAAC,KAAK;AACrC;AAEO,SAASC,yBACdD,MAAyB;IAEzB,OAAOA,AAAiC,YAAjCA,OAAO,eAAe,CAAC,KAAK;AACrC"}
+{"version":3,"file":"ai-model/shared/model-locate-result/types.mjs","sources":["../../../../../src/ai-model/shared/model-locate-result/types.ts"],"sourcesContent":["import type { Rect } from '@midscene/shared/types';\nimport type { Bbox } from '../../../types';\n\nexport type { Bbox };\nexport type LocateResultBbox = Bbox;\nexport type PixelBbox = Bbox;\nexport type NonEmptyArray<T> = [T, ...T[]];\nexport type RawLocateValue = unknown;\n\nexport type LocateResultPoint = [number, number];\nexport type PointLocateResultCoordinates = ResolvedLocateResultCoordinates & {\n  shape: 'point';\n};\nexport type BboxLocateResultCoordinates = ResolvedLocateResultCoordinates & {\n  shape: 'bbox';\n};\n\nexport type LocateResultValue =\n  | {\n      coordinates: LocateResultPoint;\n      coordinatesMeta: PointLocateResultCoordinates;\n    }\n  | {\n      coordinates: LocateResultBbox;\n      coordinatesMeta: BboxLocateResultCoordinates;\n    };\n\nexport type PointLocateResultValue = Extract<\n  LocateResultValue,\n  { coordinatesMeta: { shape: 'point' } }\n>;\nexport type BboxLocateResultValue = Extract<\n  LocateResultValue,\n  { coordinatesMeta: { shape: 'bbox' } }\n>;\n\nexport function isBboxLocateResultValue(\n  result: LocateResultValue,\n): result is BboxLocateResultValue {\n  return result.coordinatesMeta.shape === 'bbox';\n}\n\nexport function isPointLocateResultValue(\n  result: LocateResultValue,\n): result is PointLocateResultValue {\n  return result.coordinatesMeta.shape === 'point';\n}\n\nexport type LocateResultShape = 'bbox' | 'point';\n\nexport interface LocateResultContext {\n  preparedSize: {\n    width: number;\n    height: number;\n  };\n  contentSize?: {\n    width: number;\n    height: number;\n  };\n}\n\nexport interface LocateResultPromptSpec {\n  resultKey: string;\n  /** Parse-only alternatives, tried in order after resultKey. */\n  resultKeyAliases?: string[];\n  resultValueSchema: string;\n  resultValueDescription: string;\n  resultNoun: string;\n  resultNounPlural: string;\n  exampleValues: NonEmptyArray<unknown>;\n}\n\nexport interface PixelLocateResult {\n  /** Raw bbox midpoint (or raw point), mapped to the locate image using the configured rounding. */\n  center: LocateResultPoint;\n  /**\n   * Original bbox mapped to locate-image pixels and clipped to content bounds.\n   * Undefined for point results; never derive center or click coordinates from it.\n   */\n  rect?: Rect;\n}\n\nexport interface LocateResultCodec {\n  promptSpec: LocateResultPromptSpec;\n  /** Parse once and independently map the center and optional original bbox. */\n  toPixelResult(\n    input: RawLocateValue,\n    ctx: LocateResultContext,\n  ): PixelLocateResult;\n}\n\nexport type CoordinateRounding = 'round' | 'trunc' | 'none';\n\nexport type BboxBoundsType = 'inclusive' | 'edges';\n\nexport interface LocateResultCoordinates {\n  /** Bbox boundary semantics; defaults to inclusive pixel endpoints. */\n  boundsType?: BboxBoundsType;\n  shape: LocateResultShape;\n  /** Axis order in the raw coordinates; defaults to xy. */\n  order?: 'xy' | 'yx';\n  /** Normalization range, e.g. 1 or 1000. Omit for model-image pixel coordinates. */\n  normalizedBy?: number;\n  /** Rounding after normalized-to-pixel mapping; defaults to round. Pixel inputs retain their precision. */\n  rounding?: CoordinateRounding;\n}\n\n/** Coordinate metadata with defaults resolved and an explicit rounding policy. */\nexport type ResolvedLocateResultCoordinates =\n  | {\n      shape: 'point';\n      order: 'xy' | 'yx';\n      normalizedBy?: number;\n      rounding: CoordinateRounding;\n    }\n  | {\n      shape: 'bbox';\n      boundsType?: BboxBoundsType;\n      order: 'xy' | 'yx';\n      normalizedBy?: number;\n      rounding: CoordinateRounding;\n    };\n\nexport type RawLocateValueParser = (input: RawLocateValue) => LocateResultValue;\n\n/**\n * Format of one model coordinate value, shared by element and search-area locate.\n * The operation protocol extracts target/reference values from the response;\n * the codec parses each value once and maps it to `{ center, rect }` using the\n * parsed result's coordinatesMeta. Rect is undefined for actual point results.\n * Raw-value parsing and normalized-coordinate rounding are configurable;\n * coordinate mapping is shared.\n */\nexport type LocateResultFormatDefinition = {\n  /** Response field used in prompts and parsing; defaults to bbox or point. */\n  resultKey?: string;\n  /** Parse-only alternatives, tried in order when the primary field is undefined. */\n  resultKeyAliases?: string[];\n  /**\n   * Expected shape, axis order, and normalization used by the prompt and default\n   * parser. A custom parser may return different metadata for an actual response.\n   */\n  coordinates: LocateResultCoordinates;\n  /**\n   * Optional model-specific parser for nonstandard formats or point/bbox fallback.\n   * Return `{ coordinates, coordinatesMeta }` describing the actual shape, axis\n   * order, and normalization. Preserve coordinate precision; do not round or map\n   * to pixels here. Include an explicit rounding policy in `coordinatesMeta`.\n   * Omit the parser to parse numeric values according to `coordinates`.\n   */\n  parseRawLocateValue?: RawLocateValueParser;\n};\n"],"names":["isBboxLocateResultValue","result","isPointLocateResultValue"],"mappings":"AAoCO,SAASA,wBACdC,MAAyB;IAEzB,OAAOA,AAAiC,WAAjCA,OAAO,eAAe,CAAC,KAAK;AACrC;AAEO,SAASC,yBACdD,MAAyB;IAEzB,OAAOA,AAAiC,YAAjCA,OAAO,eAAe,CAAC,KAAK;AACrC"}

package/dist/es/ai-model/shared/model-locate-result/validation.mjs

@@ -6,6 +6,10 @@ function assertLocateResultStructure(result) {
     if (!expectedLength) throw new Error(`invalid parsed locate result: unsupported coordinatesMeta.shape ${JSON.stringify(coordinatesMeta?.shape)}`);
     const coordinates = result.coordinates;
     if (!Array.isArray(coordinates) || coordinates.length !== expectedLength || !coordinates.every((value)=>'number' == typeof value && Number.isFinite(value))) throw new Error(`invalid parsed locate result: ${coordinatesMeta.shape} coordinates must be ${expectedLength} finite numbers, got ${JSON.stringify(coordinates)}`);
+    if (isBboxLocateResultValue(result) && void 0 !== result.coordinatesMeta.boundsType && ![
+        'inclusive',
+        'edges'
+    ].includes(result.coordinatesMeta.boundsType)) throw new Error(`invalid locate bbox boundsType: ${result.coordinatesMeta.boundsType}`);
     const { order, normalizedBy, rounding } = coordinatesMeta;
     if (![
         'round',

package/dist/es/ai-model/shared/model-locate-result/validation.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/shared/model-locate-result/validation.mjs","sources":["../../../../../src/ai-model/shared/model-locate-result/validation.ts"],"sourcesContent":["import { isBboxLocateResultValue } from './types';\nimport type { LocateResultValue } from './types';\n\nexport function assertLocateResultStructure(result: LocateResultValue): void {\n  if (!result || typeof result !== 'object') {\n    throw new Error(\n      `invalid parsed locate result: expected object, got ${JSON.stringify(\n        result,\n      )}`,\n    );\n  }\n\n  const coordinatesMeta = result.coordinatesMeta;\n  const expectedLength =\n    coordinatesMeta?.shape === 'bbox'\n      ? 4\n      : coordinatesMeta?.shape === 'point'\n        ? 2\n        : 0;\n  if (!expectedLength) {\n    throw new Error(\n      `invalid parsed locate result: unsupported coordinatesMeta.shape ${JSON.stringify(\n        coordinatesMeta?.shape,\n      )}`,\n    );\n  }\n\n  const coordinates = result.coordinates;\n  if (\n    !Array.isArray(coordinates) ||\n    coordinates.length !== expectedLength ||\n    !coordinates.every(\n      (value) => typeof value === 'number' && Number.isFinite(value),\n    )\n  ) {\n    throw new Error(\n      `invalid parsed locate result: ${coordinatesMeta.shape} coordinates must be ${expectedLength} finite numbers, got ${JSON.stringify(\n        coordinates,\n      )}`,\n    );\n  }\n\n  const { order, normalizedBy, rounding } = coordinatesMeta;\n  if (!['round', 'trunc', 'none'].includes(rounding)) {\n    throw new Error(`invalid locate coordinate rounding: ${rounding}`);\n  }\n  if (order !== 'xy' && order !== 'yx') {\n    throw new Error(\n      `invalid locate coordinate order: ${JSON.stringify(order)}`,\n    );\n  }\n  if (\n    normalizedBy !== undefined &&\n    (!Number.isFinite(normalizedBy) || normalizedBy <= 0)\n  ) {\n    throw new Error(\n      `locate result normalizedBy must be positive and finite: ${normalizedBy}`,\n    );\n  }\n}\n\nfunction resolveCoordinateLimits(\n  result: LocateResultValue,\n  maxX: number,\n  maxY: number,\n): number[] {\n  const resolvedCoordinates = result.coordinatesMeta;\n  const normalizedBy = resolvedCoordinates.normalizedBy;\n  if (normalizedBy !== undefined) {\n    return result.coordinates.map(() => normalizedBy);\n  }\n\n  if (resolvedCoordinates.shape === 'bbox') {\n    return resolvedCoordinates.order === 'yx'\n      ? [maxY, maxX, maxY, maxX]\n      : [maxX, maxY, maxX, maxY];\n  }\n\n  return resolvedCoordinates.order === 'yx' ? [maxY, maxX] : [maxX, maxY];\n}\n\nexport function assertLocateResultCoordinateRangeAndOrder(\n  result: LocateResultValue,\n  maxX: number,\n  maxY: number,\n) {\n  const coordinatesMeta = result.coordinatesMeta;\n  const { normalizedBy } = coordinatesMeta;\n  const limits = resolveCoordinateLimits(result, maxX, maxY);\n  const outOfRange = result.coordinates.some(\n    (value, index) => value < 0 || value > limits[index],\n  );\n\n  if (outOfRange) {\n    const source =\n      normalizedBy !== undefined\n        ? `normalized range [0, ${normalizedBy}]`\n        : `image size [0, ${maxX}]x[0, ${maxY}]`;\n    const normalizedInfo =\n      normalizedBy !== undefined ? ` normalizedBy=${normalizedBy}` : '';\n    throw new Error(\n      `locate result coordinates ${JSON.stringify(\n        result.coordinates,\n      )} exceed ${source}. shape=${\n        coordinatesMeta.shape\n      } order=${coordinatesMeta.order}${normalizedInfo} limits=${JSON.stringify(\n        limits,\n      )}`,\n    );\n  }\n\n  if (\n    isBboxLocateResultValue(result) &&\n    (result.coordinates[2] < result.coordinates[0] ||\n      result.coordinates[3] < result.coordinates[1])\n  ) {\n    throw new Error(\n      `invalid locate bbox coordinate order: ${JSON.stringify(result.coordinates)}`,\n    );\n  }\n}\n"],"names":["assertLocateResultStructure","result","Error","JSON","coordinatesMeta","expectedLength","coordinates","Array","value","Number","order","normalizedBy","rounding","undefined","resolveCoordinateLimits","maxX","maxY","resolvedCoordinates","assertLocateResultCoordinateRangeAndOrder","limits","outOfRange","index","source","normalizedInfo","isBboxLocateResultValue"],"mappings":";AAGO,SAASA,4BAA4BC,MAAyB;IACnE,IAAI,CAACA,UAAU,AAAkB,YAAlB,OAAOA,QACpB,MAAM,IAAIC,MACR,CAAC,mDAAmD,EAAEC,KAAK,SAAS,CAClEF,SACC;IAIP,MAAMG,kBAAkBH,OAAO,eAAe;IAC9C,MAAMI,iBACJD,iBAAiB,UAAU,SACvB,IACAA,iBAAiB,UAAU,UACzB,IACA;IACR,IAAI,CAACC,gBACH,MAAM,IAAIH,MACR,CAAC,gEAAgE,EAAEC,KAAK,SAAS,CAC/EC,iBAAiB,QAChB;IAIP,MAAME,cAAcL,OAAO,WAAW;IACtC,IACE,CAACM,MAAM,OAAO,CAACD,gBACfA,YAAY,MAAM,KAAKD,kBACvB,CAACC,YAAY,KAAK,CAChB,CAACE,QAAU,AAAiB,YAAjB,OAAOA,SAAsBC,OAAO,QAAQ,CAACD,SAG1D,MAAM,IAAIN,MACR,CAAC,8BAA8B,EAAEE,gBAAgB,KAAK,CAAC,qBAAqB,EAAEC,eAAe,qBAAqB,EAAEF,KAAK,SAAS,CAChIG,cACC;IAIP,MAAM,EAAEI,KAAK,EAAEC,YAAY,EAAEC,QAAQ,EAAE,GAAGR;IAC1C,IAAI,CAAC;QAAC;QAAS;QAAS;KAAO,CAAC,QAAQ,CAACQ,WACvC,MAAM,IAAIV,MAAM,CAAC,oCAAoC,EAAEU,UAAU;IAEnE,IAAIF,AAAU,SAAVA,SAAkBA,AAAU,SAAVA,OACpB,MAAM,IAAIR,MACR,CAAC,iCAAiC,EAAEC,KAAK,SAAS,CAACO,QAAQ;IAG/D,IACEC,AAAiBE,WAAjBF,gBACC,EAACF,OAAO,QAAQ,CAACE,iBAAiBA,gBAAgB,IAEnD,MAAM,IAAIT,MACR,CAAC,wDAAwD,EAAES,cAAc;AAG/E;AAEA,SAASG,wBACPb,MAAyB,EACzBc,IAAY,EACZC,IAAY;IAEZ,MAAMC,sBAAsBhB,OAAO,eAAe;IAClD,MAAMU,eAAeM,oBAAoB,YAAY;IACrD,IAAIN,AAAiBE,WAAjBF,cACF,OAAOV,OAAO,WAAW,CAAC,GAAG,CAAC,IAAMU;IAGtC,IAAIM,AAA8B,WAA9BA,oBAAoB,KAAK,EAC3B,OAAOA,AAA8B,SAA9BA,oBAAoB,KAAK,GAC5B;QAACD;QAAMD;QAAMC;QAAMD;KAAK,GACxB;QAACA;QAAMC;QAAMD;QAAMC;KAAK;IAG9B,OAAOC,AAA8B,SAA9BA,oBAAoB,KAAK,GAAY;QAACD;QAAMD;KAAK,GAAG;QAACA;QAAMC;KAAK;AACzE;AAEO,SAASE,0CACdjB,MAAyB,EACzBc,IAAY,EACZC,IAAY;IAEZ,MAAMZ,kBAAkBH,OAAO,eAAe;IAC9C,MAAM,EAAEU,YAAY,EAAE,GAAGP;IACzB,MAAMe,SAASL,wBAAwBb,QAAQc,MAAMC;IACrD,MAAMI,aAAanB,OAAO,WAAW,CAAC,IAAI,CACxC,CAACO,OAAOa,QAAUb,QAAQ,KAAKA,QAAQW,MAAM,CAACE,MAAM;IAGtD,IAAID,YAAY;QACd,MAAME,SACJX,AAAiBE,WAAjBF,eACI,CAAC,qBAAqB,EAAEA,aAAa,CAAC,CAAC,GACvC,CAAC,eAAe,EAAEI,KAAK,MAAM,EAAEC,KAAK,CAAC,CAAC;QAC5C,MAAMO,iBACJZ,AAAiBE,WAAjBF,eAA6B,CAAC,cAAc,EAAEA,cAAc,GAAG;QACjE,MAAM,IAAIT,MACR,CAAC,0BAA0B,EAAEC,KAAK,SAAS,CACzCF,OAAO,WAAW,EAClB,QAAQ,EAAEqB,OAAO,QAAQ,EACzBlB,gBAAgB,KAAK,CACtB,OAAO,EAAEA,gBAAgB,KAAK,GAAGmB,eAAe,QAAQ,EAAEpB,KAAK,SAAS,CACvEgB,SACC;IAEP;IAEA,IACEK,wBAAwBvB,WACvBA,CAAAA,OAAO,WAAW,CAAC,EAAE,GAAGA,OAAO,WAAW,CAAC,EAAE,IAC5CA,OAAO,WAAW,CAAC,EAAE,GAAGA,OAAO,WAAW,CAAC,EAAC,GAE9C,MAAM,IAAIC,MACR,CAAC,sCAAsC,EAAEC,KAAK,SAAS,CAACF,OAAO,WAAW,GAAG;AAGnF"}
+{"version":3,"file":"ai-model/shared/model-locate-result/validation.mjs","sources":["../../../../../src/ai-model/shared/model-locate-result/validation.ts"],"sourcesContent":["import { isBboxLocateResultValue } from './types';\nimport type { LocateResultValue } from './types';\n\nexport function assertLocateResultStructure(result: LocateResultValue): void {\n  if (!result || typeof result !== 'object') {\n    throw new Error(\n      `invalid parsed locate result: expected object, got ${JSON.stringify(\n        result,\n      )}`,\n    );\n  }\n\n  const coordinatesMeta = result.coordinatesMeta;\n  const expectedLength =\n    coordinatesMeta?.shape === 'bbox'\n      ? 4\n      : coordinatesMeta?.shape === 'point'\n        ? 2\n        : 0;\n  if (!expectedLength) {\n    throw new Error(\n      `invalid parsed locate result: unsupported coordinatesMeta.shape ${JSON.stringify(\n        coordinatesMeta?.shape,\n      )}`,\n    );\n  }\n\n  const coordinates = result.coordinates;\n  if (\n    !Array.isArray(coordinates) ||\n    coordinates.length !== expectedLength ||\n    !coordinates.every(\n      (value) => typeof value === 'number' && Number.isFinite(value),\n    )\n  ) {\n    throw new Error(\n      `invalid parsed locate result: ${coordinatesMeta.shape} coordinates must be ${expectedLength} finite numbers, got ${JSON.stringify(\n        coordinates,\n      )}`,\n    );\n  }\n\n  if (\n    isBboxLocateResultValue(result) &&\n    result.coordinatesMeta.boundsType !== undefined &&\n    !['inclusive', 'edges'].includes(result.coordinatesMeta.boundsType)\n  ) {\n    throw new Error(\n      `invalid locate bbox boundsType: ${result.coordinatesMeta.boundsType}`,\n    );\n  }\n  const { order, normalizedBy, rounding } = coordinatesMeta;\n  if (!['round', 'trunc', 'none'].includes(rounding)) {\n    throw new Error(`invalid locate coordinate rounding: ${rounding}`);\n  }\n  if (order !== 'xy' && order !== 'yx') {\n    throw new Error(\n      `invalid locate coordinate order: ${JSON.stringify(order)}`,\n    );\n  }\n  if (\n    normalizedBy !== undefined &&\n    (!Number.isFinite(normalizedBy) || normalizedBy <= 0)\n  ) {\n    throw new Error(\n      `locate result normalizedBy must be positive and finite: ${normalizedBy}`,\n    );\n  }\n}\n\nfunction resolveCoordinateLimits(\n  result: LocateResultValue,\n  maxX: number,\n  maxY: number,\n): number[] {\n  const resolvedCoordinates = result.coordinatesMeta;\n  const normalizedBy = resolvedCoordinates.normalizedBy;\n  if (normalizedBy !== undefined) {\n    return result.coordinates.map(() => normalizedBy);\n  }\n\n  if (resolvedCoordinates.shape === 'bbox') {\n    return resolvedCoordinates.order === 'yx'\n      ? [maxY, maxX, maxY, maxX]\n      : [maxX, maxY, maxX, maxY];\n  }\n\n  return resolvedCoordinates.order === 'yx' ? [maxY, maxX] : [maxX, maxY];\n}\n\nexport function assertLocateResultCoordinateRangeAndOrder(\n  result: LocateResultValue,\n  maxX: number,\n  maxY: number,\n) {\n  const coordinatesMeta = result.coordinatesMeta;\n  const { normalizedBy } = coordinatesMeta;\n  const limits = resolveCoordinateLimits(result, maxX, maxY);\n  const outOfRange = result.coordinates.some(\n    (value, index) => value < 0 || value > limits[index],\n  );\n\n  if (outOfRange) {\n    const source =\n      normalizedBy !== undefined\n        ? `normalized range [0, ${normalizedBy}]`\n        : `image size [0, ${maxX}]x[0, ${maxY}]`;\n    const normalizedInfo =\n      normalizedBy !== undefined ? ` normalizedBy=${normalizedBy}` : '';\n    throw new Error(\n      `locate result coordinates ${JSON.stringify(\n        result.coordinates,\n      )} exceed ${source}. shape=${\n        coordinatesMeta.shape\n      } order=${coordinatesMeta.order}${normalizedInfo} limits=${JSON.stringify(\n        limits,\n      )}`,\n    );\n  }\n\n  if (\n    isBboxLocateResultValue(result) &&\n    (result.coordinates[2] < result.coordinates[0] ||\n      result.coordinates[3] < result.coordinates[1])\n  ) {\n    throw new Error(\n      `invalid locate bbox coordinate order: ${JSON.stringify(result.coordinates)}`,\n    );\n  }\n}\n"],"names":["assertLocateResultStructure","result","Error","JSON","coordinatesMeta","expectedLength","coordinates","Array","value","Number","isBboxLocateResultValue","undefined","order","normalizedBy","rounding","resolveCoordinateLimits","maxX","maxY","resolvedCoordinates","assertLocateResultCoordinateRangeAndOrder","limits","outOfRange","index","source","normalizedInfo"],"mappings":";AAGO,SAASA,4BAA4BC,MAAyB;IACnE,IAAI,CAACA,UAAU,AAAkB,YAAlB,OAAOA,QACpB,MAAM,IAAIC,MACR,CAAC,mDAAmD,EAAEC,KAAK,SAAS,CAClEF,SACC;IAIP,MAAMG,kBAAkBH,OAAO,eAAe;IAC9C,MAAMI,iBACJD,iBAAiB,UAAU,SACvB,IACAA,iBAAiB,UAAU,UACzB,IACA;IACR,IAAI,CAACC,gBACH,MAAM,IAAIH,MACR,CAAC,gEAAgE,EAAEC,KAAK,SAAS,CAC/EC,iBAAiB,QAChB;IAIP,MAAME,cAAcL,OAAO,WAAW;IACtC,IACE,CAACM,MAAM,OAAO,CAACD,gBACfA,YAAY,MAAM,KAAKD,kBACvB,CAACC,YAAY,KAAK,CAChB,CAACE,QAAU,AAAiB,YAAjB,OAAOA,SAAsBC,OAAO,QAAQ,CAACD,SAG1D,MAAM,IAAIN,MACR,CAAC,8BAA8B,EAAEE,gBAAgB,KAAK,CAAC,qBAAqB,EAAEC,eAAe,qBAAqB,EAAEF,KAAK,SAAS,CAChIG,cACC;IAIP,IACEI,wBAAwBT,WACxBA,AAAsCU,WAAtCV,OAAO,eAAe,CAAC,UAAU,IACjC,CAAC;QAAC;QAAa;KAAQ,CAAC,QAAQ,CAACA,OAAO,eAAe,CAAC,UAAU,GAElE,MAAM,IAAIC,MACR,CAAC,gCAAgC,EAAED,OAAO,eAAe,CAAC,UAAU,EAAE;IAG1E,MAAM,EAAEW,KAAK,EAAEC,YAAY,EAAEC,QAAQ,EAAE,GAAGV;IAC1C,IAAI,CAAC;QAAC;QAAS;QAAS;KAAO,CAAC,QAAQ,CAACU,WACvC,MAAM,IAAIZ,MAAM,CAAC,oCAAoC,EAAEY,UAAU;IAEnE,IAAIF,AAAU,SAAVA,SAAkBA,AAAU,SAAVA,OACpB,MAAM,IAAIV,MACR,CAAC,iCAAiC,EAAEC,KAAK,SAAS,CAACS,QAAQ;IAG/D,IACEC,AAAiBF,WAAjBE,gBACC,EAACJ,OAAO,QAAQ,CAACI,iBAAiBA,gBAAgB,IAEnD,MAAM,IAAIX,MACR,CAAC,wDAAwD,EAAEW,cAAc;AAG/E;AAEA,SAASE,wBACPd,MAAyB,EACzBe,IAAY,EACZC,IAAY;IAEZ,MAAMC,sBAAsBjB,OAAO,eAAe;IAClD,MAAMY,eAAeK,oBAAoB,YAAY;IACrD,IAAIL,AAAiBF,WAAjBE,cACF,OAAOZ,OAAO,WAAW,CAAC,GAAG,CAAC,IAAMY;IAGtC,IAAIK,AAA8B,WAA9BA,oBAAoB,KAAK,EAC3B,OAAOA,AAA8B,SAA9BA,oBAAoB,KAAK,GAC5B;QAACD;QAAMD;QAAMC;QAAMD;KAAK,GACxB;QAACA;QAAMC;QAAMD;QAAMC;KAAK;IAG9B,OAAOC,AAA8B,SAA9BA,oBAAoB,KAAK,GAAY;QAACD;QAAMD;KAAK,GAAG;QAACA;QAAMC;KAAK;AACzE;AAEO,SAASE,0CACdlB,MAAyB,EACzBe,IAAY,EACZC,IAAY;IAEZ,MAAMb,kBAAkBH,OAAO,eAAe;IAC9C,MAAM,EAAEY,YAAY,EAAE,GAAGT;IACzB,MAAMgB,SAASL,wBAAwBd,QAAQe,MAAMC;IACrD,MAAMI,aAAapB,OAAO,WAAW,CAAC,IAAI,CACxC,CAACO,OAAOc,QAAUd,QAAQ,KAAKA,QAAQY,MAAM,CAACE,MAAM;IAGtD,IAAID,YAAY;QACd,MAAME,SACJV,AAAiBF,WAAjBE,eACI,CAAC,qBAAqB,EAAEA,aAAa,CAAC,CAAC,GACvC,CAAC,eAAe,EAAEG,KAAK,MAAM,EAAEC,KAAK,CAAC,CAAC;QAC5C,MAAMO,iBACJX,AAAiBF,WAAjBE,eAA6B,CAAC,cAAc,EAAEA,cAAc,GAAG;QACjE,MAAM,IAAIX,MACR,CAAC,0BAA0B,EAAEC,KAAK,SAAS,CACzCF,OAAO,WAAW,EAClB,QAAQ,EAAEsB,OAAO,QAAQ,EACzBnB,gBAAgB,KAAK,CACtB,OAAO,EAAEA,gBAAgB,KAAK,GAAGoB,eAAe,QAAQ,EAAErB,KAAK,SAAS,CACvEiB,SACC;IAEP;IAEA,IACEV,wBAAwBT,WACvBA,CAAAA,OAAO,WAAW,CAAC,EAAE,GAAGA,OAAO,WAAW,CAAC,EAAE,IAC5CA,OAAO,WAAW,CAAC,EAAE,GAAGA,OAAO,WAAW,CAAC,EAAC,GAE9C,MAAM,IAAIC,MACR,CAAC,sCAAsC,EAAEC,KAAK,SAAS,CAACF,OAAO,WAAW,GAAG;AAGnF"}

package/dist/es/ai-model/workflows/grounding/locate.mjs

@@ -2,7 +2,7 @@ import { getDebug } from "@midscene/shared/logger";
 import { assert } from "@midscene/shared/utils";
 import { userPromptToString } from "../../../common.mjs";
 import { buildElementLocateSystemPrompt } from "../../prompt/locate/index.mjs";
-import { AIResponseParseError, callAI } from "../../service-caller/index.mjs";
+import { AIResponseParseError, callGenerativeAI } from "../../service-caller/index.mjs";
 import { callAiAndParseWithRetry, withSemanticRetryFeedback } from "../../service-caller/semantic-retry.mjs";
 import { mapSearchAreaResultToOriginalResult } from "./search-area-mapping.mjs";
 import { formatLocateModelContext, prepareLocateModelInput } from "./utils.mjs";
@@ -88,7 +88,7 @@ async function genericLocate(locateRequest) {
     });
     try {
         return await callAiAndParseWithRetry({
-            callAi: (retryAttempt, previousParseError)=>callAI(withSemanticRetryFeedback(messages, previousParseError), modelRuntime, {
+            callAi: (retryAttempt, previousParseError)=>callGenerativeAI(withSemanticRetryFeedback(messages, previousParseError), modelRuntime, {
                     abortSignal: options.abortSignal,
                     expectedJsonObjectResponse: protocol.expectedJsonObjectResponse,
                     semanticRetryAttempt: retryAttempt