@midscene/core 1.17.5 → 1.17.7

package/dist/lib/ai-model/service-caller/openai/siwc/authorization.js.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/service-caller/openai/siwc/authorization.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../../../src/ai-model/service-caller/openai/siwc/authorization.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { type JWTVerifyGetKey, createRemoteJWKSet, jwtVerify } from 'jose';\nimport { tokenResponseError, withSiwcStage } from './errors';\n\nconst ISSUER = 'https://auth.openai.com';\nconst RESOURCE = 'https://api.openai.com/v1';\nexport interface SiwcLoginAttempt {\n  hostId: string;\n  redirectUri: string;\n  clientId?: string;\n  state: string;\n  nonce: string;\n  verifier: string;\n  expiresAt: number;\n}\n\nfunction base64url(bytes: Uint8Array): string {\n  return btoa(String.fromCharCode(...bytes))\n    .replace(/\\+/g, '-')\n    .replace(/\\//g, '_')\n    .replace(/=+$/, '');\n}\n\nexport async function createSiwcAuthorization(options: {\n  hostId: string;\n  clientId?: string;\n  redirectUri: string;\n  agentName: string;\n}) {\n  const { hostId, clientId, redirectUri, agentName } = options;\n  const random = () => base64url(crypto.getRandomValues(new Uint8Array(32)));\n  const attempt: SiwcLoginAttempt = {\n    hostId,\n    redirectUri,\n    clientId,\n    state: random(),\n    nonce: random(),\n    verifier: random(),\n    expiresAt: Date.now() + 5 * 60_000,\n  };\n  const challenge = await crypto.subtle.digest(\n    'SHA-256',\n    new TextEncoder().encode(attempt.verifier),\n  );\n  const url = new URL(`${ISSUER}/api/accounts/authorize`);\n  url.search = new URLSearchParams({\n    client_id: clientId ?? 'dynamic_agent_client',\n    ...(clientId ? {} : { agent_name_hint: agentName }),\n    ext_agent_host_id: hostId,\n    response_type: 'code',\n    redirect_uri: attempt.redirectUri,\n    scope:\n      'openid profile email offline_access resource.invoke chatgpt.tokens.use.direct',\n    resource: RESOURCE,\n    state: attempt.state,\n    nonce: attempt.nonce,\n    code_challenge_method: 'S256',\n    code_challenge: base64url(new Uint8Array(challenge)),\n  }).toString();\n  return { attempt, url: url.toString() };\n}\n\nexport function parseSiwcCallback(url: string, attempt: SiwcLoginAttempt) {\n  const callback = new URL(url);\n  const expected = new URL(attempt.redirectUri);\n  if (\n    callback.origin !== expected.origin ||\n    callback.pathname !== expected.pathname\n  ) {\n    return undefined;\n  }\n  const params = callback.searchParams;\n  if (\n    params.getAll('state').length !== 1 ||\n    params.get('state') !== attempt.state\n  ) {\n    return undefined;\n  }\n  if (Date.now() >= attempt.expiresAt) {\n    throw new Error('Authorization timed out. Start authorization again.');\n  }\n  if (params.has('error')) {\n    throw new Error(\n      `OpenAI authorization failed: ${params.get('error_description') || params.get('error')}`,\n    );\n  }\n  const code = params.get('code');\n  const clientId = params.get('client_id') || attempt.clientId;\n  if (\n    !code ||\n    !clientId ||\n    clientId === 'dynamic_agent_client' ||\n    params.getAll('code').length !== 1 ||\n    params.getAll('client_id').length > 1 ||\n    (attempt.clientId && clientId !== attempt.clientId)\n  ) {\n    throw new Error('Invalid OpenAI authorization callback.');\n  }\n  return { code, clientId };\n}\n\nexport async function exchangeSiwcCode(\n  attempt: SiwcLoginAttempt,\n  callback: { code: string; clientId: string },\n  signal: AbortSignal,\n  dependencies?: { fetch: typeof fetch; keys: JWTVerifyGetKey },\n) {\n  signal.throwIfAborted();\n  const response = await withSiwcStage('Token exchange', async () => {\n    const response = await (dependencies?.fetch ?? fetch)(\n      `${ISSUER}/api/accounts/oauth/token`,\n      {\n        method: 'POST',\n        headers: { 'Content-Type': 'application/x-www-form-urlencoded' },\n        body: new URLSearchParams({\n          grant_type: 'authorization_code',\n          client_id: callback.clientId,\n          code: callback.code,\n          code_verifier: attempt.verifier,\n          redirect_uri: attempt.redirectUri,\n          resource: RESOURCE,\n        }),\n        signal,\n        redirect: 'error',\n      },\n    );\n    if (!response.ok) {\n      throw await tokenResponseError(response);\n    }\n    return response;\n  });\n  const tokens = await withSiwcStage('Token response parsing', () =>\n    response.json(),\n  );\n  if (\n    !tokens ||\n    typeof tokens.access_token !== 'string' ||\n    !tokens.access_token ||\n    typeof tokens.refresh_token !== 'string' ||\n    !tokens.refresh_token ||\n    typeof tokens.id_token !== 'string' ||\n    typeof tokens.scope !== 'string' ||\n    typeof tokens.expires_in !== 'number' ||\n    !Number.isFinite(tokens.expires_in) ||\n    tokens.expires_in <= 0 ||\n    typeof tokens.token_type !== 'string' ||\n    tokens.token_type.toLowerCase() !== 'bearer'\n  ) {\n    throw new Error('OpenAI returned an invalid token response.');\n  }\n  const scopes = tokens.scope.split(/\\s+/);\n  if (!scopes.includes('chatgpt.tokens.use.direct')) {\n    throw new Error('ChatGPT plan usage was not authorized.');\n  }\n  const { payload } = await withSiwcStage('ID token verification', () =>\n    jwtVerify(\n      tokens.id_token,\n      dependencies?.keys ??\n        createRemoteJWKSet(new URL(`${ISSUER}/.well-known/jwks.json`)),\n      {\n        issuer: ISSUER,\n        audience: callback.clientId,\n        algorithms: ['RS256'],\n        // Some CI machines have incorrect system clocks. Do not compare ID token\n        // exp/nbf claims against local time when accepting newly returned tokens.\n        // Keep signature, issuer, audience and required-claim validation.\n        clockTolerance: Number.POSITIVE_INFINITY,\n        requiredClaims: ['sub', 'exp', 'iat', 'nonce'],\n      },\n    ),\n  );\n  if (payload.nonce !== attempt.nonce || !payload.sub) {\n    throw new Error('OpenAI login identity or nonce did not match.');\n  }\n  signal.throwIfAborted();\n  return {\n    ...(typeof payload.email === 'string' ? { email: payload.email } : {}),\n    ext_agent_host_id: attempt.hostId,\n    client_id: callback.clientId,\n    subject: payload.sub,\n    issuer: ISSUER,\n    access_token: tokens.access_token as string,\n    refresh_token: tokens.refresh_token as string,\n    id_token: tokens.id_token as string,\n    scopes,\n    expires_at: Date.now() + tokens.expires_in * 1000,\n  };\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","ISSUER","RESOURCE","base64url","bytes","btoa","String","createSiwcAuthorization","options","hostId","clientId","redirectUri","agentName","random","crypto","Uint8Array","attempt","Date","challenge","TextEncoder","url","URL","URLSearchParams","parseSiwcCallback","callback","expected","params","Error","code","exchangeSiwcCode","signal","dependencies","response","withSiwcStage","fetch","tokenResponseError","tokens","Number","scopes","payload","jwtVerify","createRemoteJWKSet"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;ACHA,MAAMI,SAAS;AACf,MAAMC,WAAW;AAWjB,SAASC,UAAUC,KAAiB;IAClC,OAAOC,KAAKC,OAAO,YAAY,IAAIF,QAChC,OAAO,CAAC,OAAO,KACf,OAAO,CAAC,OAAO,KACf,OAAO,CAAC,OAAO;AACpB;AAEO,eAAeG,wBAAwBC,OAK7C;IACC,MAAM,EAAEC,MAAM,EAAEC,QAAQ,EAAEC,WAAW,EAAEC,SAAS,EAAE,GAAGJ;IACrD,MAAMK,SAAS,IAAMV,UAAUW,OAAO,eAAe,CAAC,IAAIC,WAAW;IACrE,MAAMC,UAA4B;QAChCP;QACAE;QACAD;QACA,OAAOG;QACP,OAAOA;QACP,UAAUA;QACV,WAAWI,KAAK,GAAG,KAAK;IAC1B;IACA,MAAMC,YAAY,MAAMJ,OAAO,MAAM,CAAC,MAAM,CAC1C,WACA,IAAIK,cAAc,MAAM,CAACH,QAAQ,QAAQ;IAE3C,MAAMI,MAAM,IAAIC,IAAI,GAAGpB,OAAO,uBAAuB,CAAC;IACtDmB,IAAI,MAAM,GAAG,IAAIE,gBAAgB;QAC/B,WAAWZ,YAAY;QACvB,GAAIA,WAAW,CAAC,IAAI;YAAE,iBAAiBE;QAAU,CAAC;QAClD,mBAAmBH;QACnB,eAAe;QACf,cAAcO,QAAQ,WAAW;QACjC,OACE;QACF,UAAUd;QACV,OAAOc,QAAQ,KAAK;QACpB,OAAOA,QAAQ,KAAK;QACpB,uBAAuB;QACvB,gBAAgBb,UAAU,IAAIY,WAAWG;IAC3C,GAAG,QAAQ;IACX,OAAO;QAAEF;QAAS,KAAKI,IAAI,QAAQ;IAAG;AACxC;AAEO,SAASG,kBAAkBH,GAAW,EAAEJ,OAAyB;IACtE,MAAMQ,WAAW,IAAIH,IAAID;IACzB,MAAMK,WAAW,IAAIJ,IAAIL,QAAQ,WAAW;IAC5C,IACEQ,SAAS,MAAM,KAAKC,SAAS,MAAM,IACnCD,SAAS,QAAQ,KAAKC,SAAS,QAAQ,EAEvC;IAEF,MAAMC,SAASF,SAAS,YAAY;IACpC,IACEE,AAAkC,MAAlCA,OAAO,MAAM,CAAC,SAAS,MAAM,IAC7BA,OAAO,GAAG,CAAC,aAAaV,QAAQ,KAAK,EAErC;IAEF,IAAIC,KAAK,GAAG,MAAMD,QAAQ,SAAS,EACjC,MAAM,IAAIW,MAAM;IAElB,IAAID,OAAO,GAAG,CAAC,UACb,MAAM,IAAIC,MACR,CAAC,6BAA6B,EAAED,OAAO,GAAG,CAAC,wBAAwBA,OAAO,GAAG,CAAC,UAAU;IAG5F,MAAME,OAAOF,OAAO,GAAG,CAAC;IACxB,MAAMhB,WAAWgB,OAAO,GAAG,CAAC,gBAAgBV,QAAQ,QAAQ;IAC5D,IACE,CAACY,QACD,CAAClB,YACDA,AAAa,2BAAbA,YACAgB,AAAiC,MAAjCA,OAAO,MAAM,CAAC,QAAQ,MAAM,IAC5BA,OAAO,MAAM,CAAC,aAAa,MAAM,GAAG,KACnCV,QAAQ,QAAQ,IAAIN,aAAaM,QAAQ,QAAQ,EAElD,MAAM,IAAIW,MAAM;IAElB,OAAO;QAAEC;QAAMlB;IAAS;AAC1B;AAEO,eAAemB,iBACpBb,OAAyB,EACzBQ,QAA4C,EAC5CM,MAAmB,EACnBC,YAA6D;IAE7DD,OAAO,cAAc;IACrB,MAAME,WAAW,MAAMC,AAAAA,IAAAA,mCAAAA,aAAAA,AAAAA,EAAc,kBAAkB;QACrD,MAAMD,WAAW,MAAOD,AAAAA,CAAAA,cAAc,SAASG,KAAI,EACjD,GAAGjC,OAAO,yBAAyB,CAAC,EACpC;YACE,QAAQ;YACR,SAAS;gBAAE,gBAAgB;YAAoC;YAC/D,MAAM,IAAIqB,gBAAgB;gBACxB,YAAY;gBACZ,WAAWE,SAAS,QAAQ;gBAC5B,MAAMA,SAAS,IAAI;gBACnB,eAAeR,QAAQ,QAAQ;gBAC/B,cAAcA,QAAQ,WAAW;gBACjC,UAAUd;YACZ;YACA4B;YACA,UAAU;QACZ;QAEF,IAAI,CAACE,SAAS,EAAE,EACd,MAAM,MAAMG,AAAAA,IAAAA,mCAAAA,kBAAAA,AAAAA,EAAmBH;QAEjC,OAAOA;IACT;IACA,MAAMI,SAAS,MAAMH,AAAAA,IAAAA,mCAAAA,aAAAA,AAAAA,EAAc,0BAA0B,IAC3DD,SAAS,IAAI;IAEf,IACE,CAACI,UACD,AAA+B,YAA/B,OAAOA,OAAO,YAAY,IAC1B,CAACA,OAAO,YAAY,IACpB,AAAgC,YAAhC,OAAOA,OAAO,aAAa,IAC3B,CAACA,OAAO,aAAa,IACrB,AAA2B,YAA3B,OAAOA,OAAO,QAAQ,IACtB,AAAwB,YAAxB,OAAOA,OAAO,KAAK,IACnB,AAA6B,YAA7B,OAAOA,OAAO,UAAU,IACxB,CAACC,OAAO,QAAQ,CAACD,OAAO,UAAU,KAClCA,OAAO,UAAU,IAAI,KACrB,AAA6B,YAA7B,OAAOA,OAAO,UAAU,IACxBA,AAAoC,aAApCA,OAAO,UAAU,CAAC,WAAW,IAE7B,MAAM,IAAIT,MAAM;IAElB,MAAMW,SAASF,OAAO,KAAK,CAAC,KAAK,CAAC;IAClC,IAAI,CAACE,OAAO,QAAQ,CAAC,8BACnB,MAAM,IAAIX,MAAM;IAElB,MAAM,EAAEY,OAAO,EAAE,GAAG,MAAMN,AAAAA,IAAAA,mCAAAA,aAAAA,AAAAA,EAAc,yBAAyB,IAC/DO,AAAAA,IAAAA,8BAAAA,SAAAA,AAAAA,EACEJ,OAAO,QAAQ,EACfL,cAAc,QACZU,AAAAA,IAAAA,8BAAAA,kBAAAA,AAAAA,EAAmB,IAAIpB,IAAI,GAAGpB,OAAO,sBAAsB,CAAC,IAC9D;YACE,QAAQA;YACR,UAAUuB,SAAS,QAAQ;YAC3B,YAAY;gBAAC;aAAQ;YAIrB,gBAAgBa;YAChB,gBAAgB;gBAAC;gBAAO;gBAAO;gBAAO;aAAQ;QAChD;IAGJ,IAAIE,QAAQ,KAAK,KAAKvB,QAAQ,KAAK,IAAI,CAACuB,QAAQ,GAAG,EACjD,MAAM,IAAIZ,MAAM;IAElBG,OAAO,cAAc;IACrB,OAAO;QACL,GAAI,AAAyB,YAAzB,OAAOS,QAAQ,KAAK,GAAgB;YAAE,OAAOA,QAAQ,KAAK;QAAC,IAAI,CAAC,CAAC;QACrE,mBAAmBvB,QAAQ,MAAM;QACjC,WAAWQ,SAAS,QAAQ;QAC5B,SAASe,QAAQ,GAAG;QACpB,QAAQtC;QACR,cAAcmC,OAAO,YAAY;QACjC,eAAeA,OAAO,aAAa;QACnC,UAAUA,OAAO,QAAQ;QACzBE;QACA,YAAYrB,KAAK,GAAG,KAAKmB,AAAoB,OAApBA,OAAO,UAAU;IAC5C;AACF"}
+{"version":3,"file":"ai-model/service-caller/openai/siwc/authorization.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../../../src/ai-model/service-caller/openai/siwc/authorization.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { type JWTVerifyGetKey, jwtVerify } from 'jose';\nimport { tokenResponseError, withSiwcStage } from './errors';\nimport { createSiwcJWKSet, fetchSiwc } from './network';\n\nconst ISSUER = 'https://auth.openai.com';\nconst RESOURCE = 'https://api.openai.com/v1';\nexport interface SiwcLoginAttempt {\n  hostId: string;\n  redirectUri: string;\n  clientId?: string;\n  state: string;\n  nonce: string;\n  verifier: string;\n  expiresAt: number;\n}\n\nfunction base64url(bytes: Uint8Array): string {\n  return btoa(String.fromCharCode(...bytes))\n    .replace(/\\+/g, '-')\n    .replace(/\\//g, '_')\n    .replace(/=+$/, '');\n}\n\nexport async function createSiwcAuthorization(options: {\n  hostId: string;\n  clientId?: string;\n  redirectUri: string;\n  agentName: string;\n}) {\n  const { hostId, clientId, redirectUri, agentName } = options;\n  const random = () => base64url(crypto.getRandomValues(new Uint8Array(32)));\n  const attempt: SiwcLoginAttempt = {\n    hostId,\n    redirectUri,\n    clientId,\n    state: random(),\n    nonce: random(),\n    verifier: random(),\n    expiresAt: Date.now() + 5 * 60_000,\n  };\n  const challenge = await crypto.subtle.digest(\n    'SHA-256',\n    new TextEncoder().encode(attempt.verifier),\n  );\n  const url = new URL(`${ISSUER}/api/accounts/authorize`);\n  url.search = new URLSearchParams({\n    client_id: clientId ?? 'dynamic_agent_client',\n    ...(clientId ? {} : { agent_name_hint: agentName }),\n    ext_agent_host_id: hostId,\n    response_type: 'code',\n    redirect_uri: attempt.redirectUri,\n    scope:\n      'openid profile email offline_access resource.invoke chatgpt.tokens.use.direct',\n    resource: RESOURCE,\n    state: attempt.state,\n    nonce: attempt.nonce,\n    code_challenge_method: 'S256',\n    code_challenge: base64url(new Uint8Array(challenge)),\n  }).toString();\n  return { attempt, url: url.toString() };\n}\n\nexport function parseSiwcCallback(url: string, attempt: SiwcLoginAttempt) {\n  const callback = new URL(url);\n  const expected = new URL(attempt.redirectUri);\n  if (\n    callback.origin !== expected.origin ||\n    callback.pathname !== expected.pathname\n  ) {\n    return undefined;\n  }\n  const params = callback.searchParams;\n  if (\n    params.getAll('state').length !== 1 ||\n    params.get('state') !== attempt.state\n  ) {\n    return undefined;\n  }\n  if (Date.now() >= attempt.expiresAt) {\n    throw new Error('Authorization timed out. Start authorization again.');\n  }\n  if (params.has('error')) {\n    throw new Error(\n      `OpenAI authorization failed: ${params.get('error_description') || params.get('error')}`,\n    );\n  }\n  const code = params.get('code');\n  const clientId = params.get('client_id') || attempt.clientId;\n  if (\n    !code ||\n    !clientId ||\n    clientId === 'dynamic_agent_client' ||\n    params.getAll('code').length !== 1 ||\n    params.getAll('client_id').length > 1 ||\n    (attempt.clientId && clientId !== attempt.clientId)\n  ) {\n    throw new Error('Invalid OpenAI authorization callback.');\n  }\n  return { code, clientId };\n}\n\nexport async function exchangeSiwcCode(\n  attempt: SiwcLoginAttempt,\n  callback: { code: string; clientId: string },\n  signal: AbortSignal,\n  dependencies?: { fetch: typeof fetch; keys: JWTVerifyGetKey },\n) {\n  signal.throwIfAborted();\n  const response = await withSiwcStage('Token exchange', async () => {\n    const response = await (dependencies?.fetch ?? fetchSiwc)(\n      `${ISSUER}/api/accounts/oauth/token`,\n      {\n        method: 'POST',\n        headers: { 'Content-Type': 'application/x-www-form-urlencoded' },\n        body: new URLSearchParams({\n          grant_type: 'authorization_code',\n          client_id: callback.clientId,\n          code: callback.code,\n          code_verifier: attempt.verifier,\n          redirect_uri: attempt.redirectUri,\n          resource: RESOURCE,\n        }),\n        signal,\n        redirect: 'error',\n      },\n    );\n    if (!response.ok) {\n      throw await tokenResponseError(response);\n    }\n    return response;\n  });\n  const tokens = await withSiwcStage('Token response parsing', () =>\n    response.json(),\n  );\n  if (\n    !tokens ||\n    typeof tokens.access_token !== 'string' ||\n    !tokens.access_token ||\n    typeof tokens.refresh_token !== 'string' ||\n    !tokens.refresh_token ||\n    typeof tokens.id_token !== 'string' ||\n    typeof tokens.scope !== 'string' ||\n    typeof tokens.expires_in !== 'number' ||\n    !Number.isFinite(tokens.expires_in) ||\n    tokens.expires_in <= 0 ||\n    typeof tokens.token_type !== 'string' ||\n    tokens.token_type.toLowerCase() !== 'bearer'\n  ) {\n    throw new Error('OpenAI returned an invalid token response.');\n  }\n  const scopes = tokens.scope.split(/\\s+/);\n  if (!scopes.includes('chatgpt.tokens.use.direct')) {\n    throw new Error('ChatGPT plan usage was not authorized.');\n  }\n  const { payload } = await withSiwcStage('ID token verification', () =>\n    jwtVerify(\n      tokens.id_token,\n      dependencies?.keys ?? createSiwcJWKSet(ISSUER, signal),\n      {\n        issuer: ISSUER,\n        audience: callback.clientId,\n        algorithms: ['RS256'],\n        // Some CI machines have incorrect system clocks. Do not compare ID token\n        // exp/nbf claims against local time when accepting newly returned tokens.\n        // Keep signature, issuer, audience and required-claim validation.\n        clockTolerance: Number.POSITIVE_INFINITY,\n        requiredClaims: ['sub', 'exp', 'iat', 'nonce'],\n      },\n    ),\n  );\n  if (payload.nonce !== attempt.nonce || !payload.sub) {\n    throw new Error('OpenAI login identity or nonce did not match.');\n  }\n  signal.throwIfAborted();\n  return {\n    ...(typeof payload.email === 'string' ? { email: payload.email } : {}),\n    ext_agent_host_id: attempt.hostId,\n    client_id: callback.clientId,\n    subject: payload.sub,\n    issuer: ISSUER,\n    access_token: tokens.access_token as string,\n    refresh_token: tokens.refresh_token as string,\n    id_token: tokens.id_token as string,\n    scopes,\n    expires_at: Date.now() + tokens.expires_in * 1000,\n  };\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","ISSUER","RESOURCE","base64url","bytes","btoa","String","createSiwcAuthorization","options","hostId","clientId","redirectUri","agentName","random","crypto","Uint8Array","attempt","Date","challenge","TextEncoder","url","URL","URLSearchParams","parseSiwcCallback","callback","expected","params","Error","code","exchangeSiwcCode","signal","dependencies","response","withSiwcStage","fetchSiwc","tokenResponseError","tokens","Number","scopes","payload","jwtVerify","createSiwcJWKSet"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;ACFA,MAAMI,SAAS;AACf,MAAMC,WAAW;AAWjB,SAASC,UAAUC,KAAiB;IAClC,OAAOC,KAAKC,OAAO,YAAY,IAAIF,QAChC,OAAO,CAAC,OAAO,KACf,OAAO,CAAC,OAAO,KACf,OAAO,CAAC,OAAO;AACpB;AAEO,eAAeG,wBAAwBC,OAK7C;IACC,MAAM,EAAEC,MAAM,EAAEC,QAAQ,EAAEC,WAAW,EAAEC,SAAS,EAAE,GAAGJ;IACrD,MAAMK,SAAS,IAAMV,UAAUW,OAAO,eAAe,CAAC,IAAIC,WAAW;IACrE,MAAMC,UAA4B;QAChCP;QACAE;QACAD;QACA,OAAOG;QACP,OAAOA;QACP,UAAUA;QACV,WAAWI,KAAK,GAAG,KAAK;IAC1B;IACA,MAAMC,YAAY,MAAMJ,OAAO,MAAM,CAAC,MAAM,CAC1C,WACA,IAAIK,cAAc,MAAM,CAACH,QAAQ,QAAQ;IAE3C,MAAMI,MAAM,IAAIC,IAAI,GAAGpB,OAAO,uBAAuB,CAAC;IACtDmB,IAAI,MAAM,GAAG,IAAIE,gBAAgB;QAC/B,WAAWZ,YAAY;QACvB,GAAIA,WAAW,CAAC,IAAI;YAAE,iBAAiBE;QAAU,CAAC;QAClD,mBAAmBH;QACnB,eAAe;QACf,cAAcO,QAAQ,WAAW;QACjC,OACE;QACF,UAAUd;QACV,OAAOc,QAAQ,KAAK;QACpB,OAAOA,QAAQ,KAAK;QACpB,uBAAuB;QACvB,gBAAgBb,UAAU,IAAIY,WAAWG;IAC3C,GAAG,QAAQ;IACX,OAAO;QAAEF;QAAS,KAAKI,IAAI,QAAQ;IAAG;AACxC;AAEO,SAASG,kBAAkBH,GAAW,EAAEJ,OAAyB;IACtE,MAAMQ,WAAW,IAAIH,IAAID;IACzB,MAAMK,WAAW,IAAIJ,IAAIL,QAAQ,WAAW;IAC5C,IACEQ,SAAS,MAAM,KAAKC,SAAS,MAAM,IACnCD,SAAS,QAAQ,KAAKC,SAAS,QAAQ,EAEvC;IAEF,MAAMC,SAASF,SAAS,YAAY;IACpC,IACEE,AAAkC,MAAlCA,OAAO,MAAM,CAAC,SAAS,MAAM,IAC7BA,OAAO,GAAG,CAAC,aAAaV,QAAQ,KAAK,EAErC;IAEF,IAAIC,KAAK,GAAG,MAAMD,QAAQ,SAAS,EACjC,MAAM,IAAIW,MAAM;IAElB,IAAID,OAAO,GAAG,CAAC,UACb,MAAM,IAAIC,MACR,CAAC,6BAA6B,EAAED,OAAO,GAAG,CAAC,wBAAwBA,OAAO,GAAG,CAAC,UAAU;IAG5F,MAAME,OAAOF,OAAO,GAAG,CAAC;IACxB,MAAMhB,WAAWgB,OAAO,GAAG,CAAC,gBAAgBV,QAAQ,QAAQ;IAC5D,IACE,CAACY,QACD,CAAClB,YACDA,AAAa,2BAAbA,YACAgB,AAAiC,MAAjCA,OAAO,MAAM,CAAC,QAAQ,MAAM,IAC5BA,OAAO,MAAM,CAAC,aAAa,MAAM,GAAG,KACnCV,QAAQ,QAAQ,IAAIN,aAAaM,QAAQ,QAAQ,EAElD,MAAM,IAAIW,MAAM;IAElB,OAAO;QAAEC;QAAMlB;IAAS;AAC1B;AAEO,eAAemB,iBACpBb,OAAyB,EACzBQ,QAA4C,EAC5CM,MAAmB,EACnBC,YAA6D;IAE7DD,OAAO,cAAc;IACrB,MAAME,WAAW,MAAMC,AAAAA,IAAAA,mCAAAA,aAAAA,AAAAA,EAAc,kBAAkB;QACrD,MAAMD,WAAW,MAAOD,AAAAA,CAAAA,cAAc,SAASG,oCAAAA,SAAQ,AAARA,EAC7C,GAAGjC,OAAO,yBAAyB,CAAC,EACpC;YACE,QAAQ;YACR,SAAS;gBAAE,gBAAgB;YAAoC;YAC/D,MAAM,IAAIqB,gBAAgB;gBACxB,YAAY;gBACZ,WAAWE,SAAS,QAAQ;gBAC5B,MAAMA,SAAS,IAAI;gBACnB,eAAeR,QAAQ,QAAQ;gBAC/B,cAAcA,QAAQ,WAAW;gBACjC,UAAUd;YACZ;YACA4B;YACA,UAAU;QACZ;QAEF,IAAI,CAACE,SAAS,EAAE,EACd,MAAM,MAAMG,AAAAA,IAAAA,mCAAAA,kBAAAA,AAAAA,EAAmBH;QAEjC,OAAOA;IACT;IACA,MAAMI,SAAS,MAAMH,AAAAA,IAAAA,mCAAAA,aAAAA,AAAAA,EAAc,0BAA0B,IAC3DD,SAAS,IAAI;IAEf,IACE,CAACI,UACD,AAA+B,YAA/B,OAAOA,OAAO,YAAY,IAC1B,CAACA,OAAO,YAAY,IACpB,AAAgC,YAAhC,OAAOA,OAAO,aAAa,IAC3B,CAACA,OAAO,aAAa,IACrB,AAA2B,YAA3B,OAAOA,OAAO,QAAQ,IACtB,AAAwB,YAAxB,OAAOA,OAAO,KAAK,IACnB,AAA6B,YAA7B,OAAOA,OAAO,UAAU,IACxB,CAACC,OAAO,QAAQ,CAACD,OAAO,UAAU,KAClCA,OAAO,UAAU,IAAI,KACrB,AAA6B,YAA7B,OAAOA,OAAO,UAAU,IACxBA,AAAoC,aAApCA,OAAO,UAAU,CAAC,WAAW,IAE7B,MAAM,IAAIT,MAAM;IAElB,MAAMW,SAASF,OAAO,KAAK,CAAC,KAAK,CAAC;IAClC,IAAI,CAACE,OAAO,QAAQ,CAAC,8BACnB,MAAM,IAAIX,MAAM;IAElB,MAAM,EAAEY,OAAO,EAAE,GAAG,MAAMN,AAAAA,IAAAA,mCAAAA,aAAAA,AAAAA,EAAc,yBAAyB,IAC/DO,AAAAA,IAAAA,8BAAAA,SAAAA,AAAAA,EACEJ,OAAO,QAAQ,EACfL,cAAc,QAAQU,AAAAA,IAAAA,oCAAAA,gBAAAA,AAAAA,EAAiBxC,QAAQ6B,SAC/C;YACE,QAAQ7B;YACR,UAAUuB,SAAS,QAAQ;YAC3B,YAAY;gBAAC;aAAQ;YAIrB,gBAAgBa;YAChB,gBAAgB;gBAAC;gBAAO;gBAAO;gBAAO;aAAQ;QAChD;IAGJ,IAAIE,QAAQ,KAAK,KAAKvB,QAAQ,KAAK,IAAI,CAACuB,QAAQ,GAAG,EACjD,MAAM,IAAIZ,MAAM;IAElBG,OAAO,cAAc;IACrB,OAAO;QACL,GAAI,AAAyB,YAAzB,OAAOS,QAAQ,KAAK,GAAgB;YAAE,OAAOA,QAAQ,KAAK;QAAC,IAAI,CAAC,CAAC;QACrE,mBAAmBvB,QAAQ,MAAM;QACjC,WAAWQ,SAAS,QAAQ;QAC5B,SAASe,QAAQ,GAAG;QACpB,QAAQtC;QACR,cAAcmC,OAAO,YAAY;QACjC,eAAeA,OAAO,aAAa;QACnC,UAAUA,OAAO,QAAQ;QACzBE;QACA,YAAYrB,KAAK,GAAG,KAAKmB,AAAoB,OAApBA,OAAO,UAAU;IAC5C;AACF"}

package/dist/lib/ai-model/service-caller/openai/siwc/network.js

@@ -0,0 +1,73 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    createSiwcJWKSet: ()=>createSiwcJWKSet,
+    fetchSiwc: ()=>fetchSiwc
+});
+const utils_namespaceObject = require("@midscene/shared/utils");
+const external_jose_namespaceObject = require("jose");
+const index_js_namespaceObject = require("../../proxy/index.js");
+async function fetchSiwc(input, init, proxy) {
+    if (utils_namespaceObject.ifInBrowser) return fetch(input, init);
+    const { httpProxy, socksProxy } = proxy ?? {
+        httpProxy: process.env.MIDSCENE_MODEL_HTTP_PROXY?.trim() || void 0,
+        socksProxy: process.env.MIDSCENE_MODEL_SOCKS_PROXY?.trim() || void 0
+    };
+    const dispatcher = await (0, index_js_namespaceObject.createProxyAgentIfNeeded)({
+        httpProxy,
+        socksProxy
+    });
+    if (!dispatcher) return fetch(input, init);
+    try {
+        return await fetch(input, {
+            ...init,
+            dispatcher
+        });
+    } finally{
+        dispatcher.close();
+    }
+}
+function createSiwcJWKSet(issuer, signal, request = fetchSiwc) {
+    return async (header, token)=>{
+        const response = await request(`${issuer}/.well-known/jwks.json`, {
+            signal,
+            redirect: 'error'
+        });
+        if (!response.ok) throw new Error(`Failed to fetch OpenAI JWKS: HTTP ${response.status}`);
+        return (0, external_jose_namespaceObject.createLocalJWKSet)(await response.json())(header, token);
+    };
+}
+exports.createSiwcJWKSet = __webpack_exports__.createSiwcJWKSet;
+exports.fetchSiwc = __webpack_exports__.fetchSiwc;
+for(var __rspack_i in __webpack_exports__)if (-1 === [
+    "createSiwcJWKSet",
+    "fetchSiwc"
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});
+
+//# sourceMappingURL=network.js.map

package/dist/lib/ai-model/service-caller/openai/siwc/network.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/service-caller/openai/siwc/network.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../../../src/ai-model/service-caller/openai/siwc/network.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { ifInBrowser } from '@midscene/shared/utils';\nimport { type JWTVerifyGetKey, createLocalJWKSet } from 'jose';\nimport type { Dispatcher } from 'undici';\nimport { createProxyAgentIfNeeded } from '../../proxy';\n\nexport interface SiwcProxyConfig {\n  httpProxy?: string;\n  socksProxy?: string;\n}\n\nexport async function fetchSiwc(\n  input: Parameters<typeof fetch>[0],\n  init?: RequestInit,\n  proxy?: SiwcProxyConfig,\n) {\n  if (ifInBrowser) {\n    return fetch(input, init);\n  }\n  const { httpProxy, socksProxy } = proxy ?? {\n    httpProxy: process.env.MIDSCENE_MODEL_HTTP_PROXY?.trim() || undefined,\n    socksProxy: process.env.MIDSCENE_MODEL_SOCKS_PROXY?.trim() || undefined,\n  };\n  const dispatcher = (await createProxyAgentIfNeeded({\n    httpProxy,\n    socksProxy,\n  })) as Dispatcher | undefined;\n  if (!dispatcher) {\n    return fetch(input, init);\n  }\n  try {\n    return await fetch(input, { ...init, dispatcher } as RequestInit);\n  } finally {\n    // Graceful close waits for the response body to be consumed by the caller.\n    void dispatcher.close();\n  }\n}\n\nexport function createSiwcJWKSet(\n  issuer: string,\n  signal: AbortSignal,\n  request: typeof fetch = fetchSiwc,\n): JWTVerifyGetKey {\n  return async (header, token) => {\n    const response = await request(`${issuer}/.well-known/jwks.json`, {\n      signal,\n      redirect: 'error',\n    });\n    if (!response.ok) {\n      throw new Error(`Failed to fetch OpenAI JWKS: HTTP ${response.status}`);\n    }\n    return createLocalJWKSet(await response.json())(header, token);\n  };\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","fetchSiwc","input","init","proxy","ifInBrowser","fetch","httpProxy","socksProxy","process","undefined","dispatcher","createProxyAgentIfNeeded","createSiwcJWKSet","issuer","signal","request","header","token","response","Error","createLocalJWKSet"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;ACIO,eAAeI,UACpBC,KAAkC,EAClCC,IAAkB,EAClBC,KAAuB;IAEvB,IAAIC,sBAAAA,WAAWA,EACb,OAAOC,MAAMJ,OAAOC;IAEtB,MAAM,EAAEI,SAAS,EAAEC,UAAU,EAAE,GAAGJ,SAAS;QACzC,WAAWK,QAAQ,GAAG,CAAC,yBAAyB,EAAE,UAAUC;QAC5D,YAAYD,QAAQ,GAAG,CAAC,0BAA0B,EAAE,UAAUC;IAChE;IACA,MAAMC,aAAc,MAAMC,AAAAA,IAAAA,yBAAAA,wBAAAA,AAAAA,EAAyB;QACjDL;QACAC;IACF;IACA,IAAI,CAACG,YACH,OAAOL,MAAMJ,OAAOC;IAEtB,IAAI;QACF,OAAO,MAAMG,MAAMJ,OAAO;YAAE,GAAGC,IAAI;YAAEQ;QAAW;IAClD,SAAU;QAEHA,WAAW,KAAK;IACvB;AACF;AAEO,SAASE,iBACdC,MAAc,EACdC,MAAmB,EACnBC,UAAwBf,SAAS;IAEjC,OAAO,OAAOgB,QAAQC;QACpB,MAAMC,WAAW,MAAMH,QAAQ,GAAGF,OAAO,sBAAsB,CAAC,EAAE;YAChEC;YACA,UAAU;QACZ;QACA,IAAI,CAACI,SAAS,EAAE,EACd,MAAM,IAAIC,MAAM,CAAC,kCAAkC,EAAED,SAAS,MAAM,EAAE;QAExE,OAAOE,AAAAA,IAAAA,8BAAAA,iBAAAA,AAAAA,EAAkB,MAAMF,SAAS,IAAI,IAAIF,QAAQC;IAC1D;AACF"}

package/dist/lib/ai-model/service-caller/openai/siwc/refresh.js

@@ -28,13 +28,15 @@ __webpack_require__.d(__webpack_exports__, {
 });
 const external_jose_namespaceObject = require("jose");
 const external_errors_js_namespaceObject = require("./errors.js");
+const external_network_js_namespaceObject = require("./network.js");
 async function refreshSiwcCredentials(saved, signal, dependencies) {
     const { client_id: clientId, refresh_token: refreshToken } = saved;
     if (!clientId || !refreshToken) throw new Error('Missing client ID or refresh token.');
+    const request = dependencies?.fetch ?? external_network_js_namespaceObject.fetchSiwc;
     const issuer = 'https://auth.openai.com';
     signal.throwIfAborted();
     const response = await (0, external_errors_js_namespaceObject.withSiwcStage)('Token refresh', async ()=>{
-        const response = await (dependencies?.fetch ?? fetch)(`${issuer}/api/accounts/oauth/token`, {
+        const response = await request(`${issuer}/api/accounts/oauth/token`, {
             method: 'POST',
             headers: {
                 'Content-Type': 'application/x-www-form-urlencoded'
@@ -57,7 +59,7 @@ async function refreshSiwcCredentials(saved, signal, dependencies) {
     if (!scopes.includes('chatgpt.tokens.use.direct')) throw new Error('ChatGPT plan usage is no longer authorized.');
     if (void 0 !== tokens.id_token) {
         if ('string' != typeof tokens.id_token || !tokens.id_token) throw new Error('Invalid refreshed ID token.');
-        const keys = dependencies?.keys ?? (0, external_jose_namespaceObject.createRemoteJWKSet)(new URL(`${issuer}/.well-known/jwks.json`));
+        const keys = dependencies?.keys ?? (0, external_network_js_namespaceObject.createSiwcJWKSet)(issuer, signal, request);
         const { payload } = await (0, external_errors_js_namespaceObject.withSiwcStage)('ID token verification', ()=>(0, external_jose_namespaceObject.jwtVerify)(tokens.id_token, keys, {
                 issuer,
                 audience: saved.client_id,

package/dist/lib/ai-model/service-caller/openai/siwc/refresh.js.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/service-caller/openai/siwc/refresh.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../../../src/ai-model/service-caller/openai/siwc/refresh.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { type JWTVerifyGetKey, createRemoteJWKSet, jwtVerify } from 'jose';\nimport { tokenResponseError, withSiwcStage } from './errors';\ninterface RefreshableCredentials {\n  client_id?: string;\n  refresh_token?: string;\n  subject?: string;\n  id_token?: string;\n}\n\nexport async function refreshSiwcCredentials<T extends RefreshableCredentials>(\n  saved: T,\n  signal: AbortSignal,\n  dependencies?: { fetch: typeof fetch; keys: JWTVerifyGetKey },\n) {\n  const { client_id: clientId, refresh_token: refreshToken } = saved;\n  if (!clientId || !refreshToken) {\n    throw new Error('Missing client ID or refresh token.');\n  }\n  const issuer = 'https://auth.openai.com';\n  signal.throwIfAborted();\n  const response = await withSiwcStage('Token refresh', async () => {\n    const response = await (dependencies?.fetch ?? fetch)(\n      `${issuer}/api/accounts/oauth/token`,\n      {\n        method: 'POST',\n        headers: { 'Content-Type': 'application/x-www-form-urlencoded' },\n        body: new URLSearchParams({\n          grant_type: 'refresh_token',\n          client_id: clientId,\n          refresh_token: refreshToken,\n          resource: 'https://api.openai.com/v1',\n        }),\n        signal,\n        redirect: 'error',\n      },\n    );\n    if (!response.ok) {\n      throw await tokenResponseError(response);\n    }\n    return response;\n  });\n  const tokens = await withSiwcStage('Refresh response parsing', () =>\n    response.json(),\n  );\n  if (\n    !tokens ||\n    typeof tokens.access_token !== 'string' ||\n    !tokens.access_token.trim() ||\n    typeof tokens.refresh_token !== 'string' ||\n    !tokens.refresh_token.trim() ||\n    typeof tokens.expires_in !== 'number' ||\n    !Number.isFinite(tokens.expires_in) ||\n    tokens.expires_in <= 0 ||\n    typeof tokens.token_type !== 'string' ||\n    tokens.token_type.toLowerCase() !== 'bearer' ||\n    typeof tokens.scope !== 'string'\n  ) {\n    throw new Error('OpenAI returned an invalid refresh response.');\n  }\n  const scopes = tokens.scope.split(/\\s+/);\n  if (!scopes.includes('chatgpt.tokens.use.direct')) {\n    throw new Error('ChatGPT plan usage is no longer authorized.');\n  }\n  if (tokens.id_token !== undefined) {\n    if (typeof tokens.id_token !== 'string' || !tokens.id_token) {\n      throw new Error('Invalid refreshed ID token.');\n    }\n    const keys =\n      dependencies?.keys ??\n      createRemoteJWKSet(new URL(`${issuer}/.well-known/jwks.json`));\n    const { payload } = await withSiwcStage('ID token verification', () =>\n      jwtVerify(tokens.id_token, keys, {\n        issuer,\n        audience: saved.client_id,\n        algorithms: ['RS256'],\n        // Some CI machines have incorrect system clocks. Do not compare ID token\n        // exp/nbf claims against local time when accepting newly returned tokens.\n        // Keep signature, issuer, audience and required-claim validation.\n        clockTolerance: Number.POSITIVE_INFINITY,\n        requiredClaims: ['sub', 'exp', 'iat'],\n      }),\n    );\n    if (payload.sub !== saved.subject) {\n      throw new Error('Refreshed identity does not match the saved account.');\n    }\n  }\n  signal.throwIfAborted();\n  return {\n    ...saved,\n    access_token: tokens.access_token,\n    refresh_token: tokens.refresh_token,\n    id_token: tokens.id_token ?? saved.id_token,\n    scopes,\n    expires_at: Date.now() + tokens.expires_in * 1000,\n  };\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","refreshSiwcCredentials","saved","signal","dependencies","clientId","refreshToken","Error","issuer","response","withSiwcStage","fetch","URLSearchParams","tokenResponseError","tokens","Number","scopes","undefined","keys","createRemoteJWKSet","URL","payload","jwtVerify","Date"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;ACGO,eAAeI,uBACpBC,KAAQ,EACRC,MAAmB,EACnBC,YAA6D;IAE7D,MAAM,EAAE,WAAWC,QAAQ,EAAE,eAAeC,YAAY,EAAE,GAAGJ;IAC7D,IAAI,CAACG,YAAY,CAACC,cAChB,MAAM,IAAIC,MAAM;IAElB,MAAMC,SAAS;IACfL,OAAO,cAAc;IACrB,MAAMM,WAAW,MAAMC,AAAAA,IAAAA,mCAAAA,aAAAA,AAAAA,EAAc,iBAAiB;QACpD,MAAMD,WAAW,MAAOL,AAAAA,CAAAA,cAAc,SAASO,KAAI,EACjD,GAAGH,OAAO,yBAAyB,CAAC,EACpC;YACE,QAAQ;YACR,SAAS;gBAAE,gBAAgB;YAAoC;YAC/D,MAAM,IAAII,gBAAgB;gBACxB,YAAY;gBACZ,WAAWP;gBACX,eAAeC;gBACf,UAAU;YACZ;YACAH;YACA,UAAU;QACZ;QAEF,IAAI,CAACM,SAAS,EAAE,EACd,MAAM,MAAMI,AAAAA,IAAAA,mCAAAA,kBAAAA,AAAAA,EAAmBJ;QAEjC,OAAOA;IACT;IACA,MAAMK,SAAS,MAAMJ,AAAAA,IAAAA,mCAAAA,aAAAA,AAAAA,EAAc,4BAA4B,IAC7DD,SAAS,IAAI;IAEf,IACE,CAACK,UACD,AAA+B,YAA/B,OAAOA,OAAO,YAAY,IAC1B,CAACA,OAAO,YAAY,CAAC,IAAI,MACzB,AAAgC,YAAhC,OAAOA,OAAO,aAAa,IAC3B,CAACA,OAAO,aAAa,CAAC,IAAI,MAC1B,AAA6B,YAA7B,OAAOA,OAAO,UAAU,IACxB,CAACC,OAAO,QAAQ,CAACD,OAAO,UAAU,KAClCA,OAAO,UAAU,IAAI,KACrB,AAA6B,YAA7B,OAAOA,OAAO,UAAU,IACxBA,AAAoC,aAApCA,OAAO,UAAU,CAAC,WAAW,MAC7B,AAAwB,YAAxB,OAAOA,OAAO,KAAK,EAEnB,MAAM,IAAIP,MAAM;IAElB,MAAMS,SAASF,OAAO,KAAK,CAAC,KAAK,CAAC;IAClC,IAAI,CAACE,OAAO,QAAQ,CAAC,8BACnB,MAAM,IAAIT,MAAM;IAElB,IAAIO,AAAoBG,WAApBH,OAAO,QAAQ,EAAgB;QACjC,IAAI,AAA2B,YAA3B,OAAOA,OAAO,QAAQ,IAAiB,CAACA,OAAO,QAAQ,EACzD,MAAM,IAAIP,MAAM;QAElB,MAAMW,OACJd,cAAc,QACde,AAAAA,IAAAA,8BAAAA,kBAAAA,AAAAA,EAAmB,IAAIC,IAAI,GAAGZ,OAAO,sBAAsB,CAAC;QAC9D,MAAM,EAAEa,OAAO,EAAE,GAAG,MAAMX,AAAAA,IAAAA,mCAAAA,aAAAA,AAAAA,EAAc,yBAAyB,IAC/DY,AAAAA,IAAAA,8BAAAA,SAAAA,AAAAA,EAAUR,OAAO,QAAQ,EAAEI,MAAM;gBAC/BV;gBACA,UAAUN,MAAM,SAAS;gBACzB,YAAY;oBAAC;iBAAQ;gBAIrB,gBAAgBa;gBAChB,gBAAgB;oBAAC;oBAAO;oBAAO;iBAAM;YACvC;QAEF,IAAIM,QAAQ,GAAG,KAAKnB,MAAM,OAAO,EAC/B,MAAM,IAAIK,MAAM;IAEpB;IACAJ,OAAO,cAAc;IACrB,OAAO;QACL,GAAGD,KAAK;QACR,cAAcY,OAAO,YAAY;QACjC,eAAeA,OAAO,aAAa;QACnC,UAAUA,OAAO,QAAQ,IAAIZ,MAAM,QAAQ;QAC3Cc;QACA,YAAYO,KAAK,GAAG,KAAKT,AAAoB,OAApBA,OAAO,UAAU;IAC5C;AACF"}
+{"version":3,"file":"ai-model/service-caller/openai/siwc/refresh.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../../../src/ai-model/service-caller/openai/siwc/refresh.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { type JWTVerifyGetKey, jwtVerify } from 'jose';\nimport { tokenResponseError, withSiwcStage } from './errors';\nimport { createSiwcJWKSet, fetchSiwc } from './network';\ninterface RefreshableCredentials {\n  client_id?: string;\n  refresh_token?: string;\n  subject?: string;\n  id_token?: string;\n}\n\nexport async function refreshSiwcCredentials<T extends RefreshableCredentials>(\n  saved: T,\n  signal: AbortSignal,\n  dependencies?: { fetch: typeof fetch; keys?: JWTVerifyGetKey },\n) {\n  const { client_id: clientId, refresh_token: refreshToken } = saved;\n  if (!clientId || !refreshToken) {\n    throw new Error('Missing client ID or refresh token.');\n  }\n  const request = dependencies?.fetch ?? fetchSiwc;\n  const issuer = 'https://auth.openai.com';\n  signal.throwIfAborted();\n  const response = await withSiwcStage('Token refresh', async () => {\n    const response = await request(`${issuer}/api/accounts/oauth/token`, {\n      method: 'POST',\n      headers: { 'Content-Type': 'application/x-www-form-urlencoded' },\n      body: new URLSearchParams({\n        grant_type: 'refresh_token',\n        client_id: clientId,\n        refresh_token: refreshToken,\n        resource: 'https://api.openai.com/v1',\n      }),\n      signal,\n      redirect: 'error',\n    });\n    if (!response.ok) {\n      throw await tokenResponseError(response);\n    }\n    return response;\n  });\n  const tokens = await withSiwcStage('Refresh response parsing', () =>\n    response.json(),\n  );\n  if (\n    !tokens ||\n    typeof tokens.access_token !== 'string' ||\n    !tokens.access_token.trim() ||\n    typeof tokens.refresh_token !== 'string' ||\n    !tokens.refresh_token.trim() ||\n    typeof tokens.expires_in !== 'number' ||\n    !Number.isFinite(tokens.expires_in) ||\n    tokens.expires_in <= 0 ||\n    typeof tokens.token_type !== 'string' ||\n    tokens.token_type.toLowerCase() !== 'bearer' ||\n    typeof tokens.scope !== 'string'\n  ) {\n    throw new Error('OpenAI returned an invalid refresh response.');\n  }\n  const scopes = tokens.scope.split(/\\s+/);\n  if (!scopes.includes('chatgpt.tokens.use.direct')) {\n    throw new Error('ChatGPT plan usage is no longer authorized.');\n  }\n  if (tokens.id_token !== undefined) {\n    if (typeof tokens.id_token !== 'string' || !tokens.id_token) {\n      throw new Error('Invalid refreshed ID token.');\n    }\n    const keys =\n      dependencies?.keys ?? createSiwcJWKSet(issuer, signal, request);\n    const { payload } = await withSiwcStage('ID token verification', () =>\n      jwtVerify(tokens.id_token, keys, {\n        issuer,\n        audience: saved.client_id,\n        algorithms: ['RS256'],\n        // Some CI machines have incorrect system clocks. Do not compare ID token\n        // exp/nbf claims against local time when accepting newly returned tokens.\n        // Keep signature, issuer, audience and required-claim validation.\n        clockTolerance: Number.POSITIVE_INFINITY,\n        requiredClaims: ['sub', 'exp', 'iat'],\n      }),\n    );\n    if (payload.sub !== saved.subject) {\n      throw new Error('Refreshed identity does not match the saved account.');\n    }\n  }\n  signal.throwIfAborted();\n  return {\n    ...saved,\n    access_token: tokens.access_token,\n    refresh_token: tokens.refresh_token,\n    id_token: tokens.id_token ?? saved.id_token,\n    scopes,\n    expires_at: Date.now() + tokens.expires_in * 1000,\n  };\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","refreshSiwcCredentials","saved","signal","dependencies","clientId","refreshToken","Error","request","fetchSiwc","issuer","response","withSiwcStage","URLSearchParams","tokenResponseError","tokens","Number","scopes","undefined","keys","createSiwcJWKSet","payload","jwtVerify","Date"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;ACIO,eAAeI,uBACpBC,KAAQ,EACRC,MAAmB,EACnBC,YAA8D;IAE9D,MAAM,EAAE,WAAWC,QAAQ,EAAE,eAAeC,YAAY,EAAE,GAAGJ;IAC7D,IAAI,CAACG,YAAY,CAACC,cAChB,MAAM,IAAIC,MAAM;IAElB,MAAMC,UAAUJ,cAAc,SAASK,oCAAAA,SAASA;IAChD,MAAMC,SAAS;IACfP,OAAO,cAAc;IACrB,MAAMQ,WAAW,MAAMC,AAAAA,IAAAA,mCAAAA,aAAAA,AAAAA,EAAc,iBAAiB;QACpD,MAAMD,WAAW,MAAMH,QAAQ,GAAGE,OAAO,yBAAyB,CAAC,EAAE;YACnE,QAAQ;YACR,SAAS;gBAAE,gBAAgB;YAAoC;YAC/D,MAAM,IAAIG,gBAAgB;gBACxB,YAAY;gBACZ,WAAWR;gBACX,eAAeC;gBACf,UAAU;YACZ;YACAH;YACA,UAAU;QACZ;QACA,IAAI,CAACQ,SAAS,EAAE,EACd,MAAM,MAAMG,AAAAA,IAAAA,mCAAAA,kBAAAA,AAAAA,EAAmBH;QAEjC,OAAOA;IACT;IACA,MAAMI,SAAS,MAAMH,AAAAA,IAAAA,mCAAAA,aAAAA,AAAAA,EAAc,4BAA4B,IAC7DD,SAAS,IAAI;IAEf,IACE,CAACI,UACD,AAA+B,YAA/B,OAAOA,OAAO,YAAY,IAC1B,CAACA,OAAO,YAAY,CAAC,IAAI,MACzB,AAAgC,YAAhC,OAAOA,OAAO,aAAa,IAC3B,CAACA,OAAO,aAAa,CAAC,IAAI,MAC1B,AAA6B,YAA7B,OAAOA,OAAO,UAAU,IACxB,CAACC,OAAO,QAAQ,CAACD,OAAO,UAAU,KAClCA,OAAO,UAAU,IAAI,KACrB,AAA6B,YAA7B,OAAOA,OAAO,UAAU,IACxBA,AAAoC,aAApCA,OAAO,UAAU,CAAC,WAAW,MAC7B,AAAwB,YAAxB,OAAOA,OAAO,KAAK,EAEnB,MAAM,IAAIR,MAAM;IAElB,MAAMU,SAASF,OAAO,KAAK,CAAC,KAAK,CAAC;IAClC,IAAI,CAACE,OAAO,QAAQ,CAAC,8BACnB,MAAM,IAAIV,MAAM;IAElB,IAAIQ,AAAoBG,WAApBH,OAAO,QAAQ,EAAgB;QACjC,IAAI,AAA2B,YAA3B,OAAOA,OAAO,QAAQ,IAAiB,CAACA,OAAO,QAAQ,EACzD,MAAM,IAAIR,MAAM;QAElB,MAAMY,OACJf,cAAc,QAAQgB,AAAAA,IAAAA,oCAAAA,gBAAAA,AAAAA,EAAiBV,QAAQP,QAAQK;QACzD,MAAM,EAAEa,OAAO,EAAE,GAAG,MAAMT,AAAAA,IAAAA,mCAAAA,aAAAA,AAAAA,EAAc,yBAAyB,IAC/DU,AAAAA,IAAAA,8BAAAA,SAAAA,AAAAA,EAAUP,OAAO,QAAQ,EAAEI,MAAM;gBAC/BT;gBACA,UAAUR,MAAM,SAAS;gBACzB,YAAY;oBAAC;iBAAQ;gBAIrB,gBAAgBc;gBAChB,gBAAgB;oBAAC;oBAAO;oBAAO;iBAAM;YACvC;QAEF,IAAIK,QAAQ,GAAG,KAAKnB,MAAM,OAAO,EAC/B,MAAM,IAAIK,MAAM;IAEpB;IACAJ,OAAO,cAAc;IACrB,OAAO;QACL,GAAGD,KAAK;QACR,cAAca,OAAO,YAAY;QACjC,eAAeA,OAAO,aAAa;QACnC,UAAUA,OAAO,QAAQ,IAAIb,MAAM,QAAQ;QAC3Ce;QACA,YAAYM,KAAK,GAAG,KAAKR,AAAoB,OAApBA,OAAO,UAAU;IAC5C;AACF"}

package/dist/lib/ai-model/service-caller/openai/siwc/token-manager.js

@@ -27,6 +27,7 @@ __webpack_require__.d(__webpack_exports__, {
     siwcTokenManager: ()=>siwcTokenManager
 });
 const external_credential_storage_js_namespaceObject = require("./credential-storage.js");
+const external_network_js_namespaceObject = require("./network.js");
 const external_refresh_js_namespaceObject = require("./refresh.js");
 function _define_property(obj, key, value) {
     if (key in obj) Object.defineProperty(obj, key, {
@@ -44,19 +45,21 @@ class SiwcTokenManager {
         const record = await storage.read();
         return record.credentials.access_token;
     }
-    refreshAccessToken(previousAccessToken) {
-        this.pendingRefresh ??= this.refreshAndSave(previousAccessToken).finally(()=>{
+    refreshAccessToken(previousAccessToken, proxy = {}) {
+        this.pendingRefresh ??= this.refreshAndSave(previousAccessToken, proxy).finally(()=>{
             this.pendingRefresh = void 0;
         });
         return this.pendingRefresh;
     }
-    async refreshAndSave(previousAccessToken) {
+    async refreshAndSave(previousAccessToken, proxy) {
         const storage = await (0, external_credential_storage_js_namespaceObject.getCredentialStorage)();
         const record = await storage.read();
         const saved = record.credentials;
         if (saved.access_token !== previousAccessToken) return saved.access_token;
         try {
-            const credentials = await (0, external_refresh_js_namespaceObject.refreshSiwcCredentials)(saved, AbortSignal.timeout(30000));
+            const credentials = await (0, external_refresh_js_namespaceObject.refreshSiwcCredentials)(saved, AbortSignal.timeout(30000), {
+                fetch: (input, init)=>(0, external_network_js_namespaceObject.fetchSiwc)(input, init, proxy)
+            });
             await storage.save({
                 ...record,
                 updated_at: new Date().toISOString(),

package/dist/lib/ai-model/service-caller/openai/siwc/token-manager.js.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/service-caller/openai/siwc/token-manager.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../../../src/ai-model/service-caller/openai/siwc/token-manager.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { getCredentialStorage } from './credential-storage';\nimport { refreshSiwcCredentials } from './refresh';\n\nclass SiwcTokenManager {\n  private pendingRefresh?: Promise<string>;\n\n  async getAccessToken(): Promise<string> {\n    const storage = await getCredentialStorage();\n    const record = await storage.read();\n    // Some CI machines have incorrect system clocks. Let the server reject\n    // expired tokens instead of comparing expires_at against local time.\n    return record.credentials.access_token;\n  }\n\n  /** Only refresh operations are shared; ordinary reads remain independent. */\n  refreshAccessToken(previousAccessToken: string): Promise<string> {\n    this.pendingRefresh ??= this.refreshAndSave(previousAccessToken).finally(\n      () => {\n        this.pendingRefresh = undefined;\n      },\n    );\n    return this.pendingRefresh;\n  }\n\n  private async refreshAndSave(previousAccessToken: string): Promise<string> {\n    // Re-read after acquiring the refresh slot: another caller may have saved\n    // a new token since this request first read its credentials.\n    const storage = await getCredentialStorage();\n    const record = await storage.read();\n    const saved = record.credentials;\n    if (saved.access_token !== previousAccessToken) {\n      return saved.access_token;\n    }\n    try {\n      const credentials = await refreshSiwcCredentials(\n        saved,\n        AbortSignal.timeout(30_000),\n      );\n      await storage.save({\n        ...record,\n        updated_at: new Date().toISOString(),\n        credentials,\n      });\n      return credentials.access_token;\n    } catch (cause) {\n      throw new Error(\n        `Unable to refresh SIWC credentials: ${cause instanceof Error ? cause.message : String(cause)} ${storage.reauthorizationHint}`,\n        { cause },\n      );\n    }\n  }\n}\n\nexport const siwcTokenManager = new SiwcTokenManager();\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","SiwcTokenManager","storage","getCredentialStorage","record","previousAccessToken","undefined","saved","credentials","refreshSiwcCredentials","AbortSignal","Date","cause","Error","String","siwcTokenManager"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;ACHA,MAAMI;IAGJ,MAAM,iBAAkC;QACtC,MAAMC,UAAU,MAAMC,AAAAA,IAAAA,+CAAAA,oBAAAA,AAAAA;QACtB,MAAMC,SAAS,MAAMF,QAAQ,IAAI;QAGjC,OAAOE,OAAO,WAAW,CAAC,YAAY;IACxC;IAGA,mBAAmBC,mBAA2B,EAAmB;QAC/D,IAAI,CAAC,cAAc,KAAK,IAAI,CAAC,cAAc,CAACA,qBAAqB,OAAO,CACtE;YACE,IAAI,CAAC,cAAc,GAAGC;QACxB;QAEF,OAAO,IAAI,CAAC,cAAc;IAC5B;IAEA,MAAc,eAAeD,mBAA2B,EAAmB;QAGzE,MAAMH,UAAU,MAAMC,AAAAA,IAAAA,+CAAAA,oBAAAA,AAAAA;QACtB,MAAMC,SAAS,MAAMF,QAAQ,IAAI;QACjC,MAAMK,QAAQH,OAAO,WAAW;QAChC,IAAIG,MAAM,YAAY,KAAKF,qBACzB,OAAOE,MAAM,YAAY;QAE3B,IAAI;YACF,MAAMC,cAAc,MAAMC,AAAAA,IAAAA,oCAAAA,sBAAAA,AAAAA,EACxBF,OACAG,YAAY,OAAO,CAAC;YAEtB,MAAMR,QAAQ,IAAI,CAAC;gBACjB,GAAGE,MAAM;gBACT,YAAY,IAAIO,OAAO,WAAW;gBAClCH;YACF;YACA,OAAOA,YAAY,YAAY;QACjC,EAAE,OAAOI,OAAO;YACd,MAAM,IAAIC,MACR,CAAC,oCAAoC,EAAED,iBAAiBC,QAAQD,MAAM,OAAO,GAAGE,OAAOF,OAAO,CAAC,EAAEV,QAAQ,mBAAmB,EAAE,EAC9H;gBAAEU;YAAM;QAEZ;IACF;;QA9CA,uBAAQ,kBAAR;;AA+CF;AAEO,MAAMG,mBAAmB,IAAId"}
+{"version":3,"file":"ai-model/service-caller/openai/siwc/token-manager.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../../../src/ai-model/service-caller/openai/siwc/token-manager.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { getCredentialStorage } from './credential-storage';\nimport { type SiwcProxyConfig, fetchSiwc } from './network';\nimport { refreshSiwcCredentials } from './refresh';\n\nclass SiwcTokenManager {\n  private pendingRefresh?: Promise<string>;\n\n  async getAccessToken(): Promise<string> {\n    const storage = await getCredentialStorage();\n    const record = await storage.read();\n    // Some CI machines have incorrect system clocks. Let the server reject\n    // expired tokens instead of comparing expires_at against local time.\n    return record.credentials.access_token;\n  }\n\n  /** Only refresh operations are shared; ordinary reads remain independent. */\n  refreshAccessToken(\n    previousAccessToken: string,\n    proxy: SiwcProxyConfig = {},\n  ): Promise<string> {\n    this.pendingRefresh ??= this.refreshAndSave(\n      previousAccessToken,\n      proxy,\n    ).finally(() => {\n      this.pendingRefresh = undefined;\n    });\n    return this.pendingRefresh;\n  }\n\n  private async refreshAndSave(\n    previousAccessToken: string,\n    proxy: SiwcProxyConfig,\n  ): Promise<string> {\n    // Re-read after acquiring the refresh slot: another caller may have saved\n    // a new token since this request first read its credentials.\n    const storage = await getCredentialStorage();\n    const record = await storage.read();\n    const saved = record.credentials;\n    if (saved.access_token !== previousAccessToken) {\n      return saved.access_token;\n    }\n    try {\n      const credentials = await refreshSiwcCredentials(\n        saved,\n        AbortSignal.timeout(30_000),\n        { fetch: (input, init) => fetchSiwc(input, init, proxy) },\n      );\n      await storage.save({\n        ...record,\n        updated_at: new Date().toISOString(),\n        credentials,\n      });\n      return credentials.access_token;\n    } catch (cause) {\n      throw new Error(\n        `Unable to refresh SIWC credentials: ${cause instanceof Error ? cause.message : String(cause)} ${storage.reauthorizationHint}`,\n        { cause },\n      );\n    }\n  }\n}\n\nexport const siwcTokenManager = new SiwcTokenManager();\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","SiwcTokenManager","storage","getCredentialStorage","record","previousAccessToken","proxy","undefined","saved","credentials","refreshSiwcCredentials","AbortSignal","input","init","fetchSiwc","Date","cause","Error","String","siwcTokenManager"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;ACFA,MAAMI;IAGJ,MAAM,iBAAkC;QACtC,MAAMC,UAAU,MAAMC,AAAAA,IAAAA,+CAAAA,oBAAAA,AAAAA;QACtB,MAAMC,SAAS,MAAMF,QAAQ,IAAI;QAGjC,OAAOE,OAAO,WAAW,CAAC,YAAY;IACxC;IAGA,mBACEC,mBAA2B,EAC3BC,QAAyB,CAAC,CAAC,EACV;QACjB,IAAI,CAAC,cAAc,KAAK,IAAI,CAAC,cAAc,CACzCD,qBACAC,OACA,OAAO,CAAC;YACR,IAAI,CAAC,cAAc,GAAGC;QACxB;QACA,OAAO,IAAI,CAAC,cAAc;IAC5B;IAEA,MAAc,eACZF,mBAA2B,EAC3BC,KAAsB,EACL;QAGjB,MAAMJ,UAAU,MAAMC,AAAAA,IAAAA,+CAAAA,oBAAAA,AAAAA;QACtB,MAAMC,SAAS,MAAMF,QAAQ,IAAI;QACjC,MAAMM,QAAQJ,OAAO,WAAW;QAChC,IAAII,MAAM,YAAY,KAAKH,qBACzB,OAAOG,MAAM,YAAY;QAE3B,IAAI;YACF,MAAMC,cAAc,MAAMC,AAAAA,IAAAA,oCAAAA,sBAAAA,AAAAA,EACxBF,OACAG,YAAY,OAAO,CAAC,QACpB;gBAAE,OAAO,CAACC,OAAOC,OAASC,AAAAA,IAAAA,oCAAAA,SAAAA,AAAAA,EAAUF,OAAOC,MAAMP;YAAO;YAE1D,MAAMJ,QAAQ,IAAI,CAAC;gBACjB,GAAGE,MAAM;gBACT,YAAY,IAAIW,OAAO,WAAW;gBAClCN;YACF;YACA,OAAOA,YAAY,YAAY;QACjC,EAAE,OAAOO,OAAO;YACd,MAAM,IAAIC,MACR,CAAC,oCAAoC,EAAED,iBAAiBC,QAAQD,MAAM,OAAO,GAAGE,OAAOF,OAAO,CAAC,EAAEd,QAAQ,mBAAmB,EAAE,EAC9H;gBAAEc;YAAM;QAEZ;IACF;;QAtDA,uBAAQ,kBAAR;;AAuDF;AAEO,MAAMG,mBAAmB,IAAIlB"}

package/dist/lib/ai-model/shared/json.js

@@ -31,9 +31,13 @@ __webpack_require__.d(__webpack_exports__, {
 const external_jsonrepair_namespaceObject = require("jsonrepair");
 function extractJSONFromCodeBlock(response) {
     try {
-        const jsonMatch = response.match(/^\s*(\{[\s\S]*\})\s*$/);
+        JSON.parse(response);
+        return response.trim();
+    } catch  {}
+    try {
+        const jsonMatch = response.match(/^\s*(\{[\s\S]*\}|\[[\s\S]*\])\s*$/);
         if (jsonMatch) return jsonMatch[1];
-        const codeBlockMatch = response.match(/```(?:json)?\s*(\{[\s\S]*?\})\s*```/);
+        const codeBlockMatch = response.match(/```(?:json)?\s*([\s\S]*?)\s*```/i);
         if (codeBlockMatch) return codeBlockMatch[1];
         const jsonLikeMatch = response.match(/\{[\s\S]*\}/);
         if (jsonLikeMatch) return jsonLikeMatch[0];

package/dist/lib/ai-model/shared/json.js.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/shared/json.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../src/ai-model/shared/json.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { jsonrepair } from 'jsonrepair';\n\n/**\n * Extract the JSON portion from a model response.\n *\n * This mainly handles fenced JSON like ```json { ... } ``` by removing the\n * fence. If that does not produce a JSON object, it falls back to a greedy regex\n * that extracts from the first \"{\" to the last \"}\". If that still fails, the\n * original response is returned.\n *\n * Expected model responses are JSON objects, possibly wrapped in markdown\n * fences. Natural language mixed with JSON, or arrays like\n * [{\"type\":\"Tap\"}, {\"type\":\"Hover\"}], are outside the supported contract and\n * are not reliably recoverable.\n *\n * This extractor is also used by extraction responses that can be any\n * JSON value. In those cases, arrays/strings/numbers have no object braces and\n * pass through unchanged.\n */\nexport function extractJSONFromCodeBlock(response: string) {\n  try {\n    // First, try to match a JSON object directly in the response\n    const jsonMatch = response.match(/^\\s*(\\{[\\s\\S]*\\})\\s*$/);\n    if (jsonMatch) {\n      return jsonMatch[1];\n    }\n\n    // If no direct JSON object is found, try to extract JSON from a code block\n    const codeBlockMatch = response.match(\n      /```(?:json)?\\s*(\\{[\\s\\S]*?\\})\\s*```/,\n    );\n    if (codeBlockMatch) {\n      return codeBlockMatch[1];\n    }\n\n    // If no code block is found, try to find a JSON-like structure in the text\n    const jsonLikeMatch = response.match(/\\{[\\s\\S]*\\}/);\n    if (jsonLikeMatch) {\n      return jsonLikeMatch[0];\n    }\n  } catch {}\n  // If no JSON-like structure is found, return the original response\n  return response;\n}\n\nexport type JsonParserSource =\n  | 'generic-object'\n  | 'planning-action-param'\n  | 'locate'\n  | 'section-locator';\n\nexport interface JsonParserContext {\n  source: JsonParserSource;\n  preserveStringValueKeys?: string[];\n}\n\nexport type JsonParser = (raw: string, context?: JsonParserContext) => unknown;\n\n/**\n * Trim whitespace in parsed JSON by normalizing:\n * 1. All object keys (e.g., \" prompt \" -> \"prompt\")\n * 2. All string values (e.g., \" Tap \" -> \"Tap\")\n * This handles LLM output that may include leading/trailing spaces.\n */\nfunction trimParsedJsonStrings(\n  obj: any,\n  context: Pick<JsonParserContext, 'preserveStringValueKeys'> = {},\n): any {\n  // Handle null and undefined\n  if (obj === null || obj === undefined) {\n    return obj;\n  }\n\n  // Handle arrays - recursively normalize each element\n  if (Array.isArray(obj)) {\n    return obj.map((item) => trimParsedJsonStrings(item, context));\n  }\n\n  // Handle objects\n  if (typeof obj === 'object') {\n    const normalized: any = {};\n\n    for (const [key, value] of Object.entries(obj)) {\n      // Trim the key to remove leading/trailing spaces\n      const trimmedKey = key.trim();\n      const preserveStringValue =\n        context.preserveStringValueKeys?.includes(trimmedKey) ?? false;\n\n      const normalizedValue =\n        typeof value === 'string'\n          ? preserveStringValue\n            ? value\n            : value.trim()\n          : trimParsedJsonStrings(value, context);\n\n      normalized[trimmedKey] = normalizedValue;\n    }\n\n    return normalized;\n  }\n\n  // Handle primitive strings\n  if (typeof obj === 'string') {\n    return obj.trim();\n  }\n\n  // Return other primitives as-is\n  return obj;\n}\n\nfunction repairKnownJsonIssues(\n  jsonBlock: string,\n  _rawResponse: string,\n): string {\n  // TODO: Add project-specific repairs that jsonrepair cannot handle.\n  return jsonBlock;\n}\n\nexport function assertJsonObject(\n  parsed: unknown,\n): asserts parsed is Record<string, unknown> {\n  if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {\n    const parsedType = Array.isArray(parsed)\n      ? 'array'\n      : parsed === null\n        ? 'null'\n        : typeof parsed;\n    throw new Error(\n      `LLM response is valid JSON but does not match the expected schema. Expected to be a JSON object, got ${parsedType}: ${JSON.stringify(parsed)}.`,\n    );\n  }\n}\n\nfunction parseJsonWithRepair(jsonStr: string) {\n  try {\n    return JSON.parse(jsonStr);\n  } catch {\n    return JSON.parse(jsonrepair(jsonStr));\n  }\n}\n\nexport function parseModelResponseJson(\n  raw: string,\n  context?: JsonParserContext,\n) {\n  const cleanJsonString = extractJSONFromCodeBlock(raw);\n\n  let parsedObj: unknown;\n\n  try {\n    parsedObj = parseJsonWithRepair(cleanJsonString);\n  } catch (e1) {\n    const code = repairKnownJsonIssues(cleanJsonString, raw);\n    if (code === cleanJsonString) {\n      throw new Error(\n        `failed to parse LLM response into JSON. Error - ${String(\n          e1,\n        )}. Response - \\n ${raw}`,\n      );\n    }\n\n    try {\n      parsedObj = parseJsonWithRepair(code);\n    } catch (e2) {\n      throw new Error(\n        `failed to parse LLM response into JSON. First error - ${String(\n          e1,\n        )}. Second error - ${String(e2)}. Response - \\n ${raw}`,\n      );\n    }\n  }\n\n  return trimParsedJsonStrings(parsedObj, context);\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","extractJSONFromCodeBlock","response","jsonMatch","codeBlockMatch","jsonLikeMatch","trimParsedJsonStrings","context","Array","item","normalized","value","trimmedKey","preserveStringValue","normalizedValue","repairKnownJsonIssues","jsonBlock","_rawResponse","assertJsonObject","parsed","parsedType","Error","JSON","parseJsonWithRepair","jsonStr","jsonrepair","parseModelResponseJson","raw","cleanJsonString","parsedObj","e1","code","String","e2"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;ACaO,SAASI,yBAAyBC,QAAgB;IACvD,IAAI;QAEF,MAAMC,YAAYD,SAAS,KAAK,CAAC;QACjC,IAAIC,WACF,OAAOA,SAAS,CAAC,EAAE;QAIrB,MAAMC,iBAAiBF,SAAS,KAAK,CACnC;QAEF,IAAIE,gBACF,OAAOA,cAAc,CAAC,EAAE;QAI1B,MAAMC,gBAAgBH,SAAS,KAAK,CAAC;QACrC,IAAIG,eACF,OAAOA,aAAa,CAAC,EAAE;IAE3B,EAAE,OAAM,CAAC;IAET,OAAOH;AACT;AAqBA,SAASI,sBACPR,GAAQ,EACRS,UAA8D,CAAC,CAAC;IAGhE,IAAIT,QAAAA,KACF,OAAOA;IAIT,IAAIU,MAAM,OAAO,CAACV,MAChB,OAAOA,IAAI,GAAG,CAAC,CAACW,OAASH,sBAAsBG,MAAMF;IAIvD,IAAI,AAAe,YAAf,OAAOT,KAAkB;QAC3B,MAAMY,aAAkB,CAAC;QAEzB,KAAK,MAAM,CAACd,KAAKe,MAAM,IAAId,OAAO,OAAO,CAACC,KAAM;YAE9C,MAAMc,aAAahB,IAAI,IAAI;YAC3B,MAAMiB,sBACJN,QAAQ,uBAAuB,EAAE,SAASK,eAAe;YAE3D,MAAME,kBACJ,AAAiB,YAAjB,OAAOH,QACHE,sBACEF,QACAA,MAAM,IAAI,KACZL,sBAAsBK,OAAOJ;YAEnCG,UAAU,CAACE,WAAW,GAAGE;QAC3B;QAEA,OAAOJ;IACT;IAGA,IAAI,AAAe,YAAf,OAAOZ,KACT,OAAOA,IAAI,IAAI;IAIjB,OAAOA;AACT;AAEA,SAASiB,sBACPC,SAAiB,EACjBC,YAAoB;IAGpB,OAAOD;AACT;AAEO,SAASE,iBACdC,MAAe;IAEf,IAAI,CAACA,UAAU,AAAkB,YAAlB,OAAOA,UAAuBX,MAAM,OAAO,CAACW,SAAS;QAClE,MAAMC,aAAaZ,MAAM,OAAO,CAACW,UAC7B,UACAA,AAAW,SAAXA,SACE,SACA,OAAOA;QACb,MAAM,IAAIE,MACR,CAAC,qGAAqG,EAAED,WAAW,EAAE,EAAEE,KAAK,SAAS,CAACH,QAAQ,CAAC,CAAC;IAEpJ;AACF;AAEA,SAASI,oBAAoBC,OAAe;IAC1C,IAAI;QACF,OAAOF,KAAK,KAAK,CAACE;IACpB,EAAE,OAAM;QACN,OAAOF,KAAK,KAAK,CAACG,AAAAA,IAAAA,oCAAAA,UAAAA,AAAAA,EAAWD;IAC/B;AACF;AAEO,SAASE,uBACdC,GAAW,EACXpB,OAA2B;IAE3B,MAAMqB,kBAAkB3B,yBAAyB0B;IAEjD,IAAIE;IAEJ,IAAI;QACFA,YAAYN,oBAAoBK;IAClC,EAAE,OAAOE,IAAI;QACX,MAAMC,OAAOhB,sBAAsBa,iBAAiBD;QACpD,IAAII,SAASH,iBACX,MAAM,IAAIP,MACR,CAAC,gDAAgD,EAAEW,OACjDF,IACA,gBAAgB,EAAEH,KAAK;QAI7B,IAAI;YACFE,YAAYN,oBAAoBQ;QAClC,EAAE,OAAOE,IAAI;YACX,MAAM,IAAIZ,MACR,CAAC,sDAAsD,EAAEW,OACvDF,IACA,iBAAiB,EAAEE,OAAOC,IAAI,gBAAgB,EAAEN,KAAK;QAE3D;IACF;IAEA,OAAOrB,sBAAsBuB,WAAWtB;AAC1C"}
+{"version":3,"file":"ai-model/shared/json.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../src/ai-model/shared/json.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { jsonrepair } from 'jsonrepair';\n\n/**\n * Preserve complete JSON values and remove optional markdown fences before\n * parsing. Keep array boundaries intact, including when the contents need repair.\n * For prose mixed with JSON, fall back to extracting the first through last\n * object brace; arbitrary prose with arrays is not a supported format.\n */\nexport function extractJSONFromCodeBlock(response: string) {\n  try {\n    JSON.parse(response);\n    return response.trim();\n  } catch {\n    // Invalid JSON may still be recoverable after extraction and repair.\n  }\n\n  try {\n    // Preserve complete object and array boundaries for jsonrepair.\n    const jsonMatch = response.match(/^\\s*(\\{[\\s\\S]*\\}|\\[[\\s\\S]*\\])\\s*$/);\n    if (jsonMatch) {\n      return jsonMatch[1];\n    }\n\n    // If no direct JSON object is found, try to extract JSON from a code block\n    const codeBlockMatch = response.match(/```(?:json)?\\s*([\\s\\S]*?)\\s*```/i);\n    if (codeBlockMatch) {\n      return codeBlockMatch[1];\n    }\n\n    // If no code block is found, try to find a JSON-like structure in the text\n    const jsonLikeMatch = response.match(/\\{[\\s\\S]*\\}/);\n    if (jsonLikeMatch) {\n      return jsonLikeMatch[0];\n    }\n  } catch {}\n  // If no JSON-like structure is found, return the original response\n  return response;\n}\n\nexport type JsonParserSource =\n  | 'generic-object'\n  | 'planning-action-param'\n  | 'locate'\n  | 'section-locator';\n\nexport interface JsonParserContext {\n  source: JsonParserSource;\n  preserveStringValueKeys?: string[];\n}\n\nexport type JsonParser = (raw: string, context?: JsonParserContext) => unknown;\n\n/**\n * Trim whitespace in parsed JSON by normalizing:\n * 1. All object keys (e.g., \" prompt \" -> \"prompt\")\n * 2. All string values (e.g., \" Tap \" -> \"Tap\")\n * This handles LLM output that may include leading/trailing spaces.\n */\nfunction trimParsedJsonStrings(\n  obj: any,\n  context: Pick<JsonParserContext, 'preserveStringValueKeys'> = {},\n): any {\n  // Handle null and undefined\n  if (obj === null || obj === undefined) {\n    return obj;\n  }\n\n  // Handle arrays - recursively normalize each element\n  if (Array.isArray(obj)) {\n    return obj.map((item) => trimParsedJsonStrings(item, context));\n  }\n\n  // Handle objects\n  if (typeof obj === 'object') {\n    const normalized: any = {};\n\n    for (const [key, value] of Object.entries(obj)) {\n      // Trim the key to remove leading/trailing spaces\n      const trimmedKey = key.trim();\n      const preserveStringValue =\n        context.preserveStringValueKeys?.includes(trimmedKey) ?? false;\n\n      const normalizedValue =\n        typeof value === 'string'\n          ? preserveStringValue\n            ? value\n            : value.trim()\n          : trimParsedJsonStrings(value, context);\n\n      normalized[trimmedKey] = normalizedValue;\n    }\n\n    return normalized;\n  }\n\n  // Handle primitive strings\n  if (typeof obj === 'string') {\n    return obj.trim();\n  }\n\n  // Return other primitives as-is\n  return obj;\n}\n\nfunction repairKnownJsonIssues(\n  jsonBlock: string,\n  _rawResponse: string,\n): string {\n  // TODO: Add project-specific repairs that jsonrepair cannot handle.\n  return jsonBlock;\n}\n\nexport function assertJsonObject(\n  parsed: unknown,\n): asserts parsed is Record<string, unknown> {\n  if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {\n    const parsedType = Array.isArray(parsed)\n      ? 'array'\n      : parsed === null\n        ? 'null'\n        : typeof parsed;\n    throw new Error(\n      `LLM response is valid JSON but does not match the expected schema. Expected to be a JSON object, got ${parsedType}: ${JSON.stringify(parsed)}.`,\n    );\n  }\n}\n\nfunction parseJsonWithRepair(jsonStr: string) {\n  try {\n    return JSON.parse(jsonStr);\n  } catch {\n    return JSON.parse(jsonrepair(jsonStr));\n  }\n}\n\nexport function parseModelResponseJson(\n  raw: string,\n  context?: JsonParserContext,\n) {\n  const cleanJsonString = extractJSONFromCodeBlock(raw);\n\n  let parsedObj: unknown;\n\n  try {\n    parsedObj = parseJsonWithRepair(cleanJsonString);\n  } catch (e1) {\n    const code = repairKnownJsonIssues(cleanJsonString, raw);\n    if (code === cleanJsonString) {\n      throw new Error(\n        `failed to parse LLM response into JSON. Error - ${String(\n          e1,\n        )}. Response - \\n ${raw}`,\n      );\n    }\n\n    try {\n      parsedObj = parseJsonWithRepair(code);\n    } catch (e2) {\n      throw new Error(\n        `failed to parse LLM response into JSON. First error - ${String(\n          e1,\n        )}. Second error - ${String(e2)}. Response - \\n ${raw}`,\n      );\n    }\n  }\n\n  return trimParsedJsonStrings(parsedObj, context);\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","extractJSONFromCodeBlock","response","JSON","jsonMatch","codeBlockMatch","jsonLikeMatch","trimParsedJsonStrings","context","Array","item","normalized","value","trimmedKey","preserveStringValue","normalizedValue","repairKnownJsonIssues","jsonBlock","_rawResponse","assertJsonObject","parsed","parsedType","Error","parseJsonWithRepair","jsonStr","jsonrepair","parseModelResponseJson","raw","cleanJsonString","parsedObj","e1","code","String","e2"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;ACEO,SAASI,yBAAyBC,QAAgB;IACvD,IAAI;QACFC,KAAK,KAAK,CAACD;QACX,OAAOA,SAAS,IAAI;IACtB,EAAE,OAAM,CAER;IAEA,IAAI;QAEF,MAAME,YAAYF,SAAS,KAAK,CAAC;QACjC,IAAIE,WACF,OAAOA,SAAS,CAAC,EAAE;QAIrB,MAAMC,iBAAiBH,SAAS,KAAK,CAAC;QACtC,IAAIG,gBACF,OAAOA,cAAc,CAAC,EAAE;QAI1B,MAAMC,gBAAgBJ,SAAS,KAAK,CAAC;QACrC,IAAII,eACF,OAAOA,aAAa,CAAC,EAAE;IAE3B,EAAE,OAAM,CAAC;IAET,OAAOJ;AACT;AAqBA,SAASK,sBACPT,GAAQ,EACRU,UAA8D,CAAC,CAAC;IAGhE,IAAIV,QAAAA,KACF,OAAOA;IAIT,IAAIW,MAAM,OAAO,CAACX,MAChB,OAAOA,IAAI,GAAG,CAAC,CAACY,OAASH,sBAAsBG,MAAMF;IAIvD,IAAI,AAAe,YAAf,OAAOV,KAAkB;QAC3B,MAAMa,aAAkB,CAAC;QAEzB,KAAK,MAAM,CAACf,KAAKgB,MAAM,IAAIf,OAAO,OAAO,CAACC,KAAM;YAE9C,MAAMe,aAAajB,IAAI,IAAI;YAC3B,MAAMkB,sBACJN,QAAQ,uBAAuB,EAAE,SAASK,eAAe;YAE3D,MAAME,kBACJ,AAAiB,YAAjB,OAAOH,QACHE,sBACEF,QACAA,MAAM,IAAI,KACZL,sBAAsBK,OAAOJ;YAEnCG,UAAU,CAACE,WAAW,GAAGE;QAC3B;QAEA,OAAOJ;IACT;IAGA,IAAI,AAAe,YAAf,OAAOb,KACT,OAAOA,IAAI,IAAI;IAIjB,OAAOA;AACT;AAEA,SAASkB,sBACPC,SAAiB,EACjBC,YAAoB;IAGpB,OAAOD;AACT;AAEO,SAASE,iBACdC,MAAe;IAEf,IAAI,CAACA,UAAU,AAAkB,YAAlB,OAAOA,UAAuBX,MAAM,OAAO,CAACW,SAAS;QAClE,MAAMC,aAAaZ,MAAM,OAAO,CAACW,UAC7B,UACAA,AAAW,SAAXA,SACE,SACA,OAAOA;QACb,MAAM,IAAIE,MACR,CAAC,qGAAqG,EAAED,WAAW,EAAE,EAAElB,KAAK,SAAS,CAACiB,QAAQ,CAAC,CAAC;IAEpJ;AACF;AAEA,SAASG,oBAAoBC,OAAe;IAC1C,IAAI;QACF,OAAOrB,KAAK,KAAK,CAACqB;IACpB,EAAE,OAAM;QACN,OAAOrB,KAAK,KAAK,CAACsB,AAAAA,IAAAA,oCAAAA,UAAAA,AAAAA,EAAWD;IAC/B;AACF;AAEO,SAASE,uBACdC,GAAW,EACXnB,OAA2B;IAE3B,MAAMoB,kBAAkB3B,yBAAyB0B;IAEjD,IAAIE;IAEJ,IAAI;QACFA,YAAYN,oBAAoBK;IAClC,EAAE,OAAOE,IAAI;QACX,MAAMC,OAAOf,sBAAsBY,iBAAiBD;QACpD,IAAII,SAASH,iBACX,MAAM,IAAIN,MACR,CAAC,gDAAgD,EAAEU,OACjDF,IACA,gBAAgB,EAAEH,KAAK;QAI7B,IAAI;YACFE,YAAYN,oBAAoBQ;QAClC,EAAE,OAAOE,IAAI;YACX,MAAM,IAAIX,MACR,CAAC,sDAAsD,EAAEU,OACvDF,IACA,iBAAiB,EAAEE,OAAOC,IAAI,gBAAgB,EAAEN,KAAK;QAE3D;IACF;IAEA,OAAOpB,sBAAsBsB,WAAWrB;AAC1C"}

package/dist/lib/ai-model/workflows/grounding/locate.js

@@ -114,7 +114,7 @@ async function genericLocate(locateRequest) {
         locateImage: locateRequest.locateImage,
         imagePreprocess: adapter.imagePreprocess,
         targetDescription: targetElementDescription,
-        userMessageContentOrder: adapter.locate.userMessageContentOrder
+        promptOrder: adapter.locate.promptOrder
     });
     try {
         return await (0, semantic_retry_js_namespaceObject.callAiAndParseWithRetry)({

package/dist/lib/ai-model/workflows/grounding/locate.js.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/workflows/grounding/locate.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../../src/ai-model/workflows/grounding/locate.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TUserPrompt } from '../../../common';\nimport { userPromptToString } from '../../../common';\nimport { buildElementLocateSystemPrompt } from '../../prompt/locate';\nimport {\n  AIResponseParseError,\n  callGenerativeAI,\n} from '../../service-caller/index';\nimport {\n  callAiAndParseWithRetry,\n  withSemanticRetryFeedback,\n} from '../../service-caller/semantic-retry';\nimport { mapSearchAreaResultToOriginalResult } from './search-area-mapping';\nimport type {\n  LocateModelResponse,\n  LocateOptions,\n  LocateRequest,\n  LocateResult,\n} from './types';\nimport { formatLocateModelContext, prepareLocateModelInput } from './utils';\n\nconst debugGrounding = getDebug('ai:grounding');\n\nexport { userPromptToString as extraTextFromUserPrompt } from '../../../common';\n\nexport async function AiLocateElement(\n  options: LocateOptions & { targetElementDescription: TUserPrompt },\n): Promise<LocateResult> {\n  const { targetElementDescription, ...locateOptions } = options;\n  assert(\n    targetElementDescription,\n    'cannot find the target element description',\n  );\n\n  const { context } = locateOptions;\n  const locateImage = locateOptions.searchConfig?.image ?? {\n    imageBase64: context.screenshot.base64,\n    width: context.shotSize.width,\n    height: context.shotSize.height,\n  };\n  const locateRequest: LocateRequest = {\n    targetElementDescription,\n    locateImage,\n    options: locateOptions,\n  };\n\n  const locateAdapter = options.modelRuntime.adapter.locate;\n  const locateFn =\n    locateAdapter.kind === 'custom' ? locateAdapter.locateFn : genericLocate;\n  const locateResponse = await locateFn(locateRequest);\n  const {\n    locatedPixelResult,\n    rawResponse,\n    rawAssistantOutput,\n    usage,\n    reasoningContent,\n    errors = [],\n  } = locateResponse;\n  const baseLocateResult = {\n    rawResponse,\n    rawAssistantOutput,\n    usage,\n    reasoning_content: reasoningContent,\n  };\n\n  if (!locatedPixelResult) {\n    return {\n      parseResult: {\n        element: undefined,\n        errors,\n      },\n      ...baseLocateResult,\n    };\n  }\n\n  try {\n    const mappedResult = mapSearchAreaResultToOriginalResult(\n      locatedPixelResult,\n      locateOptions.searchConfig?.mapping,\n    );\n    debugGrounding('center', mappedResult.center);\n\n    return {\n      parseResult: {\n        element: {\n          ...mappedResult,\n          description: userPromptToString(targetElementDescription),\n        },\n        errors: [],\n      },\n      ...baseLocateResult,\n    };\n  } catch (error) {\n    const msg =\n      error instanceof Error\n        ? `Failed to parse locate result: ${error.message}`\n        : 'unknown error in locate';\n    return {\n      parseResult: {\n        element: undefined,\n        errors: errors.length > 0 ? [...errors, `(${msg})`] : [msg],\n      },\n      ...baseLocateResult,\n    };\n  }\n}\n\nexport async function genericLocate(\n  locateRequest: LocateRequest,\n): Promise<LocateModelResponse> {\n  const { options, targetElementDescription } = locateRequest;\n  const modelRuntime = options.modelRuntime;\n  const { adapter } = modelRuntime;\n  assert(\n    adapter.locate.kind === 'standard',\n    'generic locate requires a standard locate adapter',\n  );\n  const { protocol, resultCodec } = adapter.locate.element;\n\n  const systemPrompt = buildElementLocateSystemPrompt({\n    systemPromptIntroduction: protocol.systemPromptIntroduction,\n    responseInstructions: protocol.buildResponseInstructions(\n      resultCodec.promptSpec,\n    ),\n  });\n\n  const userInstructionPrompt = protocol.buildUserPrompt(\n    userPromptToString(targetElementDescription),\n  );\n\n  const { messages, preparedImage } = await prepareLocateModelInput({\n    systemPrompt,\n    userPrompt: userInstructionPrompt,\n    locateImage: locateRequest.locateImage,\n    imagePreprocess: adapter.imagePreprocess,\n    targetDescription: targetElementDescription,\n    userMessageContentOrder: adapter.locate.userMessageContentOrder,\n  });\n\n  try {\n    return await callAiAndParseWithRetry({\n      callAi: (retryAttempt, previousParseError) =>\n        callGenerativeAI(\n          withSemanticRetryFeedback(messages, previousParseError),\n          modelRuntime,\n          {\n            abortSignal: options.abortSignal,\n            expectedJsonObjectResponse: protocol.expectedJsonObjectResponse,\n            semanticRetryAttempt: retryAttempt,\n          },\n        ),\n      parseResponse: (response): LocateModelResponse => {\n        const parsedLocateResult = protocol.parseRawResponse(\n          response.content,\n          resultCodec.promptSpec,\n        );\n        const rawResponse = response.content;\n        const locateError = parsedLocateResult.error;\n        if (parsedLocateResult.kind === 'not-found') {\n          return {\n            rawResponse,\n            rawAssistantOutput: response.rawAssistantOutput,\n            usage: response.usage,\n            reasoningContent: response.reasoning_content,\n            errors: locateError ? [locateError] : [],\n          };\n        }\n\n        try {\n          const result = resultCodec.toPixelResult(parsedLocateResult.target, {\n            preparedSize: preparedImage.preparedSize,\n            contentSize: preparedImage.contentSize,\n          });\n          return {\n            locatedPixelResult: result,\n            rawResponse,\n            rawAssistantOutput: response.rawAssistantOutput,\n            usage: response.usage,\n            reasoningContent: response.reasoning_content,\n            errors: [],\n          };\n        } catch (error) {\n          const errorMessage =\n            error instanceof Error ? error.message : String(error);\n          const message = [\n            locateError && `error in locate result: ${locateError}`,\n            `coordinate parsing error: ${errorMessage}`,\n            formatLocateModelContext(modelRuntime),\n          ]\n            .filter(Boolean)\n            .join('\\n');\n          throw new Error(message, { cause: error });\n        }\n      },\n      toParseError: (error, response) => {\n        const parseErrorMessage =\n          error instanceof Error\n            ? `Failed to parse locate result: ${error.message}`\n            : 'unknown error in locate result';\n        return new AIResponseParseError(\n          parseErrorMessage,\n          response.content,\n          response.usage,\n          response.rawAssistantOutput,\n          response.reasoning_content,\n        );\n      },\n      parseRetryTimes: modelRuntime.config.retryCount,\n      parseRetryInterval: modelRuntime.config.retryInterval,\n      abortSignal: options.abortSignal,\n      onParseRetry: (error) => {\n        debugGrounding(\n          'retrying locate after coordinate parsing failed: %s',\n          error instanceof Error ? error.message : String(error),\n        );\n      },\n    });\n  } catch (callError) {\n    if (callError instanceof AIResponseParseError) {\n      return {\n        rawResponse: callError.rawResponse,\n        rawAssistantOutput: callError.rawAssistantOutput,\n        usage: callError.usage,\n        reasoningContent: callError.reasoningContent,\n        errors: [callError.message],\n      };\n    }\n\n    const errorMessage =\n      callError instanceof Error ? callError.message : String(callError);\n    return {\n      rawResponse: errorMessage,\n      errors: [`AI call error: ${errorMessage}`],\n    };\n  }\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debugGrounding","getDebug","AiLocateElement","options","targetElementDescription","locateOptions","assert","context","locateImage","locateRequest","locateAdapter","locateFn","genericLocate","locateResponse","locatedPixelResult","rawResponse","rawAssistantOutput","usage","reasoningContent","errors","baseLocateResult","undefined","mappedResult","mapSearchAreaResultToOriginalResult","userPromptToString","error","msg","Error","modelRuntime","adapter","protocol","resultCodec","systemPrompt","buildElementLocateSystemPrompt","userInstructionPrompt","messages","preparedImage","prepareLocateModelInput","callAiAndParseWithRetry","retryAttempt","previousParseError","callGenerativeAI","withSemanticRetryFeedback","response","parsedLocateResult","locateError","result","errorMessage","String","message","formatLocateModelContext","Boolean","parseErrorMessage","AIResponseParseError","callError"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;ACgBA,MAAMI,iBAAiBC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AAIzB,eAAeC,gBACpBC,OAAkE;IAElE,MAAM,EAAEC,wBAAwB,EAAE,GAAGC,eAAe,GAAGF;IACvDG,IAAAA,sBAAAA,MAAAA,AAAAA,EACEF,0BACA;IAGF,MAAM,EAAEG,OAAO,EAAE,GAAGF;IACpB,MAAMG,cAAcH,cAAc,YAAY,EAAE,SAAS;QACvD,aAAaE,QAAQ,UAAU,CAAC,MAAM;QACtC,OAAOA,QAAQ,QAAQ,CAAC,KAAK;QAC7B,QAAQA,QAAQ,QAAQ,CAAC,MAAM;IACjC;IACA,MAAME,gBAA+B;QACnCL;QACAI;QACA,SAASH;IACX;IAEA,MAAMK,gBAAgBP,QAAQ,YAAY,CAAC,OAAO,CAAC,MAAM;IACzD,MAAMQ,WACJD,AAAuB,aAAvBA,cAAc,IAAI,GAAgBA,cAAc,QAAQ,GAAGE;IAC7D,MAAMC,iBAAiB,MAAMF,SAASF;IACtC,MAAM,EACJK,kBAAkB,EAClBC,WAAW,EACXC,kBAAkB,EAClBC,KAAK,EACLC,gBAAgB,EAChBC,SAAS,EAAE,EACZ,GAAGN;IACJ,MAAMO,mBAAmB;QACvBL;QACAC;QACAC;QACA,mBAAmBC;IACrB;IAEA,IAAI,CAACJ,oBACH,OAAO;QACL,aAAa;YACX,SAASO;YACTF;QACF;QACA,GAAGC,gBAAgB;IACrB;IAGF,IAAI;QACF,MAAME,eAAeC,AAAAA,IAAAA,gDAAAA,mCAAAA,AAAAA,EACnBT,oBACAT,cAAc,YAAY,EAAE;QAE9BL,eAAe,UAAUsB,aAAa,MAAM;QAE5C,OAAO;YACL,aAAa;gBACX,SAAS;oBACP,GAAGA,YAAY;oBACf,aAAaE,AAAAA,IAAAA,mCAAAA,kBAAAA,AAAAA,EAAmBpB;gBAClC;gBACA,QAAQ,EAAE;YACZ;YACA,GAAGgB,gBAAgB;QACrB;IACF,EAAE,OAAOK,OAAO;QACd,MAAMC,MACJD,iBAAiBE,QACb,CAAC,+BAA+B,EAAEF,MAAM,OAAO,EAAE,GACjD;QACN,OAAO;YACL,aAAa;gBACX,SAASJ;gBACT,QAAQF,OAAO,MAAM,GAAG,IAAI;uBAAIA;oBAAQ,CAAC,CAAC,EAAEO,IAAI,CAAC,CAAC;iBAAC,GAAG;oBAACA;iBAAI;YAC7D;YACA,GAAGN,gBAAgB;QACrB;IACF;AACF;AAEO,eAAeR,cACpBH,aAA4B;IAE5B,MAAM,EAAEN,OAAO,EAAEC,wBAAwB,EAAE,GAAGK;IAC9C,MAAMmB,eAAezB,QAAQ,YAAY;IACzC,MAAM,EAAE0B,OAAO,EAAE,GAAGD;IACpBtB,IAAAA,sBAAAA,MAAAA,AAAAA,EACEuB,AAAwB,eAAxBA,QAAQ,MAAM,CAAC,IAAI,EACnB;IAEF,MAAM,EAAEC,QAAQ,EAAEC,WAAW,EAAE,GAAGF,QAAQ,MAAM,CAAC,OAAO;IAExD,MAAMG,eAAeC,AAAAA,IAAAA,yBAAAA,8BAAAA,AAAAA,EAA+B;QAClD,0BAA0BH,SAAS,wBAAwB;QAC3D,sBAAsBA,SAAS,yBAAyB,CACtDC,YAAY,UAAU;IAE1B;IAEA,MAAMG,wBAAwBJ,SAAS,eAAe,CACpDN,AAAAA,IAAAA,mCAAAA,kBAAAA,AAAAA,EAAmBpB;IAGrB,MAAM,EAAE+B,QAAQ,EAAEC,aAAa,EAAE,GAAG,MAAMC,AAAAA,IAAAA,kCAAAA,uBAAAA,AAAAA,EAAwB;QAChEL;QACA,YAAYE;QACZ,aAAazB,cAAc,WAAW;QACtC,iBAAiBoB,QAAQ,eAAe;QACxC,mBAAmBzB;QACnB,yBAAyByB,QAAQ,MAAM,CAAC,uBAAuB;IACjE;IAEA,IAAI;QACF,OAAO,MAAMS,AAAAA,IAAAA,kCAAAA,uBAAAA,AAAAA,EAAwB;YACnC,QAAQ,CAACC,cAAcC,qBACrBC,AAAAA,IAAAA,iDAAAA,gBAAAA,AAAAA,EACEC,AAAAA,IAAAA,kCAAAA,yBAAAA,AAAAA,EAA0BP,UAAUK,qBACpCZ,cACA;oBACE,aAAazB,QAAQ,WAAW;oBAChC,4BAA4B2B,SAAS,0BAA0B;oBAC/D,sBAAsBS;gBACxB;YAEJ,eAAe,CAACI;gBACd,MAAMC,qBAAqBd,SAAS,gBAAgB,CAClDa,SAAS,OAAO,EAChBZ,YAAY,UAAU;gBAExB,MAAMhB,cAAc4B,SAAS,OAAO;gBACpC,MAAME,cAAcD,mBAAmB,KAAK;gBAC5C,IAAIA,AAA4B,gBAA5BA,mBAAmB,IAAI,EACzB,OAAO;oBACL7B;oBACA,oBAAoB4B,SAAS,kBAAkB;oBAC/C,OAAOA,SAAS,KAAK;oBACrB,kBAAkBA,SAAS,iBAAiB;oBAC5C,QAAQE,cAAc;wBAACA;qBAAY,GAAG,EAAE;gBAC1C;gBAGF,IAAI;oBACF,MAAMC,SAASf,YAAY,aAAa,CAACa,mBAAmB,MAAM,EAAE;wBAClE,cAAcR,cAAc,YAAY;wBACxC,aAAaA,cAAc,WAAW;oBACxC;oBACA,OAAO;wBACL,oBAAoBU;wBACpB/B;wBACA,oBAAoB4B,SAAS,kBAAkB;wBAC/C,OAAOA,SAAS,KAAK;wBACrB,kBAAkBA,SAAS,iBAAiB;wBAC5C,QAAQ,EAAE;oBACZ;gBACF,EAAE,OAAOlB,OAAO;oBACd,MAAMsB,eACJtB,iBAAiBE,QAAQF,MAAM,OAAO,GAAGuB,OAAOvB;oBAClD,MAAMwB,UAAU;wBACdJ,eAAe,CAAC,wBAAwB,EAAEA,aAAa;wBACvD,CAAC,0BAA0B,EAAEE,cAAc;wBAC3CG,IAAAA,kCAAAA,wBAAAA,AAAAA,EAAyBtB;qBAC1B,CACE,MAAM,CAACuB,SACP,IAAI,CAAC;oBACR,MAAM,IAAIxB,MAAMsB,SAAS;wBAAE,OAAOxB;oBAAM;gBAC1C;YACF;YACA,cAAc,CAACA,OAAOkB;gBACpB,MAAMS,oBACJ3B,iBAAiBE,QACb,CAAC,+BAA+B,EAAEF,MAAM,OAAO,EAAE,GACjD;gBACN,OAAO,IAAI4B,iDAAAA,oBAAoBA,CAC7BD,mBACAT,SAAS,OAAO,EAChBA,SAAS,KAAK,EACdA,SAAS,kBAAkB,EAC3BA,SAAS,iBAAiB;YAE9B;YACA,iBAAiBf,aAAa,MAAM,CAAC,UAAU;YAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;YACrD,aAAazB,QAAQ,WAAW;YAChC,cAAc,CAACsB;gBACbzB,eACE,uDACAyB,iBAAiBE,QAAQF,MAAM,OAAO,GAAGuB,OAAOvB;YAEpD;QACF;IACF,EAAE,OAAO6B,WAAW;QAClB,IAAIA,qBAAqBD,iDAAAA,oBAAoBA,EAC3C,OAAO;YACL,aAAaC,UAAU,WAAW;YAClC,oBAAoBA,UAAU,kBAAkB;YAChD,OAAOA,UAAU,KAAK;YACtB,kBAAkBA,UAAU,gBAAgB;YAC5C,QAAQ;gBAACA,UAAU,OAAO;aAAC;QAC7B;QAGF,MAAMP,eACJO,qBAAqB3B,QAAQ2B,UAAU,OAAO,GAAGN,OAAOM;QAC1D,OAAO;YACL,aAAaP;YACb,QAAQ;gBAAC,CAAC,eAAe,EAAEA,cAAc;aAAC;QAC5C;IACF;AACF"}
+{"version":3,"file":"ai-model/workflows/grounding/locate.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../../src/ai-model/workflows/grounding/locate.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TUserPrompt } from '../../../common';\nimport { userPromptToString } from '../../../common';\nimport { buildElementLocateSystemPrompt } from '../../prompt/locate';\nimport {\n  AIResponseParseError,\n  callGenerativeAI,\n} from '../../service-caller/index';\nimport {\n  callAiAndParseWithRetry,\n  withSemanticRetryFeedback,\n} from '../../service-caller/semantic-retry';\nimport { mapSearchAreaResultToOriginalResult } from './search-area-mapping';\nimport type {\n  LocateModelResponse,\n  LocateOptions,\n  LocateRequest,\n  LocateResult,\n} from './types';\nimport { formatLocateModelContext, prepareLocateModelInput } from './utils';\n\nconst debugGrounding = getDebug('ai:grounding');\n\nexport { userPromptToString as extraTextFromUserPrompt } from '../../../common';\n\nexport async function AiLocateElement(\n  options: LocateOptions & { targetElementDescription: TUserPrompt },\n): Promise<LocateResult> {\n  const { targetElementDescription, ...locateOptions } = options;\n  assert(\n    targetElementDescription,\n    'cannot find the target element description',\n  );\n\n  const { context } = locateOptions;\n  const locateImage = locateOptions.searchConfig?.image ?? {\n    imageBase64: context.screenshot.base64,\n    width: context.shotSize.width,\n    height: context.shotSize.height,\n  };\n  const locateRequest: LocateRequest = {\n    targetElementDescription,\n    locateImage,\n    options: locateOptions,\n  };\n\n  const locateAdapter = options.modelRuntime.adapter.locate;\n  const locateFn =\n    locateAdapter.kind === 'custom' ? locateAdapter.locateFn : genericLocate;\n  const locateResponse = await locateFn(locateRequest);\n  const {\n    locatedPixelResult,\n    rawResponse,\n    rawAssistantOutput,\n    usage,\n    reasoningContent,\n    errors = [],\n  } = locateResponse;\n  const baseLocateResult = {\n    rawResponse,\n    rawAssistantOutput,\n    usage,\n    reasoning_content: reasoningContent,\n  };\n\n  if (!locatedPixelResult) {\n    return {\n      parseResult: {\n        element: undefined,\n        errors,\n      },\n      ...baseLocateResult,\n    };\n  }\n\n  try {\n    const mappedResult = mapSearchAreaResultToOriginalResult(\n      locatedPixelResult,\n      locateOptions.searchConfig?.mapping,\n    );\n    debugGrounding('center', mappedResult.center);\n\n    return {\n      parseResult: {\n        element: {\n          ...mappedResult,\n          description: userPromptToString(targetElementDescription),\n        },\n        errors: [],\n      },\n      ...baseLocateResult,\n    };\n  } catch (error) {\n    const msg =\n      error instanceof Error\n        ? `Failed to parse locate result: ${error.message}`\n        : 'unknown error in locate';\n    return {\n      parseResult: {\n        element: undefined,\n        errors: errors.length > 0 ? [...errors, `(${msg})`] : [msg],\n      },\n      ...baseLocateResult,\n    };\n  }\n}\n\nexport async function genericLocate(\n  locateRequest: LocateRequest,\n): Promise<LocateModelResponse> {\n  const { options, targetElementDescription } = locateRequest;\n  const modelRuntime = options.modelRuntime;\n  const { adapter } = modelRuntime;\n  assert(\n    adapter.locate.kind === 'standard',\n    'generic locate requires a standard locate adapter',\n  );\n  const { protocol, resultCodec } = adapter.locate.element;\n\n  const systemPrompt = buildElementLocateSystemPrompt({\n    systemPromptIntroduction: protocol.systemPromptIntroduction,\n    responseInstructions: protocol.buildResponseInstructions(\n      resultCodec.promptSpec,\n    ),\n  });\n\n  const userInstructionPrompt = protocol.buildUserPrompt(\n    userPromptToString(targetElementDescription),\n  );\n\n  const { messages, preparedImage } = await prepareLocateModelInput({\n    systemPrompt,\n    userPrompt: userInstructionPrompt,\n    locateImage: locateRequest.locateImage,\n    imagePreprocess: adapter.imagePreprocess,\n    targetDescription: targetElementDescription,\n    promptOrder: adapter.locate.promptOrder,\n  });\n\n  try {\n    return await callAiAndParseWithRetry({\n      callAi: (retryAttempt, previousParseError) =>\n        callGenerativeAI(\n          withSemanticRetryFeedback(messages, previousParseError),\n          modelRuntime,\n          {\n            abortSignal: options.abortSignal,\n            expectedJsonObjectResponse: protocol.expectedJsonObjectResponse,\n            semanticRetryAttempt: retryAttempt,\n          },\n        ),\n      parseResponse: (response): LocateModelResponse => {\n        const parsedLocateResult = protocol.parseRawResponse(\n          response.content,\n          resultCodec.promptSpec,\n        );\n        const rawResponse = response.content;\n        const locateError = parsedLocateResult.error;\n        if (parsedLocateResult.kind === 'not-found') {\n          return {\n            rawResponse,\n            rawAssistantOutput: response.rawAssistantOutput,\n            usage: response.usage,\n            reasoningContent: response.reasoning_content,\n            errors: locateError ? [locateError] : [],\n          };\n        }\n\n        try {\n          const result = resultCodec.toPixelResult(parsedLocateResult.target, {\n            preparedSize: preparedImage.preparedSize,\n            contentSize: preparedImage.contentSize,\n          });\n          return {\n            locatedPixelResult: result,\n            rawResponse,\n            rawAssistantOutput: response.rawAssistantOutput,\n            usage: response.usage,\n            reasoningContent: response.reasoning_content,\n            errors: [],\n          };\n        } catch (error) {\n          const errorMessage =\n            error instanceof Error ? error.message : String(error);\n          const message = [\n            locateError && `error in locate result: ${locateError}`,\n            `coordinate parsing error: ${errorMessage}`,\n            formatLocateModelContext(modelRuntime),\n          ]\n            .filter(Boolean)\n            .join('\\n');\n          throw new Error(message, { cause: error });\n        }\n      },\n      toParseError: (error, response) => {\n        const parseErrorMessage =\n          error instanceof Error\n            ? `Failed to parse locate result: ${error.message}`\n            : 'unknown error in locate result';\n        return new AIResponseParseError(\n          parseErrorMessage,\n          response.content,\n          response.usage,\n          response.rawAssistantOutput,\n          response.reasoning_content,\n        );\n      },\n      parseRetryTimes: modelRuntime.config.retryCount,\n      parseRetryInterval: modelRuntime.config.retryInterval,\n      abortSignal: options.abortSignal,\n      onParseRetry: (error) => {\n        debugGrounding(\n          'retrying locate after coordinate parsing failed: %s',\n          error instanceof Error ? error.message : String(error),\n        );\n      },\n    });\n  } catch (callError) {\n    if (callError instanceof AIResponseParseError) {\n      return {\n        rawResponse: callError.rawResponse,\n        rawAssistantOutput: callError.rawAssistantOutput,\n        usage: callError.usage,\n        reasoningContent: callError.reasoningContent,\n        errors: [callError.message],\n      };\n    }\n\n    const errorMessage =\n      callError instanceof Error ? callError.message : String(callError);\n    return {\n      rawResponse: errorMessage,\n      errors: [`AI call error: ${errorMessage}`],\n    };\n  }\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debugGrounding","getDebug","AiLocateElement","options","targetElementDescription","locateOptions","assert","context","locateImage","locateRequest","locateAdapter","locateFn","genericLocate","locateResponse","locatedPixelResult","rawResponse","rawAssistantOutput","usage","reasoningContent","errors","baseLocateResult","undefined","mappedResult","mapSearchAreaResultToOriginalResult","userPromptToString","error","msg","Error","modelRuntime","adapter","protocol","resultCodec","systemPrompt","buildElementLocateSystemPrompt","userInstructionPrompt","messages","preparedImage","prepareLocateModelInput","callAiAndParseWithRetry","retryAttempt","previousParseError","callGenerativeAI","withSemanticRetryFeedback","response","parsedLocateResult","locateError","result","errorMessage","String","message","formatLocateModelContext","Boolean","parseErrorMessage","AIResponseParseError","callError"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;ACgBA,MAAMI,iBAAiBC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AAIzB,eAAeC,gBACpBC,OAAkE;IAElE,MAAM,EAAEC,wBAAwB,EAAE,GAAGC,eAAe,GAAGF;IACvDG,IAAAA,sBAAAA,MAAAA,AAAAA,EACEF,0BACA;IAGF,MAAM,EAAEG,OAAO,EAAE,GAAGF;IACpB,MAAMG,cAAcH,cAAc,YAAY,EAAE,SAAS;QACvD,aAAaE,QAAQ,UAAU,CAAC,MAAM;QACtC,OAAOA,QAAQ,QAAQ,CAAC,KAAK;QAC7B,QAAQA,QAAQ,QAAQ,CAAC,MAAM;IACjC;IACA,MAAME,gBAA+B;QACnCL;QACAI;QACA,SAASH;IACX;IAEA,MAAMK,gBAAgBP,QAAQ,YAAY,CAAC,OAAO,CAAC,MAAM;IACzD,MAAMQ,WACJD,AAAuB,aAAvBA,cAAc,IAAI,GAAgBA,cAAc,QAAQ,GAAGE;IAC7D,MAAMC,iBAAiB,MAAMF,SAASF;IACtC,MAAM,EACJK,kBAAkB,EAClBC,WAAW,EACXC,kBAAkB,EAClBC,KAAK,EACLC,gBAAgB,EAChBC,SAAS,EAAE,EACZ,GAAGN;IACJ,MAAMO,mBAAmB;QACvBL;QACAC;QACAC;QACA,mBAAmBC;IACrB;IAEA,IAAI,CAACJ,oBACH,OAAO;QACL,aAAa;YACX,SAASO;YACTF;QACF;QACA,GAAGC,gBAAgB;IACrB;IAGF,IAAI;QACF,MAAME,eAAeC,AAAAA,IAAAA,gDAAAA,mCAAAA,AAAAA,EACnBT,oBACAT,cAAc,YAAY,EAAE;QAE9BL,eAAe,UAAUsB,aAAa,MAAM;QAE5C,OAAO;YACL,aAAa;gBACX,SAAS;oBACP,GAAGA,YAAY;oBACf,aAAaE,AAAAA,IAAAA,mCAAAA,kBAAAA,AAAAA,EAAmBpB;gBAClC;gBACA,QAAQ,EAAE;YACZ;YACA,GAAGgB,gBAAgB;QACrB;IACF,EAAE,OAAOK,OAAO;QACd,MAAMC,MACJD,iBAAiBE,QACb,CAAC,+BAA+B,EAAEF,MAAM,OAAO,EAAE,GACjD;QACN,OAAO;YACL,aAAa;gBACX,SAASJ;gBACT,QAAQF,OAAO,MAAM,GAAG,IAAI;uBAAIA;oBAAQ,CAAC,CAAC,EAAEO,IAAI,CAAC,CAAC;iBAAC,GAAG;oBAACA;iBAAI;YAC7D;YACA,GAAGN,gBAAgB;QACrB;IACF;AACF;AAEO,eAAeR,cACpBH,aAA4B;IAE5B,MAAM,EAAEN,OAAO,EAAEC,wBAAwB,EAAE,GAAGK;IAC9C,MAAMmB,eAAezB,QAAQ,YAAY;IACzC,MAAM,EAAE0B,OAAO,EAAE,GAAGD;IACpBtB,IAAAA,sBAAAA,MAAAA,AAAAA,EACEuB,AAAwB,eAAxBA,QAAQ,MAAM,CAAC,IAAI,EACnB;IAEF,MAAM,EAAEC,QAAQ,EAAEC,WAAW,EAAE,GAAGF,QAAQ,MAAM,CAAC,OAAO;IAExD,MAAMG,eAAeC,AAAAA,IAAAA,yBAAAA,8BAAAA,AAAAA,EAA+B;QAClD,0BAA0BH,SAAS,wBAAwB;QAC3D,sBAAsBA,SAAS,yBAAyB,CACtDC,YAAY,UAAU;IAE1B;IAEA,MAAMG,wBAAwBJ,SAAS,eAAe,CACpDN,AAAAA,IAAAA,mCAAAA,kBAAAA,AAAAA,EAAmBpB;IAGrB,MAAM,EAAE+B,QAAQ,EAAEC,aAAa,EAAE,GAAG,MAAMC,AAAAA,IAAAA,kCAAAA,uBAAAA,AAAAA,EAAwB;QAChEL;QACA,YAAYE;QACZ,aAAazB,cAAc,WAAW;QACtC,iBAAiBoB,QAAQ,eAAe;QACxC,mBAAmBzB;QACnB,aAAayB,QAAQ,MAAM,CAAC,WAAW;IACzC;IAEA,IAAI;QACF,OAAO,MAAMS,AAAAA,IAAAA,kCAAAA,uBAAAA,AAAAA,EAAwB;YACnC,QAAQ,CAACC,cAAcC,qBACrBC,AAAAA,IAAAA,iDAAAA,gBAAAA,AAAAA,EACEC,AAAAA,IAAAA,kCAAAA,yBAAAA,AAAAA,EAA0BP,UAAUK,qBACpCZ,cACA;oBACE,aAAazB,QAAQ,WAAW;oBAChC,4BAA4B2B,SAAS,0BAA0B;oBAC/D,sBAAsBS;gBACxB;YAEJ,eAAe,CAACI;gBACd,MAAMC,qBAAqBd,SAAS,gBAAgB,CAClDa,SAAS,OAAO,EAChBZ,YAAY,UAAU;gBAExB,MAAMhB,cAAc4B,SAAS,OAAO;gBACpC,MAAME,cAAcD,mBAAmB,KAAK;gBAC5C,IAAIA,AAA4B,gBAA5BA,mBAAmB,IAAI,EACzB,OAAO;oBACL7B;oBACA,oBAAoB4B,SAAS,kBAAkB;oBAC/C,OAAOA,SAAS,KAAK;oBACrB,kBAAkBA,SAAS,iBAAiB;oBAC5C,QAAQE,cAAc;wBAACA;qBAAY,GAAG,EAAE;gBAC1C;gBAGF,IAAI;oBACF,MAAMC,SAASf,YAAY,aAAa,CAACa,mBAAmB,MAAM,EAAE;wBAClE,cAAcR,cAAc,YAAY;wBACxC,aAAaA,cAAc,WAAW;oBACxC;oBACA,OAAO;wBACL,oBAAoBU;wBACpB/B;wBACA,oBAAoB4B,SAAS,kBAAkB;wBAC/C,OAAOA,SAAS,KAAK;wBACrB,kBAAkBA,SAAS,iBAAiB;wBAC5C,QAAQ,EAAE;oBACZ;gBACF,EAAE,OAAOlB,OAAO;oBACd,MAAMsB,eACJtB,iBAAiBE,QAAQF,MAAM,OAAO,GAAGuB,OAAOvB;oBAClD,MAAMwB,UAAU;wBACdJ,eAAe,CAAC,wBAAwB,EAAEA,aAAa;wBACvD,CAAC,0BAA0B,EAAEE,cAAc;wBAC3CG,IAAAA,kCAAAA,wBAAAA,AAAAA,EAAyBtB;qBAC1B,CACE,MAAM,CAACuB,SACP,IAAI,CAAC;oBACR,MAAM,IAAIxB,MAAMsB,SAAS;wBAAE,OAAOxB;oBAAM;gBAC1C;YACF;YACA,cAAc,CAACA,OAAOkB;gBACpB,MAAMS,oBACJ3B,iBAAiBE,QACb,CAAC,+BAA+B,EAAEF,MAAM,OAAO,EAAE,GACjD;gBACN,OAAO,IAAI4B,iDAAAA,oBAAoBA,CAC7BD,mBACAT,SAAS,OAAO,EAChBA,SAAS,KAAK,EACdA,SAAS,kBAAkB,EAC3BA,SAAS,iBAAiB;YAE9B;YACA,iBAAiBf,aAAa,MAAM,CAAC,UAAU;YAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;YACrD,aAAazB,QAAQ,WAAW;YAChC,cAAc,CAACsB;gBACbzB,eACE,uDACAyB,iBAAiBE,QAAQF,MAAM,OAAO,GAAGuB,OAAOvB;YAEpD;QACF;IACF,EAAE,OAAO6B,WAAW;QAClB,IAAIA,qBAAqBD,iDAAAA,oBAAoBA,EAC3C,OAAO;YACL,aAAaC,UAAU,WAAW;YAClC,oBAAoBA,UAAU,kBAAkB;YAChD,OAAOA,UAAU,KAAK;YACtB,kBAAkBA,UAAU,gBAAgB;YAC5C,QAAQ;gBAACA,UAAU,OAAO;aAAC;QAC7B;QAGF,MAAMP,eACJO,qBAAqB3B,QAAQ2B,UAAU,OAAO,GAAGN,OAAOM;QAC1D,OAAO;YACL,aAAaP;YACb,QAAQ;gBAAC,CAAC,eAAe,EAAEA,cAAc;aAAC;QAC5C;IACF;AACF"}

package/dist/lib/ai-model/workflows/grounding/section-locate.js

@@ -59,7 +59,7 @@ async function AiLocateSection(options) {
         },
         imagePreprocess: adapter.imagePreprocess,
         targetDescription: sectionDescription,
-        userMessageContentOrder: adapter.locate.userMessageContentOrder
+        promptOrder: adapter.locate.promptOrder
     });
     let parsedResult;
     try {

package/dist/lib/ai-model/workflows/grounding/section-locate.js.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/workflows/grounding/section-locate.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../../src/ai-model/workflows/grounding/section-locate.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { RawAssistantOutput } from '@/types';\nimport type { AIUsageInfo, Rect, UIContext } from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TUserPrompt } from '../../../common';\nimport { userPromptToString } from '../../../common';\nimport type { ModelRuntime } from '../../models';\nimport { buildSearchAreaLocateSystemPrompt } from '../../prompt/locate';\nimport {\n  AIResponseParseError,\n  callGenerativeAI,\n} from '../../service-caller/index';\nimport {\n  callAiAndParseWithRetry,\n  withSemanticRetryFeedback,\n} from '../../service-caller/semantic-retry';\nimport {\n  buildSearchAreaConfig,\n  expandSearchArea,\n  mergeSearchAreaResults,\n} from './search-area';\nimport type { SearchAreaConfig } from './types';\nimport { formatLocateModelContext, prepareLocateModelInput } from './utils';\n\nconst debugSection = getDebug('ai:grounding:section');\n\ntype SectionLocateRawResponse = Awaited<ReturnType<typeof callGenerativeAI>>;\n\nexport async function AiLocateSection(options: {\n  context: UIContext;\n  sectionDescription: TUserPrompt;\n  modelRuntime: ModelRuntime;\n  abortSignal?: AbortSignal;\n}): Promise<{\n  searchAreaConfig?: SearchAreaConfig;\n  error?: string;\n  rawResponse: string;\n  rawAssistantOutput?: RawAssistantOutput;\n  usage?: AIUsageInfo;\n}> {\n  const { context, sectionDescription } = options;\n  const modelRuntime = options.modelRuntime;\n  const { adapter } = modelRuntime;\n  assert(\n    adapter.locate.kind === 'standard',\n    'section locate requires a standard locate adapter',\n  );\n  const searchArea = adapter.locate.searchArea;\n  assert(searchArea, 'section locate requires a search area operation');\n  const { protocol: searchAreaProtocol, resultCodec } = searchArea;\n  const screenshotBase64 = context.screenshot.base64;\n\n  const systemPrompt = buildSearchAreaLocateSystemPrompt({\n    systemPromptIntroduction: searchAreaProtocol.systemPromptIntroduction,\n    responseInstructions: searchAreaProtocol.buildResponseInstructions(\n      resultCodec.promptSpec,\n    ),\n  });\n\n  const userInstructionPrompt = searchAreaProtocol.buildUserPrompt(\n    userPromptToString(sectionDescription),\n  );\n\n  const { messages, preparedImage } = await prepareLocateModelInput({\n    systemPrompt,\n    userPrompt: userInstructionPrompt,\n    locateImage: {\n      imageBase64: screenshotBase64,\n      width: context.shotSize.width,\n      height: context.shotSize.height,\n    },\n    imagePreprocess: adapter.imagePreprocess,\n    targetDescription: sectionDescription,\n    userMessageContentOrder: adapter.locate.userMessageContentOrder,\n  });\n\n  let parsedResult:\n    | {\n        result: SectionLocateRawResponse;\n        sectionError?: string;\n        mergedRect?: undefined;\n      }\n    | {\n        result: SectionLocateRawResponse;\n        sectionError?: string;\n        mergedRect: Rect;\n      };\n\n  try {\n    parsedResult = await callAiAndParseWithRetry({\n      callAi: (retryAttempt, previousParseError) =>\n        callGenerativeAI(\n          withSemanticRetryFeedback(messages, previousParseError),\n          modelRuntime,\n          {\n            abortSignal: options.abortSignal,\n            expectedJsonObjectResponse:\n              searchAreaProtocol.expectedJsonObjectResponse,\n            semanticRetryAttempt: retryAttempt,\n          },\n        ),\n      parseResponse: (response) => {\n        const parsedLocateResult = searchAreaProtocol.parseRawResponse(\n          response.content,\n          resultCodec.promptSpec,\n        );\n        const sectionError = parsedLocateResult.error;\n        if (parsedLocateResult.kind === 'not-found') {\n          return { result: response, sectionError };\n        }\n\n        try {\n          const locateResultContext = {\n            preparedSize: preparedImage.preparedSize,\n            contentSize: preparedImage.contentSize,\n          };\n          const target = resultCodec.toPixelResult(\n            parsedLocateResult.target,\n            locateResultContext,\n          );\n          const references = parsedLocateResult.references?.map((reference) =>\n            resultCodec.toPixelResult(reference, locateResultContext),\n          );\n          const mergedRect = mergeSearchAreaResults(target, references);\n          debugSection('mergedRect %j', mergedRect);\n          return {\n            result: response,\n            sectionError,\n            mergedRect,\n          };\n        } catch (error) {\n          const errorMessage =\n            error instanceof Error ? error.message : String(error);\n          const message = [\n            sectionError && `error in section locate result: ${sectionError}`,\n            `coordinate parsing error: ${errorMessage}`,\n            formatLocateModelContext(modelRuntime),\n          ]\n            .filter(Boolean)\n            .join('\\n');\n          throw new Error(message, { cause: error });\n        }\n      },\n      toParseError: (error, response) => {\n        const parseErrorMessage =\n          error instanceof Error\n            ? `Failed to parse section locate result: ${error.message}`\n            : 'unknown error in section locate';\n        return new AIResponseParseError(\n          parseErrorMessage,\n          response.content,\n          response.usage,\n          response.rawAssistantOutput,\n          response.reasoning_content,\n        );\n      },\n      parseRetryTimes: modelRuntime.config.retryCount,\n      parseRetryInterval: modelRuntime.config.retryInterval,\n      abortSignal: options.abortSignal,\n      onParseRetry: (error) => {\n        debugSection(\n          'retrying section locate after coordinate parsing failed: %s',\n          error instanceof Error ? error.message : String(error),\n        );\n      },\n    });\n  } catch (callError) {\n    if (callError instanceof AIResponseParseError) {\n      return {\n        searchAreaConfig: undefined,\n        error: callError.message,\n        rawResponse: callError.rawResponse,\n        rawAssistantOutput: callError.rawAssistantOutput,\n        usage: callError.usage,\n      };\n    }\n\n    const errorMessage =\n      callError instanceof Error ? callError.message : String(callError);\n    return {\n      searchAreaConfig: undefined,\n      error: `AI call error: ${errorMessage}`,\n      rawResponse: errorMessage,\n    };\n  }\n\n  const { result, sectionError, mergedRect } = parsedResult;\n  if (!mergedRect) {\n    return {\n      searchAreaConfig: undefined,\n      error: sectionError,\n      rawResponse: result.content,\n      rawAssistantOutput: result.rawAssistantOutput,\n      usage: result.usage,\n    };\n  }\n\n  try {\n    const expandedRect = expandSearchArea(mergedRect, context.shotSize);\n    const originalWidth = expandedRect.width;\n    const originalHeight = expandedRect.height;\n    debugSection('expanded sectionRect %j', expandedRect);\n\n    const searchAreaConfig = await buildSearchAreaConfig({\n      context,\n      baseRect: mergedRect,\n    });\n\n    debugSection(\n      'scaled section image from %dx%d to %dx%d (scale=%d)',\n      originalWidth,\n      originalHeight,\n      searchAreaConfig.image.width,\n      searchAreaConfig.image.height,\n      searchAreaConfig.mapping.scale,\n    );\n    return {\n      searchAreaConfig,\n      error: sectionError,\n      rawResponse: result.content,\n      rawAssistantOutput: result.rawAssistantOutput,\n      usage: result.usage,\n    };\n  } catch (error) {\n    const parseErrorMessage =\n      error instanceof Error\n        ? `Failed to parse section locate result: ${error.message}`\n        : 'unknown error in section locate';\n    const errorMessage = sectionError\n      ? `${sectionError} (${parseErrorMessage})`\n      : parseErrorMessage;\n    return {\n      searchAreaConfig: undefined,\n      error: errorMessage,\n      rawResponse: result.content,\n      rawAssistantOutput: result.rawAssistantOutput,\n      usage: result.usage,\n    };\n  }\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debugSection","getDebug","AiLocateSection","options","context","sectionDescription","modelRuntime","adapter","assert","searchArea","searchAreaProtocol","resultCodec","screenshotBase64","systemPrompt","buildSearchAreaLocateSystemPrompt","userInstructionPrompt","userPromptToString","messages","preparedImage","prepareLocateModelInput","parsedResult","callAiAndParseWithRetry","retryAttempt","previousParseError","callGenerativeAI","withSemanticRetryFeedback","response","parsedLocateResult","sectionError","locateResultContext","target","references","reference","mergedRect","mergeSearchAreaResults","error","errorMessage","Error","String","message","formatLocateModelContext","Boolean","parseErrorMessage","AIResponseParseError","callError","undefined","result","expandedRect","expandSearchArea","originalWidth","originalHeight","searchAreaConfig","buildSearchAreaConfig"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;ACkBA,MAAMI,eAAeC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AAIvB,eAAeC,gBAAgBC,OAKrC;IAOC,MAAM,EAAEC,OAAO,EAAEC,kBAAkB,EAAE,GAAGF;IACxC,MAAMG,eAAeH,QAAQ,YAAY;IACzC,MAAM,EAAEI,OAAO,EAAE,GAAGD;IACpBE,IAAAA,sBAAAA,MAAAA,AAAAA,EACED,AAAwB,eAAxBA,QAAQ,MAAM,CAAC,IAAI,EACnB;IAEF,MAAME,aAAaF,QAAQ,MAAM,CAAC,UAAU;IAC5CC,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOC,YAAY;IACnB,MAAM,EAAE,UAAUC,kBAAkB,EAAEC,WAAW,EAAE,GAAGF;IACtD,MAAMG,mBAAmBR,QAAQ,UAAU,CAAC,MAAM;IAElD,MAAMS,eAAeC,AAAAA,IAAAA,yBAAAA,iCAAAA,AAAAA,EAAkC;QACrD,0BAA0BJ,mBAAmB,wBAAwB;QACrE,sBAAsBA,mBAAmB,yBAAyB,CAChEC,YAAY,UAAU;IAE1B;IAEA,MAAMI,wBAAwBL,mBAAmB,eAAe,CAC9DM,AAAAA,IAAAA,mCAAAA,kBAAAA,AAAAA,EAAmBX;IAGrB,MAAM,EAAEY,QAAQ,EAAEC,aAAa,EAAE,GAAG,MAAMC,AAAAA,IAAAA,kCAAAA,uBAAAA,AAAAA,EAAwB;QAChEN;QACA,YAAYE;QACZ,aAAa;YACX,aAAaH;YACb,OAAOR,QAAQ,QAAQ,CAAC,KAAK;YAC7B,QAAQA,QAAQ,QAAQ,CAAC,MAAM;QACjC;QACA,iBAAiBG,QAAQ,eAAe;QACxC,mBAAmBF;QACnB,yBAAyBE,QAAQ,MAAM,CAAC,uBAAuB;IACjE;IAEA,IAAIa;IAYJ,IAAI;QACFA,eAAe,MAAMC,AAAAA,IAAAA,kCAAAA,uBAAAA,AAAAA,EAAwB;YAC3C,QAAQ,CAACC,cAAcC,qBACrBC,AAAAA,IAAAA,iDAAAA,gBAAAA,AAAAA,EACEC,AAAAA,IAAAA,kCAAAA,yBAAAA,AAAAA,EAA0BR,UAAUM,qBACpCjB,cACA;oBACE,aAAaH,QAAQ,WAAW;oBAChC,4BACEO,mBAAmB,0BAA0B;oBAC/C,sBAAsBY;gBACxB;YAEJ,eAAe,CAACI;gBACd,MAAMC,qBAAqBjB,mBAAmB,gBAAgB,CAC5DgB,SAAS,OAAO,EAChBf,YAAY,UAAU;gBAExB,MAAMiB,eAAeD,mBAAmB,KAAK;gBAC7C,IAAIA,AAA4B,gBAA5BA,mBAAmB,IAAI,EACzB,OAAO;oBAAE,QAAQD;oBAAUE;gBAAa;gBAG1C,IAAI;oBACF,MAAMC,sBAAsB;wBAC1B,cAAcX,cAAc,YAAY;wBACxC,aAAaA,cAAc,WAAW;oBACxC;oBACA,MAAMY,SAASnB,YAAY,aAAa,CACtCgB,mBAAmB,MAAM,EACzBE;oBAEF,MAAME,aAAaJ,mBAAmB,UAAU,EAAE,IAAI,CAACK,YACrDrB,YAAY,aAAa,CAACqB,WAAWH;oBAEvC,MAAMI,aAAaC,AAAAA,IAAAA,wCAAAA,sBAAAA,AAAAA,EAAuBJ,QAAQC;oBAClD/B,aAAa,iBAAiBiC;oBAC9B,OAAO;wBACL,QAAQP;wBACRE;wBACAK;oBACF;gBACF,EAAE,OAAOE,OAAO;oBACd,MAAMC,eACJD,iBAAiBE,QAAQF,MAAM,OAAO,GAAGG,OAAOH;oBAClD,MAAMI,UAAU;wBACdX,gBAAgB,CAAC,gCAAgC,EAAEA,cAAc;wBACjE,CAAC,0BAA0B,EAAEQ,cAAc;wBAC3CI,IAAAA,kCAAAA,wBAAAA,AAAAA,EAAyBlC;qBAC1B,CACE,MAAM,CAACmC,SACP,IAAI,CAAC;oBACR,MAAM,IAAIJ,MAAME,SAAS;wBAAE,OAAOJ;oBAAM;gBAC1C;YACF;YACA,cAAc,CAACA,OAAOT;gBACpB,MAAMgB,oBACJP,iBAAiBE,QACb,CAAC,uCAAuC,EAAEF,MAAM,OAAO,EAAE,GACzD;gBACN,OAAO,IAAIQ,iDAAAA,oBAAoBA,CAC7BD,mBACAhB,SAAS,OAAO,EAChBA,SAAS,KAAK,EACdA,SAAS,kBAAkB,EAC3BA,SAAS,iBAAiB;YAE9B;YACA,iBAAiBpB,aAAa,MAAM,CAAC,UAAU;YAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;YACrD,aAAaH,QAAQ,WAAW;YAChC,cAAc,CAACgC;gBACbnC,aACE,+DACAmC,iBAAiBE,QAAQF,MAAM,OAAO,GAAGG,OAAOH;YAEpD;QACF;IACF,EAAE,OAAOS,WAAW;QAClB,IAAIA,qBAAqBD,iDAAAA,oBAAoBA,EAC3C,OAAO;YACL,kBAAkBE;YAClB,OAAOD,UAAU,OAAO;YACxB,aAAaA,UAAU,WAAW;YAClC,oBAAoBA,UAAU,kBAAkB;YAChD,OAAOA,UAAU,KAAK;QACxB;QAGF,MAAMR,eACJQ,qBAAqBP,QAAQO,UAAU,OAAO,GAAGN,OAAOM;QAC1D,OAAO;YACL,kBAAkBC;YAClB,OAAO,CAAC,eAAe,EAAET,cAAc;YACvC,aAAaA;QACf;IACF;IAEA,MAAM,EAAEU,MAAM,EAAElB,YAAY,EAAEK,UAAU,EAAE,GAAGb;IAC7C,IAAI,CAACa,YACH,OAAO;QACL,kBAAkBY;QAClB,OAAOjB;QACP,aAAakB,OAAO,OAAO;QAC3B,oBAAoBA,OAAO,kBAAkB;QAC7C,OAAOA,OAAO,KAAK;IACrB;IAGF,IAAI;QACF,MAAMC,eAAeC,AAAAA,IAAAA,wCAAAA,gBAAAA,AAAAA,EAAiBf,YAAY7B,QAAQ,QAAQ;QAClE,MAAM6C,gBAAgBF,aAAa,KAAK;QACxC,MAAMG,iBAAiBH,aAAa,MAAM;QAC1C/C,aAAa,2BAA2B+C;QAExC,MAAMI,mBAAmB,MAAMC,AAAAA,IAAAA,wCAAAA,qBAAAA,AAAAA,EAAsB;YACnDhD;YACA,UAAU6B;QACZ;QAEAjC,aACE,uDACAiD,eACAC,gBACAC,iBAAiB,KAAK,CAAC,KAAK,EAC5BA,iBAAiB,KAAK,CAAC,MAAM,EAC7BA,iBAAiB,OAAO,CAAC,KAAK;QAEhC,OAAO;YACLA;YACA,OAAOvB;YACP,aAAakB,OAAO,OAAO;YAC3B,oBAAoBA,OAAO,kBAAkB;YAC7C,OAAOA,OAAO,KAAK;QACrB;IACF,EAAE,OAAOX,OAAO;QACd,MAAMO,oBACJP,iBAAiBE,QACb,CAAC,uCAAuC,EAAEF,MAAM,OAAO,EAAE,GACzD;QACN,MAAMC,eAAeR,eACjB,GAAGA,aAAa,EAAE,EAAEc,kBAAkB,CAAC,CAAC,GACxCA;QACJ,OAAO;YACL,kBAAkBG;YAClB,OAAOT;YACP,aAAaU,OAAO,OAAO;YAC3B,oBAAoBA,OAAO,kBAAkB;YAC7C,OAAOA,OAAO,KAAK;QACrB;IACF;AACF"}
+{"version":3,"file":"ai-model/workflows/grounding/section-locate.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../../src/ai-model/workflows/grounding/section-locate.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { RawAssistantOutput } from '@/types';\nimport type { AIUsageInfo, Rect, UIContext } from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TUserPrompt } from '../../../common';\nimport { userPromptToString } from '../../../common';\nimport type { ModelRuntime } from '../../models';\nimport { buildSearchAreaLocateSystemPrompt } from '../../prompt/locate';\nimport {\n  AIResponseParseError,\n  callGenerativeAI,\n} from '../../service-caller/index';\nimport {\n  callAiAndParseWithRetry,\n  withSemanticRetryFeedback,\n} from '../../service-caller/semantic-retry';\nimport {\n  buildSearchAreaConfig,\n  expandSearchArea,\n  mergeSearchAreaResults,\n} from './search-area';\nimport type { SearchAreaConfig } from './types';\nimport { formatLocateModelContext, prepareLocateModelInput } from './utils';\n\nconst debugSection = getDebug('ai:grounding:section');\n\ntype SectionLocateRawResponse = Awaited<ReturnType<typeof callGenerativeAI>>;\n\nexport async function AiLocateSection(options: {\n  context: UIContext;\n  sectionDescription: TUserPrompt;\n  modelRuntime: ModelRuntime;\n  abortSignal?: AbortSignal;\n}): Promise<{\n  searchAreaConfig?: SearchAreaConfig;\n  error?: string;\n  rawResponse: string;\n  rawAssistantOutput?: RawAssistantOutput;\n  usage?: AIUsageInfo;\n}> {\n  const { context, sectionDescription } = options;\n  const modelRuntime = options.modelRuntime;\n  const { adapter } = modelRuntime;\n  assert(\n    adapter.locate.kind === 'standard',\n    'section locate requires a standard locate adapter',\n  );\n  const searchArea = adapter.locate.searchArea;\n  assert(searchArea, 'section locate requires a search area operation');\n  const { protocol: searchAreaProtocol, resultCodec } = searchArea;\n  const screenshotBase64 = context.screenshot.base64;\n\n  const systemPrompt = buildSearchAreaLocateSystemPrompt({\n    systemPromptIntroduction: searchAreaProtocol.systemPromptIntroduction,\n    responseInstructions: searchAreaProtocol.buildResponseInstructions(\n      resultCodec.promptSpec,\n    ),\n  });\n\n  const userInstructionPrompt = searchAreaProtocol.buildUserPrompt(\n    userPromptToString(sectionDescription),\n  );\n\n  const { messages, preparedImage } = await prepareLocateModelInput({\n    systemPrompt,\n    userPrompt: userInstructionPrompt,\n    locateImage: {\n      imageBase64: screenshotBase64,\n      width: context.shotSize.width,\n      height: context.shotSize.height,\n    },\n    imagePreprocess: adapter.imagePreprocess,\n    targetDescription: sectionDescription,\n    promptOrder: adapter.locate.promptOrder,\n  });\n\n  let parsedResult:\n    | {\n        result: SectionLocateRawResponse;\n        sectionError?: string;\n        mergedRect?: undefined;\n      }\n    | {\n        result: SectionLocateRawResponse;\n        sectionError?: string;\n        mergedRect: Rect;\n      };\n\n  try {\n    parsedResult = await callAiAndParseWithRetry({\n      callAi: (retryAttempt, previousParseError) =>\n        callGenerativeAI(\n          withSemanticRetryFeedback(messages, previousParseError),\n          modelRuntime,\n          {\n            abortSignal: options.abortSignal,\n            expectedJsonObjectResponse:\n              searchAreaProtocol.expectedJsonObjectResponse,\n            semanticRetryAttempt: retryAttempt,\n          },\n        ),\n      parseResponse: (response) => {\n        const parsedLocateResult = searchAreaProtocol.parseRawResponse(\n          response.content,\n          resultCodec.promptSpec,\n        );\n        const sectionError = parsedLocateResult.error;\n        if (parsedLocateResult.kind === 'not-found') {\n          return { result: response, sectionError };\n        }\n\n        try {\n          const locateResultContext = {\n            preparedSize: preparedImage.preparedSize,\n            contentSize: preparedImage.contentSize,\n          };\n          const target = resultCodec.toPixelResult(\n            parsedLocateResult.target,\n            locateResultContext,\n          );\n          const references = parsedLocateResult.references?.map((reference) =>\n            resultCodec.toPixelResult(reference, locateResultContext),\n          );\n          const mergedRect = mergeSearchAreaResults(target, references);\n          debugSection('mergedRect %j', mergedRect);\n          return {\n            result: response,\n            sectionError,\n            mergedRect,\n          };\n        } catch (error) {\n          const errorMessage =\n            error instanceof Error ? error.message : String(error);\n          const message = [\n            sectionError && `error in section locate result: ${sectionError}`,\n            `coordinate parsing error: ${errorMessage}`,\n            formatLocateModelContext(modelRuntime),\n          ]\n            .filter(Boolean)\n            .join('\\n');\n          throw new Error(message, { cause: error });\n        }\n      },\n      toParseError: (error, response) => {\n        const parseErrorMessage =\n          error instanceof Error\n            ? `Failed to parse section locate result: ${error.message}`\n            : 'unknown error in section locate';\n        return new AIResponseParseError(\n          parseErrorMessage,\n          response.content,\n          response.usage,\n          response.rawAssistantOutput,\n          response.reasoning_content,\n        );\n      },\n      parseRetryTimes: modelRuntime.config.retryCount,\n      parseRetryInterval: modelRuntime.config.retryInterval,\n      abortSignal: options.abortSignal,\n      onParseRetry: (error) => {\n        debugSection(\n          'retrying section locate after coordinate parsing failed: %s',\n          error instanceof Error ? error.message : String(error),\n        );\n      },\n    });\n  } catch (callError) {\n    if (callError instanceof AIResponseParseError) {\n      return {\n        searchAreaConfig: undefined,\n        error: callError.message,\n        rawResponse: callError.rawResponse,\n        rawAssistantOutput: callError.rawAssistantOutput,\n        usage: callError.usage,\n      };\n    }\n\n    const errorMessage =\n      callError instanceof Error ? callError.message : String(callError);\n    return {\n      searchAreaConfig: undefined,\n      error: `AI call error: ${errorMessage}`,\n      rawResponse: errorMessage,\n    };\n  }\n\n  const { result, sectionError, mergedRect } = parsedResult;\n  if (!mergedRect) {\n    return {\n      searchAreaConfig: undefined,\n      error: sectionError,\n      rawResponse: result.content,\n      rawAssistantOutput: result.rawAssistantOutput,\n      usage: result.usage,\n    };\n  }\n\n  try {\n    const expandedRect = expandSearchArea(mergedRect, context.shotSize);\n    const originalWidth = expandedRect.width;\n    const originalHeight = expandedRect.height;\n    debugSection('expanded sectionRect %j', expandedRect);\n\n    const searchAreaConfig = await buildSearchAreaConfig({\n      context,\n      baseRect: mergedRect,\n    });\n\n    debugSection(\n      'scaled section image from %dx%d to %dx%d (scale=%d)',\n      originalWidth,\n      originalHeight,\n      searchAreaConfig.image.width,\n      searchAreaConfig.image.height,\n      searchAreaConfig.mapping.scale,\n    );\n    return {\n      searchAreaConfig,\n      error: sectionError,\n      rawResponse: result.content,\n      rawAssistantOutput: result.rawAssistantOutput,\n      usage: result.usage,\n    };\n  } catch (error) {\n    const parseErrorMessage =\n      error instanceof Error\n        ? `Failed to parse section locate result: ${error.message}`\n        : 'unknown error in section locate';\n    const errorMessage = sectionError\n      ? `${sectionError} (${parseErrorMessage})`\n      : parseErrorMessage;\n    return {\n      searchAreaConfig: undefined,\n      error: errorMessage,\n      rawResponse: result.content,\n      rawAssistantOutput: result.rawAssistantOutput,\n      usage: result.usage,\n    };\n  }\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debugSection","getDebug","AiLocateSection","options","context","sectionDescription","modelRuntime","adapter","assert","searchArea","searchAreaProtocol","resultCodec","screenshotBase64","systemPrompt","buildSearchAreaLocateSystemPrompt","userInstructionPrompt","userPromptToString","messages","preparedImage","prepareLocateModelInput","parsedResult","callAiAndParseWithRetry","retryAttempt","previousParseError","callGenerativeAI","withSemanticRetryFeedback","response","parsedLocateResult","sectionError","locateResultContext","target","references","reference","mergedRect","mergeSearchAreaResults","error","errorMessage","Error","String","message","formatLocateModelContext","Boolean","parseErrorMessage","AIResponseParseError","callError","undefined","result","expandedRect","expandSearchArea","originalWidth","originalHeight","searchAreaConfig","buildSearchAreaConfig"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;ACkBA,MAAMI,eAAeC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AAIvB,eAAeC,gBAAgBC,OAKrC;IAOC,MAAM,EAAEC,OAAO,EAAEC,kBAAkB,EAAE,GAAGF;IACxC,MAAMG,eAAeH,QAAQ,YAAY;IACzC,MAAM,EAAEI,OAAO,EAAE,GAAGD;IACpBE,IAAAA,sBAAAA,MAAAA,AAAAA,EACED,AAAwB,eAAxBA,QAAQ,MAAM,CAAC,IAAI,EACnB;IAEF,MAAME,aAAaF,QAAQ,MAAM,CAAC,UAAU;IAC5CC,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOC,YAAY;IACnB,MAAM,EAAE,UAAUC,kBAAkB,EAAEC,WAAW,EAAE,GAAGF;IACtD,MAAMG,mBAAmBR,QAAQ,UAAU,CAAC,MAAM;IAElD,MAAMS,eAAeC,AAAAA,IAAAA,yBAAAA,iCAAAA,AAAAA,EAAkC;QACrD,0BAA0BJ,mBAAmB,wBAAwB;QACrE,sBAAsBA,mBAAmB,yBAAyB,CAChEC,YAAY,UAAU;IAE1B;IAEA,MAAMI,wBAAwBL,mBAAmB,eAAe,CAC9DM,AAAAA,IAAAA,mCAAAA,kBAAAA,AAAAA,EAAmBX;IAGrB,MAAM,EAAEY,QAAQ,EAAEC,aAAa,EAAE,GAAG,MAAMC,AAAAA,IAAAA,kCAAAA,uBAAAA,AAAAA,EAAwB;QAChEN;QACA,YAAYE;QACZ,aAAa;YACX,aAAaH;YACb,OAAOR,QAAQ,QAAQ,CAAC,KAAK;YAC7B,QAAQA,QAAQ,QAAQ,CAAC,MAAM;QACjC;QACA,iBAAiBG,QAAQ,eAAe;QACxC,mBAAmBF;QACnB,aAAaE,QAAQ,MAAM,CAAC,WAAW;IACzC;IAEA,IAAIa;IAYJ,IAAI;QACFA,eAAe,MAAMC,AAAAA,IAAAA,kCAAAA,uBAAAA,AAAAA,EAAwB;YAC3C,QAAQ,CAACC,cAAcC,qBACrBC,AAAAA,IAAAA,iDAAAA,gBAAAA,AAAAA,EACEC,AAAAA,IAAAA,kCAAAA,yBAAAA,AAAAA,EAA0BR,UAAUM,qBACpCjB,cACA;oBACE,aAAaH,QAAQ,WAAW;oBAChC,4BACEO,mBAAmB,0BAA0B;oBAC/C,sBAAsBY;gBACxB;YAEJ,eAAe,CAACI;gBACd,MAAMC,qBAAqBjB,mBAAmB,gBAAgB,CAC5DgB,SAAS,OAAO,EAChBf,YAAY,UAAU;gBAExB,MAAMiB,eAAeD,mBAAmB,KAAK;gBAC7C,IAAIA,AAA4B,gBAA5BA,mBAAmB,IAAI,EACzB,OAAO;oBAAE,QAAQD;oBAAUE;gBAAa;gBAG1C,IAAI;oBACF,MAAMC,sBAAsB;wBAC1B,cAAcX,cAAc,YAAY;wBACxC,aAAaA,cAAc,WAAW;oBACxC;oBACA,MAAMY,SAASnB,YAAY,aAAa,CACtCgB,mBAAmB,MAAM,EACzBE;oBAEF,MAAME,aAAaJ,mBAAmB,UAAU,EAAE,IAAI,CAACK,YACrDrB,YAAY,aAAa,CAACqB,WAAWH;oBAEvC,MAAMI,aAAaC,AAAAA,IAAAA,wCAAAA,sBAAAA,AAAAA,EAAuBJ,QAAQC;oBAClD/B,aAAa,iBAAiBiC;oBAC9B,OAAO;wBACL,QAAQP;wBACRE;wBACAK;oBACF;gBACF,EAAE,OAAOE,OAAO;oBACd,MAAMC,eACJD,iBAAiBE,QAAQF,MAAM,OAAO,GAAGG,OAAOH;oBAClD,MAAMI,UAAU;wBACdX,gBAAgB,CAAC,gCAAgC,EAAEA,cAAc;wBACjE,CAAC,0BAA0B,EAAEQ,cAAc;wBAC3CI,IAAAA,kCAAAA,wBAAAA,AAAAA,EAAyBlC;qBAC1B,CACE,MAAM,CAACmC,SACP,IAAI,CAAC;oBACR,MAAM,IAAIJ,MAAME,SAAS;wBAAE,OAAOJ;oBAAM;gBAC1C;YACF;YACA,cAAc,CAACA,OAAOT;gBACpB,MAAMgB,oBACJP,iBAAiBE,QACb,CAAC,uCAAuC,EAAEF,MAAM,OAAO,EAAE,GACzD;gBACN,OAAO,IAAIQ,iDAAAA,oBAAoBA,CAC7BD,mBACAhB,SAAS,OAAO,EAChBA,SAAS,KAAK,EACdA,SAAS,kBAAkB,EAC3BA,SAAS,iBAAiB;YAE9B;YACA,iBAAiBpB,aAAa,MAAM,CAAC,UAAU;YAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;YACrD,aAAaH,QAAQ,WAAW;YAChC,cAAc,CAACgC;gBACbnC,aACE,+DACAmC,iBAAiBE,QAAQF,MAAM,OAAO,GAAGG,OAAOH;YAEpD;QACF;IACF,EAAE,OAAOS,WAAW;QAClB,IAAIA,qBAAqBD,iDAAAA,oBAAoBA,EAC3C,OAAO;YACL,kBAAkBE;YAClB,OAAOD,UAAU,OAAO;YACxB,aAAaA,UAAU,WAAW;YAClC,oBAAoBA,UAAU,kBAAkB;YAChD,OAAOA,UAAU,KAAK;QACxB;QAGF,MAAMR,eACJQ,qBAAqBP,QAAQO,UAAU,OAAO,GAAGN,OAAOM;QAC1D,OAAO;YACL,kBAAkBC;YAClB,OAAO,CAAC,eAAe,EAAET,cAAc;YACvC,aAAaA;QACf;IACF;IAEA,MAAM,EAAEU,MAAM,EAAElB,YAAY,EAAEK,UAAU,EAAE,GAAGb;IAC7C,IAAI,CAACa,YACH,OAAO;QACL,kBAAkBY;QAClB,OAAOjB;QACP,aAAakB,OAAO,OAAO;QAC3B,oBAAoBA,OAAO,kBAAkB;QAC7C,OAAOA,OAAO,KAAK;IACrB;IAGF,IAAI;QACF,MAAMC,eAAeC,AAAAA,IAAAA,wCAAAA,gBAAAA,AAAAA,EAAiBf,YAAY7B,QAAQ,QAAQ;QAClE,MAAM6C,gBAAgBF,aAAa,KAAK;QACxC,MAAMG,iBAAiBH,aAAa,MAAM;QAC1C/C,aAAa,2BAA2B+C;QAExC,MAAMI,mBAAmB,MAAMC,AAAAA,IAAAA,wCAAAA,qBAAAA,AAAAA,EAAsB;YACnDhD;YACA,UAAU6B;QACZ;QAEAjC,aACE,uDACAiD,eACAC,gBACAC,iBAAiB,KAAK,CAAC,KAAK,EAC5BA,iBAAiB,KAAK,CAAC,MAAM,EAC7BA,iBAAiB,OAAO,CAAC,KAAK;QAEhC,OAAO;YACLA;YACA,OAAOvB;YACP,aAAakB,OAAO,OAAO;YAC3B,oBAAoBA,OAAO,kBAAkB;YAC7C,OAAOA,OAAO,KAAK;QACrB;IACF,EAAE,OAAOX,OAAO;QACd,MAAMO,oBACJP,iBAAiBE,QACb,CAAC,uCAAuC,EAAEF,MAAM,OAAO,EAAE,GACzD;QACN,MAAMC,eAAeR,eACjB,GAAGA,aAAa,EAAE,EAAEc,kBAAkB,CAAC,CAAC,GACxCA;QACJ,OAAO;YACL,kBAAkBG;YAClB,OAAOT;YACP,aAAaU,OAAO,OAAO;YAC3B,oBAAoBA,OAAO,kBAAkB;YAC7C,OAAOA,OAAO,KAAK;QACrB;IACF;AACF"}

package/dist/lib/ai-model/workflows/grounding/utils.js

@@ -29,8 +29,9 @@ __webpack_require__.d(__webpack_exports__, {
 });
 const external_common_js_namespaceObject = require("../../../common.js");
 const image_preprocess_js_namespaceObject = require("../../model-adapter/image-preprocess.js");
+const locate_prompt_order_js_namespaceObject = require("../../model-adapter/locate-prompt-order.js");
 const multimodal_prompt_js_namespaceObject = require("../../shared/multimodal-prompt.js");
-async function prepareLocateModelInput({ locateImage, targetDescription, systemPrompt, userPrompt, imagePreprocess, userMessageContentOrder }) {
+async function prepareLocateModelInput({ locateImage, targetDescription, systemPrompt, userPrompt, imagePreprocess, promptOrder }) {
     const preparedImage = await (0, image_preprocess_js_namespaceObject.prepareModelImage)({
         imageBase64: locateImage.imageBase64,
         width: locateImage.width,
@@ -38,33 +39,14 @@ async function prepareLocateModelInput({ locateImage, targetDescription, systemP
         policy: imagePreprocess
     });
     const referenceImageMessages = await (0, multimodal_prompt_js_namespaceObject.multimodalPromptToMessages)((0, external_common_js_namespaceObject.userPromptToMultimodalPrompt)(targetDescription));
-    const imageContent = {
-        type: 'image',
-        url: preparedImage.imageBase64
-    };
-    const promptContent = {
-        type: 'text',
-        text: userPrompt
-    };
     return {
         preparedImage,
-        messages: [
-            {
-                role: 'system',
-                content: systemPrompt
-            },
-            {
-                role: 'user',
-                content: 'prompt-first' === userMessageContentOrder ? [
-                    promptContent,
-                    imageContent
-                ] : [
-                    imageContent,
-                    promptContent
-                ]
-            },
-            ...referenceImageMessages
-        ]
+        messages: (0, locate_prompt_order_js_namespaceObject.buildLocateMessages)(promptOrder, {
+            instructions: systemPrompt,
+            screenshot: preparedImage.imageBase64,
+            task: userPrompt,
+            references: referenceImageMessages
+        })
     };
 }
 function formatLocateModelContext(modelRuntime) {

package/dist/lib/ai-model/workflows/grounding/utils.js.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/workflows/grounding/utils.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../../src/ai-model/workflows/grounding/utils.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type {\n  ConversationSystemMessage,\n  ConversationUserMessage,\n  MessageContent,\n} from '@/ai-model/service-caller/types';\nimport { type TUserPrompt, userPromptToMultimodalPrompt } from '@/common';\nimport {\n  type ImagePreprocessPolicy,\n  type PreparedModelImage,\n  prepareModelImage,\n} from '../../model-adapter/image-preprocess';\nimport type { LocateUserMessageContentOrder } from '../../model-adapter/types';\nimport type { ModelRuntime } from '../../models';\nimport { multimodalPromptToMessages } from '../../shared/multimodal-prompt';\n\nexport type GroundingAIArgs = [\n  ConversationSystemMessage,\n  ...ConversationUserMessage[],\n];\n\nexport async function prepareLocateModelInput({\n  locateImage,\n  targetDescription,\n  systemPrompt,\n  userPrompt,\n  imagePreprocess,\n  userMessageContentOrder,\n}: {\n  locateImage: {\n    imageBase64: string;\n    width: number;\n    height: number;\n  };\n  targetDescription: TUserPrompt;\n  systemPrompt: string;\n  userPrompt: string;\n  imagePreprocess: ImagePreprocessPolicy;\n  userMessageContentOrder: LocateUserMessageContentOrder;\n}): Promise<{\n  messages: GroundingAIArgs;\n  preparedImage: PreparedModelImage;\n}> {\n  const preparedImage = await prepareModelImage({\n    imageBase64: locateImage.imageBase64,\n    width: locateImage.width,\n    height: locateImage.height,\n    policy: imagePreprocess,\n  });\n  const referenceImageMessages = await multimodalPromptToMessages(\n    userPromptToMultimodalPrompt(targetDescription),\n  );\n  const imageContent: MessageContent = {\n    type: 'image',\n    url: preparedImage.imageBase64,\n  };\n  const promptContent: MessageContent = {\n    type: 'text',\n    text: userPrompt,\n  };\n\n  return {\n    preparedImage,\n    messages: [\n      { role: 'system', content: systemPrompt },\n      {\n        role: 'user',\n        content:\n          userMessageContentOrder === 'prompt-first'\n            ? [promptContent, imageContent]\n            : [imageContent, promptContent],\n      },\n      ...referenceImageMessages,\n    ],\n  };\n}\n\nexport function formatLocateModelContext(modelRuntime: ModelRuntime): string {\n  const { modelFamily, modelName } = modelRuntime.config;\n  return `modelName=${modelName ?? 'unset'} modelFamily=${modelFamily ?? 'unset'}`;\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","prepareLocateModelInput","locateImage","targetDescription","systemPrompt","userPrompt","imagePreprocess","userMessageContentOrder","preparedImage","prepareModelImage","referenceImageMessages","multimodalPromptToMessages","userPromptToMultimodalPrompt","imageContent","promptContent","formatLocateModelContext","modelRuntime","modelFamily","modelName"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;ACcO,eAAeI,wBAAwB,EAC5CC,WAAW,EACXC,iBAAiB,EACjBC,YAAY,EACZC,UAAU,EACVC,eAAe,EACfC,uBAAuB,EAYxB;IAIC,MAAMC,gBAAgB,MAAMC,AAAAA,IAAAA,oCAAAA,iBAAAA,AAAAA,EAAkB;QAC5C,aAAaP,YAAY,WAAW;QACpC,OAAOA,YAAY,KAAK;QACxB,QAAQA,YAAY,MAAM;QAC1B,QAAQI;IACV;IACA,MAAMI,yBAAyB,MAAMC,AAAAA,IAAAA,qCAAAA,0BAAAA,AAAAA,EACnCC,AAAAA,IAAAA,mCAAAA,4BAAAA,AAAAA,EAA6BT;IAE/B,MAAMU,eAA+B;QACnC,MAAM;QACN,KAAKL,cAAc,WAAW;IAChC;IACA,MAAMM,gBAAgC;QACpC,MAAM;QACN,MAAMT;IACR;IAEA,OAAO;QACLG;QACA,UAAU;YACR;gBAAE,MAAM;gBAAU,SAASJ;YAAa;YACxC;gBACE,MAAM;gBACN,SACEG,AAA4B,mBAA5BA,0BACI;oBAACO;oBAAeD;iBAAa,GAC7B;oBAACA;oBAAcC;iBAAc;YACrC;eACGJ;SACJ;IACH;AACF;AAEO,SAASK,yBAAyBC,YAA0B;IACjE,MAAM,EAAEC,WAAW,EAAEC,SAAS,EAAE,GAAGF,aAAa,MAAM;IACtD,OAAO,CAAC,UAAU,EAAEE,aAAa,QAAQ,aAAa,EAAED,eAAe,SAAS;AAClF"}
+{"version":3,"file":"ai-model/workflows/grounding/utils.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../../src/ai-model/workflows/grounding/utils.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { type TUserPrompt, userPromptToMultimodalPrompt } from '@/common';\nimport {\n  type ImagePreprocessPolicy,\n  type PreparedModelImage,\n  prepareModelImage,\n} from '../../model-adapter/image-preprocess';\nimport {\n  type LocateMessages,\n  type LocatePromptOrder,\n  buildLocateMessages,\n} from '../../model-adapter/locate-prompt-order';\nimport type { ModelRuntime } from '../../models';\nimport { multimodalPromptToMessages } from '../../shared/multimodal-prompt';\n\nexport async function prepareLocateModelInput({\n  locateImage,\n  targetDescription,\n  systemPrompt,\n  userPrompt,\n  imagePreprocess,\n  promptOrder,\n}: {\n  locateImage: {\n    imageBase64: string;\n    width: number;\n    height: number;\n  };\n  targetDescription: TUserPrompt;\n  systemPrompt: string;\n  userPrompt: string;\n  imagePreprocess: ImagePreprocessPolicy;\n  promptOrder: LocatePromptOrder;\n}): Promise<{\n  messages: LocateMessages;\n  preparedImage: PreparedModelImage;\n}> {\n  const preparedImage = await prepareModelImage({\n    imageBase64: locateImage.imageBase64,\n    width: locateImage.width,\n    height: locateImage.height,\n    policy: imagePreprocess,\n  });\n  const referenceImageMessages = await multimodalPromptToMessages(\n    userPromptToMultimodalPrompt(targetDescription),\n  );\n  return {\n    preparedImage,\n    messages: buildLocateMessages(promptOrder, {\n      instructions: systemPrompt,\n      screenshot: preparedImage.imageBase64,\n      task: userPrompt,\n      references: referenceImageMessages,\n    }),\n  };\n}\n\nexport function formatLocateModelContext(modelRuntime: ModelRuntime): string {\n  const { modelFamily, modelName } = modelRuntime.config;\n  return `modelName=${modelName ?? 'unset'} modelFamily=${modelFamily ?? 'unset'}`;\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","prepareLocateModelInput","locateImage","targetDescription","systemPrompt","userPrompt","imagePreprocess","promptOrder","preparedImage","prepareModelImage","referenceImageMessages","multimodalPromptToMessages","userPromptToMultimodalPrompt","buildLocateMessages","formatLocateModelContext","modelRuntime","modelFamily","modelName"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;ACQO,eAAeI,wBAAwB,EAC5CC,WAAW,EACXC,iBAAiB,EACjBC,YAAY,EACZC,UAAU,EACVC,eAAe,EACfC,WAAW,EAYZ;IAIC,MAAMC,gBAAgB,MAAMC,AAAAA,IAAAA,oCAAAA,iBAAAA,AAAAA,EAAkB;QAC5C,aAAaP,YAAY,WAAW;QACpC,OAAOA,YAAY,KAAK;QACxB,QAAQA,YAAY,MAAM;QAC1B,QAAQI;IACV;IACA,MAAMI,yBAAyB,MAAMC,AAAAA,IAAAA,qCAAAA,0BAAAA,AAAAA,EACnCC,AAAAA,IAAAA,mCAAAA,4BAAAA,AAAAA,EAA6BT;IAE/B,OAAO;QACLK;QACA,UAAUK,AAAAA,IAAAA,uCAAAA,mBAAAA,AAAAA,EAAoBN,aAAa;YACzC,cAAcH;YACd,YAAYI,cAAc,WAAW;YACrC,MAAMH;YACN,YAAYK;QACd;IACF;AACF;AAEO,SAASI,yBAAyBC,YAA0B;IACjE,MAAM,EAAEC,WAAW,EAAEC,SAAS,EAAE,GAAGF,aAAa,MAAM;IACtD,OAAO,CAAC,UAAU,EAAEE,aAAa,QAAQ,aAAa,EAAED,eAAe,SAAS;AAClF"}